src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2016 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOOL, BOOL, BOOL *);
  17 static int_eximarith_t expanded_string_integer(const uschar *, BOOL);
  18
  19 #ifdef STAND_ALONE
  20 #ifndef SUPPORT_CRYPTEQ
  21 #define SUPPORT_CRYPTEQ
  22 #endif
  23 #endif
  24
  25 #ifdef LOOKUP_LDAP
  26 #include "lookups/ldap.h"
  27 #endif
  28
  29 #ifdef SUPPORT_CRYPTEQ
  30 #ifdef CRYPT_H
  31 #include <crypt.h>
  32 #endif
  33 #ifndef HAVE_CRYPT16
  34 extern char* crypt16(char*, char*);
  35 #endif
  36 #endif
  37
  38 /* The handling of crypt16() is a mess. I will record below the analysis of the
  39 mess that was sent to me. We decided, however, to make changing this very low
  40 priority, because in practice people are moving away from the crypt()
  41 algorithms nowadays, so it doesn't seem worth it.
  42
  43 <quote>
  44 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  45 the first 8 characters of the password using a 20-round version of crypt
  46 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  47 or an empty block if the password is less than 9 characters, using a
  48 20-round version of crypt and the same salt as was used for the first
  49 block.  Charaters after the first 16 are ignored.  It always generates
  50 a 16-byte hash, which is expressed together with the salt as a string
  51 of 24 base 64 digits.  Here are some links to peruse:
  52
  53         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  54         http://seclists.org/bugtraq/1999/Mar/0076.html
  55
  56 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  57 and OSF/1.  This is the same as the standard crypt if given a password
  58 of 8 characters or less.  If given more, it first does the same as crypt
  59 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  60 using as salt the first two base 64 digits from the first hash block.
  61 If the password is more than 16 characters then it crypts the 17th to 24th
  62 characters using as salt the first two base 64 digits from the second hash
  63 block.  And so on: I've seen references to it cutting off the password at
  64 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  65
  66         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  67         http://seclists.org/bugtraq/1999/Mar/0109.html
  68         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  69              TET1_html/sec.c222.html#no_id_208
  70
  71 Exim has something it calls "crypt16".  It will either use a native
  72 crypt16 or its own implementation.  A native crypt16 will presumably
  73 be the one that I called "crypt16" above.  The internal "crypt16"
  74 function, however, is a two-block-maximum implementation of what I called
  75 "bigcrypt".  The documentation matches the internal code.
  76
  77 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  78 that crypt16 and bigcrypt were different things.
  79
  80 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  81 to whatever it is using under that name.  This unfortunately sets a
  82 precedent for using "{crypt16}" to identify two incompatible algorithms
  83 whose output can't be distinguished.  With "{crypt16}" thus rendered
  84 ambiguous, I suggest you deprecate it and invent two new identifiers
  85 for the two algorithms.
  86
  87 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  88 of the password separately means they can be cracked separately, so
  89 the double-length hash only doubles the cracking effort instead of
  90 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  91 bcrypt ({CRYPT}$2a$).
  92 </quote>
  93 */
  94
  95
  96
  97 /*************************************************
  98 *            Local statics and tables            *
  99 *************************************************/
 100
 101 /* Table of item names, and corresponding switch numbers. The names must be in
 102 alphabetical order. */
 103
 104 static uschar *item_table[] = {
 105   US"acl",
 106   US"certextract",
 107   US"dlfunc",
 108   US"env",
 109   US"extract",
 110   US"filter",
 111   US"hash",
 112   US"hmac",
 113   US"if",
 114 #ifdef SUPPORT_I18N
 115   US"imapfolder",
 116 #endif
 117   US"length",
 118   US"listextract",
 119   US"lookup",
 120   US"map",
 121   US"nhash",
 122   US"perl",
 123   US"prvs",
 124   US"prvscheck",
 125   US"readfile",
 126   US"readsocket",
 127   US"reduce",
 128   US"run",
 129   US"sg",
 130   US"sort",
 131   US"substr",
 132   US"tr" };
 133
 134 enum {
 135   EITEM_ACL,
 136   EITEM_CERTEXTRACT,
 137   EITEM_DLFUNC,
 138   EITEM_ENV,
 139   EITEM_EXTRACT,
 140   EITEM_FILTER,
 141   EITEM_HASH,
 142   EITEM_HMAC,
 143   EITEM_IF,
 144 #ifdef SUPPORT_I18N
 145   EITEM_IMAPFOLDER,
 146 #endif
 147   EITEM_LENGTH,
 148   EITEM_LISTEXTRACT,
 149   EITEM_LOOKUP,
 150   EITEM_MAP,
 151   EITEM_NHASH,
 152   EITEM_PERL,
 153   EITEM_PRVS,
 154   EITEM_PRVSCHECK,
 155   EITEM_READFILE,
 156   EITEM_READSOCK,
 157   EITEM_REDUCE,
 158   EITEM_RUN,
 159   EITEM_SG,
 160   EITEM_SORT,
 161   EITEM_SUBSTR,
 162   EITEM_TR };
 163
 164 /* Tables of operator names, and corresponding switch numbers. The names must be
 165 in alphabetical order. There are two tables, because underscore is used in some
 166 cases to introduce arguments, whereas for other it is part of the name. This is
 167 an historical mis-design. */
 168
 169 static uschar *op_table_underscore[] = {
 170   US"from_utf8",
 171   US"local_part",
 172   US"quote_local_part",
 173   US"reverse_ip",
 174   US"time_eval",
 175   US"time_interval"
 176 #ifdef SUPPORT_I18N
 177  ,US"utf8_domain_from_alabel",
 178   US"utf8_domain_to_alabel",
 179   US"utf8_localpart_from_alabel",
 180   US"utf8_localpart_to_alabel"
 181 #endif
 182   };
 183
 184 enum {
 185   EOP_FROM_UTF8,
 186   EOP_LOCAL_PART,
 187   EOP_QUOTE_LOCAL_PART,
 188   EOP_REVERSE_IP,
 189   EOP_TIME_EVAL,
 190   EOP_TIME_INTERVAL
 191 #ifdef SUPPORT_I18N
 192  ,EOP_UTF8_DOMAIN_FROM_ALABEL,
 193   EOP_UTF8_DOMAIN_TO_ALABEL,
 194   EOP_UTF8_LOCALPART_FROM_ALABEL,
 195   EOP_UTF8_LOCALPART_TO_ALABEL
 196 #endif
 197   };
 198
 199 static uschar *op_table_main[] = {
 200   US"address",
 201   US"addresses",
 202   US"base62",
 203   US"base62d",
 204   US"base64",
 205   US"base64d",
 206   US"domain",
 207   US"escape",
 208   US"eval",
 209   US"eval10",
 210   US"expand",
 211   US"h",
 212   US"hash",
 213   US"hex2b64",
 214   US"hexquote",
 215   US"ipv6denorm",
 216   US"ipv6norm",
 217   US"l",
 218   US"lc",
 219   US"length",
 220   US"listcount",
 221   US"listnamed",
 222   US"mask",
 223   US"md5",
 224   US"nh",
 225   US"nhash",
 226   US"quote",
 227   US"randint",
 228   US"rfc2047",
 229   US"rfc2047d",
 230   US"rxquote",
 231   US"s",
 232   US"sha1",
 233   US"sha256",
 234   US"sha3",
 235   US"stat",
 236   US"str2b64",
 237   US"strlen",
 238   US"substr",
 239   US"uc",
 240   US"utf8clean" };
 241
 242 enum {
 243   EOP_ADDRESS =  nelem(op_table_underscore),
 244   EOP_ADDRESSES,
 245   EOP_BASE62,
 246   EOP_BASE62D,
 247   EOP_BASE64,
 248   EOP_BASE64D,
 249   EOP_DOMAIN,
 250   EOP_ESCAPE,
 251   EOP_EVAL,
 252   EOP_EVAL10,
 253   EOP_EXPAND,
 254   EOP_H,
 255   EOP_HASH,
 256   EOP_HEX2B64,
 257   EOP_HEXQUOTE,
 258   EOP_IPV6DENORM,
 259   EOP_IPV6NORM,
 260   EOP_L,
 261   EOP_LC,
 262   EOP_LENGTH,
 263   EOP_LISTCOUNT,
 264   EOP_LISTNAMED,
 265   EOP_MASK,
 266   EOP_MD5,
 267   EOP_NH,
 268   EOP_NHASH,
 269   EOP_QUOTE,
 270   EOP_RANDINT,
 271   EOP_RFC2047,
 272   EOP_RFC2047D,
 273   EOP_RXQUOTE,
 274   EOP_S,
 275   EOP_SHA1,
 276   EOP_SHA256,
 277   EOP_SHA3,
 278   EOP_STAT,
 279   EOP_STR2B64,
 280   EOP_STRLEN,
 281   EOP_SUBSTR,
 282   EOP_UC,
 283   EOP_UTF8CLEAN };
 284
 285
 286 /* Table of condition names, and corresponding switch numbers. The names must
 287 be in alphabetical order. */
 288
 289 static uschar *cond_table[] = {
 290   US"<",
 291   US"<=",
 292   US"=",
 293   US"==",     /* Backward compatibility */
 294   US">",
 295   US">=",
 296   US"acl",
 297   US"and",
 298   US"bool",
 299   US"bool_lax",
 300   US"crypteq",
 301   US"def",
 302   US"eq",
 303   US"eqi",
 304   US"exists",
 305   US"first_delivery",
 306   US"forall",
 307   US"forany",
 308   US"ge",
 309   US"gei",
 310   US"gt",
 311   US"gti",
 312   US"inlist",
 313   US"inlisti",
 314   US"isip",
 315   US"isip4",
 316   US"isip6",
 317   US"ldapauth",
 318   US"le",
 319   US"lei",
 320   US"lt",
 321   US"lti",
 322   US"match",
 323   US"match_address",
 324   US"match_domain",
 325   US"match_ip",
 326   US"match_local_part",
 327   US"or",
 328   US"pam",
 329   US"pwcheck",
 330   US"queue_running",
 331   US"radius",
 332   US"saslauthd"
 333 };
 334
 335 enum {
 336   ECOND_NUM_L,
 337   ECOND_NUM_LE,
 338   ECOND_NUM_E,
 339   ECOND_NUM_EE,
 340   ECOND_NUM_G,
 341   ECOND_NUM_GE,
 342   ECOND_ACL,
 343   ECOND_AND,
 344   ECOND_BOOL,
 345   ECOND_BOOL_LAX,
 346   ECOND_CRYPTEQ,
 347   ECOND_DEF,
 348   ECOND_STR_EQ,
 349   ECOND_STR_EQI,
 350   ECOND_EXISTS,
 351   ECOND_FIRST_DELIVERY,
 352   ECOND_FORALL,
 353   ECOND_FORANY,
 354   ECOND_STR_GE,
 355   ECOND_STR_GEI,
 356   ECOND_STR_GT,
 357   ECOND_STR_GTI,
 358   ECOND_INLIST,
 359   ECOND_INLISTI,
 360   ECOND_ISIP,
 361   ECOND_ISIP4,
 362   ECOND_ISIP6,
 363   ECOND_LDAPAUTH,
 364   ECOND_STR_LE,
 365   ECOND_STR_LEI,
 366   ECOND_STR_LT,
 367   ECOND_STR_LTI,
 368   ECOND_MATCH,
 369   ECOND_MATCH_ADDRESS,
 370   ECOND_MATCH_DOMAIN,
 371   ECOND_MATCH_IP,
 372   ECOND_MATCH_LOCAL_PART,
 373   ECOND_OR,
 374   ECOND_PAM,
 375   ECOND_PWCHECK,
 376   ECOND_QUEUE_RUNNING,
 377   ECOND_RADIUS,
 378   ECOND_SASLAUTHD
 379 };
 380
 381
 382 /* Types of table entry */
 383
 384 enum vtypes {
 385   vtype_int,            /* value is address of int */
 386   vtype_filter_int,     /* ditto, but recognized only when filtering */
 387   vtype_ino,            /* value is address of ino_t (not always an int) */
 388   vtype_uid,            /* value is address of uid_t (not always an int) */
 389   vtype_gid,            /* value is address of gid_t (not always an int) */
 390   vtype_bool,           /* value is address of bool */
 391   vtype_stringptr,      /* value is address of pointer to string */
 392   vtype_msgbody,        /* as stringptr, but read when first required */
 393   vtype_msgbody_end,    /* ditto, the end of the message */
 394   vtype_msgheaders,     /* the message's headers, processed */
 395   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 396   vtype_localpart,      /* extract local part from string */
 397   vtype_domain,         /* extract domain from string */
 398   vtype_string_func,    /* value is string returned by given function */
 399   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 400   vtype_tode,           /* value not used; generate tod in epoch format */
 401   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 402   vtype_todf,           /* value not used; generate full tod */
 403   vtype_todl,           /* value not used; generate log tod */
 404   vtype_todlf,          /* value not used; generate log file datestamp tod */
 405   vtype_todzone,        /* value not used; generate time zone only */
 406   vtype_todzulu,        /* value not used; generate zulu tod */
 407   vtype_reply,          /* value not used; get reply from headers */
 408   vtype_pid,            /* value not used; result is pid */
 409   vtype_host_lookup,    /* value not used; get host name */
 410   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 411   vtype_pspace,         /* partition space; value is T/F for spool/log */
 412   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 413   vtype_cert            /* SSL certificate */
 414   #ifndef DISABLE_DKIM
 415   ,vtype_dkim           /* Lookup of value in DKIM signature */
 416   #endif
 417 };
 418
 419 /* Type for main variable table */
 420
 421 typedef struct {
 422   const char *name;
 423   enum vtypes type;
 424   void       *value;
 425 } var_entry;
 426
 427 /* Type for entries pointing to address/length pairs. Not currently
 428 in use. */
 429
 430 typedef struct {
 431   uschar **address;
 432   int  *length;
 433 } alblock;
 434
 435 static uschar * fn_recipients(void);
 436
 437 /* This table must be kept in alphabetical order. */
 438
 439 static var_entry var_table[] = {
 440   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 441      they will be confused with user-creatable ACL variables. */
 442   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 443   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 444   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 445   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 446   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 447   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 448   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 449   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 450   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 451   { "acl_narg",            vtype_int,         &acl_narg },
 452   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 453   { "address_data",        vtype_stringptr,   &deliver_address_data },
 454   { "address_file",        vtype_stringptr,   &address_file },
 455   { "address_pipe",        vtype_stringptr,   &address_pipe },
 456   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 457   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 458   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 459   { "authentication_failed",vtype_int,        &authentication_failed },
 460 #ifdef WITH_CONTENT_SCAN
 461   { "av_failed",           vtype_int,         &av_failed },
 462 #endif
 463 #ifdef EXPERIMENTAL_BRIGHTMAIL
 464   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 465   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 466   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 467   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 468 #endif
 469   { "body_linecount",      vtype_int,         &body_linecount },
 470   { "body_zerocount",      vtype_int,         &body_zerocount },
 471   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 472   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 473   { "caller_gid",          vtype_gid,         &real_gid },
 474   { "caller_uid",          vtype_uid,         &real_uid },
 475   { "callout_address",     vtype_stringptr,   &callout_address },
 476   { "compile_date",        vtype_stringptr,   &version_date },
 477   { "compile_number",      vtype_stringptr,   &version_cnumber },
 478   { "config_dir",          vtype_stringptr,   &config_main_directory },
 479   { "config_file",         vtype_stringptr,   &config_main_filename },
 480   { "csa_status",          vtype_stringptr,   &csa_status },
 481 #ifdef EXPERIMENTAL_DCC
 482   { "dcc_header",          vtype_stringptr,   &dcc_header },
 483   { "dcc_result",          vtype_stringptr,   &dcc_result },
 484 #endif
 485 #ifndef DISABLE_DKIM
 486   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 487   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 488   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 489   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 490   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 491   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 492   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 493   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 494   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 495   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 496   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 497   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 498   { "dkim_key_length",     vtype_int,         &dkim_key_length },
 499   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 500   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 501   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 502   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 503   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 504   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 505   { "dkim_verify_reason",  vtype_dkim,        (void *)DKIM_VERIFY_REASON },
 506   { "dkim_verify_status",  vtype_dkim,        (void *)DKIM_VERIFY_STATUS},
 507 #endif
 508 #ifdef EXPERIMENTAL_DMARC
 509   { "dmarc_ar_header",     vtype_stringptr,   &dmarc_ar_header },
 510   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 511   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 512   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 513   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 514 #endif
 515   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 516   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 517   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 518   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 519   { "domain",              vtype_stringptr,   &deliver_domain },
 520   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 521 #ifndef DISABLE_EVENT
 522   { "event_data",          vtype_stringptr,   &event_data },
 523
 524   /*XXX want to use generic vars for as many of these as possible*/
 525   { "event_defer_errno",   vtype_int,         &event_defer_errno },
 526
 527   { "event_name",          vtype_stringptr,   &event_name },
 528 #endif
 529   { "exim_gid",            vtype_gid,         &exim_gid },
 530   { "exim_path",           vtype_stringptr,   &exim_path },
 531   { "exim_uid",            vtype_uid,         &exim_uid },
 532   { "exim_version",        vtype_stringptr,   &version_string },
 533   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 534   { "home",                vtype_stringptr,   &deliver_home },
 535   { "host",                vtype_stringptr,   &deliver_host },
 536   { "host_address",        vtype_stringptr,   &deliver_host_address },
 537   { "host_data",           vtype_stringptr,   &host_data },
 538   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 539   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 540   { "host_port",           vtype_int,         &deliver_host_port },
 541   { "initial_cwd",         vtype_stringptr,   &initial_cwd },
 542   { "inode",               vtype_ino,         &deliver_inode },
 543   { "interface_address",   vtype_stringptr,   &interface_address },
 544   { "interface_port",      vtype_int,         &interface_port },
 545   { "item",                vtype_stringptr,   &iterate_item },
 546   #ifdef LOOKUP_LDAP
 547   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 548   #endif
 549   { "load_average",        vtype_load_avg,    NULL },
 550   { "local_part",          vtype_stringptr,   &deliver_localpart },
 551   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 552   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 553   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 554   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 555   { "local_user_gid",      vtype_gid,         &local_user_gid },
 556   { "local_user_uid",      vtype_uid,         &local_user_uid },
 557   { "localhost_number",    vtype_int,         &host_number },
 558   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 559   { "log_space",           vtype_pspace,      (void *)FALSE },
 560   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 561   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 562 #ifdef WITH_CONTENT_SCAN
 563   { "malware_name",        vtype_stringptr,   &malware_name },
 564 #endif
 565   { "max_received_linelength", vtype_int,     &max_received_linelength },
 566   { "message_age",         vtype_int,         &message_age },
 567   { "message_body",        vtype_msgbody,     &message_body },
 568   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 569   { "message_body_size",   vtype_int,         &message_body_size },
 570   { "message_exim_id",     vtype_stringptr,   &message_id },
 571   { "message_headers",     vtype_msgheaders,  NULL },
 572   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 573   { "message_id",          vtype_stringptr,   &message_id },
 574   { "message_linecount",   vtype_int,         &message_linecount },
 575   { "message_size",        vtype_int,         &message_size },
 576 #ifdef SUPPORT_I18N
 577   { "message_smtputf8",    vtype_bool,        &message_smtputf8 },
 578 #endif
 579 #ifdef WITH_CONTENT_SCAN
 580   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 581   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 582   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 583   { "mime_charset",        vtype_stringptr,   &mime_charset },
 584   { "mime_content_description", vtype_stringptr, &mime_content_description },
 585   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 586   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 587   { "mime_content_size",   vtype_int,         &mime_content_size },
 588   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 589   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 590   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 591   { "mime_filename",       vtype_stringptr,   &mime_filename },
 592   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 593   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 594   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 595   { "mime_part_count",     vtype_int,         &mime_part_count },
 596 #endif
 597   { "n0",                  vtype_filter_int,  &filter_n[0] },
 598   { "n1",                  vtype_filter_int,  &filter_n[1] },
 599   { "n2",                  vtype_filter_int,  &filter_n[2] },
 600   { "n3",                  vtype_filter_int,  &filter_n[3] },
 601   { "n4",                  vtype_filter_int,  &filter_n[4] },
 602   { "n5",                  vtype_filter_int,  &filter_n[5] },
 603   { "n6",                  vtype_filter_int,  &filter_n[6] },
 604   { "n7",                  vtype_filter_int,  &filter_n[7] },
 605   { "n8",                  vtype_filter_int,  &filter_n[8] },
 606   { "n9",                  vtype_filter_int,  &filter_n[9] },
 607   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 608   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 609   { "originator_gid",      vtype_gid,         &originator_gid },
 610   { "originator_uid",      vtype_uid,         &originator_uid },
 611   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 612   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 613   { "pid",                 vtype_pid,         NULL },
 614 #ifndef DISABLE_PRDR
 615   { "prdr_requested",      vtype_bool,        &prdr_requested },
 616 #endif
 617   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 618 #if defined(SUPPORT_PROXY) || defined(SUPPORT_SOCKS)
 619   { "proxy_external_address",vtype_stringptr, &proxy_external_address },
 620   { "proxy_external_port", vtype_int,         &proxy_external_port },
 621   { "proxy_local_address", vtype_stringptr,   &proxy_local_address },
 622   { "proxy_local_port",    vtype_int,         &proxy_local_port },
 623   { "proxy_session",       vtype_bool,        &proxy_session },
 624 #endif
 625   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 626   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 627   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 628   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 629   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 630   { "rcpt_count",          vtype_int,         &rcpt_count },
 631   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 632   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 633   { "received_count",      vtype_int,         &received_count },
 634   { "received_for",        vtype_stringptr,   &received_for },
 635   { "received_ip_address", vtype_stringptr,   &interface_address },
 636   { "received_port",       vtype_int,         &interface_port },
 637   { "received_protocol",   vtype_stringptr,   &received_protocol },
 638   { "received_time",       vtype_int,         &received_time },
 639   { "recipient_data",      vtype_stringptr,   &recipient_data },
 640   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 641   { "recipients",          vtype_string_func, &fn_recipients },
 642   { "recipients_count",    vtype_int,         &recipients_count },
 643 #ifdef WITH_CONTENT_SCAN
 644   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 645 #endif
 646   { "reply_address",       vtype_reply,       NULL },
 647   { "return_path",         vtype_stringptr,   &return_path },
 648   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 649   { "router_name",         vtype_stringptr,   &router_name },
 650   { "runrc",               vtype_int,         &runrc },
 651   { "self_hostname",       vtype_stringptr,   &self_hostname },
 652   { "sender_address",      vtype_stringptr,   &sender_address },
 653   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 654   { "sender_address_domain", vtype_domain,    &sender_address },
 655   { "sender_address_local_part", vtype_localpart, &sender_address },
 656   { "sender_data",         vtype_stringptr,   &sender_data },
 657   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 658   { "sender_helo_dnssec",  vtype_bool,        &sender_helo_dnssec },
 659   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 660   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 661   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 662   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 663   { "sender_host_name",    vtype_host_lookup, NULL },
 664   { "sender_host_port",    vtype_int,         &sender_host_port },
 665   { "sender_ident",        vtype_stringptr,   &sender_ident },
 666   { "sender_rate",         vtype_stringptr,   &sender_rate },
 667   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 668   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 669   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 670   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 671   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 672   { "sending_port",        vtype_int,         &sending_port },
 673   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 674   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 675   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 676   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 677   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 678   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 679   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 680   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 681   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 682   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 683   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 684   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 685   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 686   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 687   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 688 #ifdef WITH_CONTENT_SCAN
 689   { "spam_action",         vtype_stringptr,   &spam_action },
 690   { "spam_bar",            vtype_stringptr,   &spam_bar },
 691   { "spam_report",         vtype_stringptr,   &spam_report },
 692   { "spam_score",          vtype_stringptr,   &spam_score },
 693   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 694 #endif
 695 #ifdef EXPERIMENTAL_SPF
 696   { "spf_guess",           vtype_stringptr,   &spf_guess },
 697   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 698   { "spf_received",        vtype_stringptr,   &spf_received },
 699   { "spf_result",          vtype_stringptr,   &spf_result },
 700   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 701 #endif
 702   { "spool_directory",     vtype_stringptr,   &spool_directory },
 703   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 704   { "spool_space",         vtype_pspace,      (void *)TRUE },
 705 #ifdef EXPERIMENTAL_SRS
 706   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 707   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 708   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 709   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 710   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 711   { "srs_status",          vtype_stringptr,   &srs_status },
 712 #endif
 713   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 714
 715   /* The non-(in,out) variables are now deprecated */
 716   { "tls_bits",            vtype_int,         &tls_in.bits },
 717   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 718   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 719
 720   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 721   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 722   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 723   { "tls_in_ocsp",         vtype_int,         &tls_in.ocsp },
 724   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 725   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 726   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 727 #if defined(SUPPORT_TLS)
 728   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 729 #endif
 730   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 731   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 732   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 733 #ifdef EXPERIMENTAL_DANE
 734   { "tls_out_dane",        vtype_bool,        &tls_out.dane_verified },
 735 #endif
 736   { "tls_out_ocsp",        vtype_int,         &tls_out.ocsp },
 737   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 738   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 739   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 740 #if defined(SUPPORT_TLS)
 741   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 742 #endif
 743 #ifdef EXPERIMENTAL_DANE
 744   { "tls_out_tlsa_usage",  vtype_int,         &tls_out.tlsa_usage },
 745 #endif
 746
 747   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 748 #if defined(SUPPORT_TLS)
 749   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 750 #endif
 751
 752   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 753   { "tod_epoch",           vtype_tode,        NULL },
 754   { "tod_epoch_l",         vtype_todel,       NULL },
 755   { "tod_full",            vtype_todf,        NULL },
 756   { "tod_log",             vtype_todl,        NULL },
 757   { "tod_logfile",         vtype_todlf,       NULL },
 758   { "tod_zone",            vtype_todzone,     NULL },
 759   { "tod_zulu",            vtype_todzulu,     NULL },
 760   { "transport_name",      vtype_stringptr,   &transport_name },
 761   { "value",               vtype_stringptr,   &lookup_value },
 762   { "verify_mode",         vtype_stringptr,   &verify_mode },
 763   { "version_number",      vtype_stringptr,   &version_string },
 764   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 765   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 766   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 767   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 768   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 769   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 770 };
 771
 772 static int var_table_size = nelem(var_table);
 773 static uschar var_buffer[256];
 774 static BOOL malformed_header;
 775
 776 /* For textual hashes */
 777
 778 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 779                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 780                                "0123456789";
 781
 782 enum { HMAC_MD5, HMAC_SHA1 };
 783
 784 /* For numeric hashes */
 785
 786 static unsigned int prime[] = {
 787   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 788  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 789  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 790
 791 /* For printing modes in symbolic form */
 792
 793 static uschar *mtable_normal[] =
 794   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 795
 796 static uschar *mtable_setid[] =
 797   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 798
 799 static uschar *mtable_sticky[] =
 800   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 801
 802
 803
 804 /*************************************************
 805 *           Tables for UTF-8 support             *
 806 *************************************************/
 807
 808 /* Table of the number of extra characters, indexed by the first character
 809 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 810 0x3d. */
 811
 812 static uschar utf8_table1[] = {
 813   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 814   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 815   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 816   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 817
 818 /* These are the masks for the data bits in the first byte of a character,
 819 indexed by the number of additional bytes. */
 820
 821 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 822
 823 /* Get the next UTF-8 character, advancing the pointer. */
 824
 825 #define GETUTF8INC(c, ptr) \
 826   c = *ptr++; \
 827   if ((c & 0xc0) == 0xc0) \
 828     { \
 829     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 830     int s = 6*a; \
 831     c = (c & utf8_table2[a]) << s; \
 832     while (a-- > 0) \
 833       { \
 834       s -= 6; \
 835       c |= (*ptr++ & 0x3f) << s; \
 836       } \
 837     }
 838
 839
 840 /*************************************************
 841 *           Binary chop search on a table        *
 842 *************************************************/
 843
 844 /* This is used for matching expansion items and operators.
 845
 846 Arguments:
 847   name        the name that is being sought
 848   table       the table to search
 849   table_size  the number of items in the table
 850
 851 Returns:      the offset in the table, or -1
 852 */
 853
 854 static int
 855 chop_match(uschar *name, uschar **table, int table_size)
 856 {
 857 uschar **bot = table;
 858 uschar **top = table + table_size;
 859
 860 while (top > bot)
 861   {
 862   uschar **mid = bot + (top - bot)/2;
 863   int c = Ustrcmp(name, *mid);
 864   if (c == 0) return mid - table;
 865   if (c > 0) bot = mid + 1; else top = mid;
 866   }
 867
 868 return -1;
 869 }
 870
 871
 872
 873 /*************************************************
 874 *          Check a condition string              *
 875 *************************************************/
 876
 877 /* This function is called to expand a string, and test the result for a "true"
 878 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 879 forced fail or lookup defer.
 880
 881 We used to release all store used, but this is not not safe due
 882 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 883 is reasonably careful to release what it can.
 884
 885 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 886
 887 Arguments:
 888   condition     the condition string
 889   m1            text to be incorporated in panic error
 890   m2            ditto
 891
 892 Returns:        TRUE if condition is met, FALSE if not
 893 */
 894
 895 BOOL
 896 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 897 {
 898 int rc;
 899 uschar *ss = expand_string(condition);
 900 if (ss == NULL)
 901   {
 902   if (!expand_string_forcedfail && !search_find_defer)
 903     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 904       "for %s %s: %s", condition, m1, m2, expand_string_message);
 905   return FALSE;
 906   }
 907 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 908   strcmpic(ss, US"false") != 0;
 909 return rc;
 910 }
 911
 912
 913
 914
 915 /*************************************************
 916 *        Pseudo-random number generation         *
 917 *************************************************/
 918
 919 /* Pseudo-random number generation.  The result is not "expected" to be
 920 cryptographically strong but not so weak that someone will shoot themselves
 921 in the foot using it as a nonce in some email header scheme or whatever
 922 weirdness they'll twist this into.  The result should ideally handle fork().
 923
 924 However, if we're stuck unable to provide this, then we'll fall back to
 925 appallingly bad randomness.
 926
 927 If SUPPORT_TLS is defined then this will not be used except as an emergency
 928 fallback.
 929
 930 Arguments:
 931   max       range maximum
 932 Returns     a random number in range [0, max-1]
 933 */
 934
 935 #ifdef SUPPORT_TLS
 936 # define vaguely_random_number vaguely_random_number_fallback
 937 #endif
 938 int
 939 vaguely_random_number(int max)
 940 {
 941 #ifdef SUPPORT_TLS
 942 # undef vaguely_random_number
 943 #endif
 944   static pid_t pid = 0;
 945   pid_t p2;
 946 #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
 947   struct timeval tv;
 948 #endif
 949
 950   p2 = getpid();
 951   if (p2 != pid)
 952     {
 953     if (pid != 0)
 954       {
 955
 956 #ifdef HAVE_ARC4RANDOM
 957       /* cryptographically strong randomness, common on *BSD platforms, not
 958       so much elsewhere.  Alas. */
 959 #ifndef NOT_HAVE_ARC4RANDOM_STIR
 960       arc4random_stir();
 961 #endif
 962 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 963 #ifdef HAVE_SRANDOMDEV
 964       /* uses random(4) for seeding */
 965       srandomdev();
 966 #else
 967       gettimeofday(&tv, NULL);
 968       srandom(tv.tv_sec | tv.tv_usec | getpid());
 969 #endif
 970 #else
 971       /* Poor randomness and no seeding here */
 972 #endif
 973
 974       }
 975     pid = p2;
 976     }
 977
 978 #ifdef HAVE_ARC4RANDOM
 979   return arc4random() % max;
 980 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 981   return random() % max;
 982 #else
 983   /* This one returns a 16-bit number, definitely not crypto-strong */
 984   return random_number(max);
 985 #endif
 986 }
 987
 988
 989
 990
 991 /*************************************************
 992 *             Pick out a name from a string      *
 993 *************************************************/
 994
 995 /* If the name is too long, it is silently truncated.
 996
 997 Arguments:
 998   name      points to a buffer into which to put the name
 999   max       is the length of the buffer
1000   s         points to the first alphabetic character of the name
1001   extras    chars other than alphanumerics to permit
1002
1003 Returns:    pointer to the first character after the name
1004
1005 Note: The test for *s != 0 in the while loop is necessary because
1006 Ustrchr() yields non-NULL if the character is zero (which is not something
1007 I expected). */
1008
1009 static const uschar *
1010 read_name(uschar *name, int max, const uschar *s, uschar *extras)
1011 {
1012 int ptr = 0;
1013 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
1014   {
1015   if (ptr < max-1) name[ptr++] = *s;
1016   s++;
1017   }
1018 name[ptr] = 0;
1019 return s;
1020 }
1021
1022
1023
1024 /*************************************************
1025 *     Pick out the rest of a header name         *
1026 *************************************************/
1027
1028 /* A variable name starting $header_ (or just $h_ for those who like
1029 abbreviations) might not be the complete header name because headers can
1030 contain any printing characters in their names, except ':'. This function is
1031 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
1032 on the end, if the name was terminated by white space.
1033
1034 Arguments:
1035   name      points to a buffer in which the name read so far exists
1036   max       is the length of the buffer
1037   s         points to the first character after the name so far, i.e. the
1038             first non-alphameric character after $header_xxxxx
1039
1040 Returns:    a pointer to the first character after the header name
1041 */
1042
1043 static const uschar *
1044 read_header_name(uschar *name, int max, const uschar *s)
1045 {
1046 int prelen = Ustrchr(name, '_') - name + 1;
1047 int ptr = Ustrlen(name) - prelen;
1048 if (ptr > 0) memmove(name, name+prelen, ptr);
1049 while (mac_isgraph(*s) && *s != ':')
1050   {
1051   if (ptr < max-1) name[ptr++] = *s;
1052   s++;
1053   }
1054 if (*s == ':') s++;
1055 name[ptr++] = ':';
1056 name[ptr] = 0;
1057 return s;
1058 }
1059
1060
1061
1062 /*************************************************
1063 *           Pick out a number from a string      *
1064 *************************************************/
1065
1066 /* Arguments:
1067   n     points to an integer into which to put the number
1068   s     points to the first digit of the number
1069
1070 Returns:  a pointer to the character after the last digit
1071 */
1072 /*XXX consider expanding to int_eximarith_t.  But the test for
1073 "overbig numbers" in 0002 still needs to overflow it. */
1074
1075 static uschar *
1076 read_number(int *n, uschar *s)
1077 {
1078 *n = 0;
1079 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1080 return s;
1081 }
1082
1083 static const uschar *
1084 read_cnumber(int *n, const uschar *s)
1085 {
1086 *n = 0;
1087 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1088 return s;
1089 }
1090
1091
1092
1093 /*************************************************
1094 *        Extract keyed subfield from a string    *
1095 *************************************************/
1096
1097 /* The yield is in dynamic store; NULL means that the key was not found.
1098
1099 Arguments:
1100   key       points to the name of the key
1101   s         points to the string from which to extract the subfield
1102
1103 Returns:    NULL if the subfield was not found, or
1104             a pointer to the subfield's data
1105 */
1106
1107 static uschar *
1108 expand_getkeyed(uschar *key, const uschar *s)
1109 {
1110 int length = Ustrlen(key);
1111 while (isspace(*s)) s++;
1112
1113 /* Loop to search for the key */
1114
1115 while (*s != 0)
1116   {
1117   int dkeylength;
1118   uschar *data;
1119   const uschar *dkey = s;
1120
1121   while (*s != 0 && *s != '=' && !isspace(*s)) s++;
1122   dkeylength = s - dkey;
1123   while (isspace(*s)) s++;
1124   if (*s == '=') while (isspace((*(++s))));
1125
1126   data = string_dequote(&s);
1127   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1128     return data;
1129
1130   while (isspace(*s)) s++;
1131   }
1132
1133 return NULL;
1134 }
1135
1136
1137
1138 static var_entry *
1139 find_var_ent(uschar * name)
1140 {
1141 int first = 0;
1142 int last = var_table_size;
1143
1144 while (last > first)
1145   {
1146   int middle = (first + last)/2;
1147   int c = Ustrcmp(name, var_table[middle].name);
1148
1149   if (c > 0) { first = middle + 1; continue; }
1150   if (c < 0) { last = middle; continue; }
1151   return &var_table[middle];
1152   }
1153 return NULL;
1154 }
1155
1156 /*************************************************
1157 *   Extract numbered subfield from string        *
1158 *************************************************/
1159
1160 /* Extracts a numbered field from a string that is divided by tokens - for
1161 example a line from /etc/passwd is divided by colon characters.  First field is
1162 numbered one.  Negative arguments count from the right. Zero returns the whole
1163 string. Returns NULL if there are insufficient tokens in the string
1164
1165 ***WARNING***
1166 Modifies final argument - this is a dynamically generated string, so that's OK.
1167
1168 Arguments:
1169   field       number of field to be extracted,
1170                 first field = 1, whole string = 0, last field = -1
1171   separators  characters that are used to break string into tokens
1172   s           points to the string from which to extract the subfield
1173
1174 Returns:      NULL if the field was not found,
1175               a pointer to the field's data inside s (modified to add 0)
1176 */
1177
1178 static uschar *
1179 expand_gettokened (int field, uschar *separators, uschar *s)
1180 {
1181 int sep = 1;
1182 int count;
1183 uschar *ss = s;
1184 uschar *fieldtext = NULL;
1185
1186 if (field == 0) return s;
1187
1188 /* Break the line up into fields in place; for field > 0 we stop when we have
1189 done the number of fields we want. For field < 0 we continue till the end of
1190 the string, counting the number of fields. */
1191
1192 count = (field > 0)? field : INT_MAX;
1193
1194 while (count-- > 0)
1195   {
1196   size_t len;
1197
1198   /* Previous field was the last one in the string. For a positive field
1199   number, this means there are not enough fields. For a negative field number,
1200   check that there are enough, and scan back to find the one that is wanted. */
1201
1202   if (sep == 0)
1203     {
1204     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1205     if ((-field) == (INT_MAX - count - 1)) return s;
1206     while (field++ < 0)
1207       {
1208       ss--;
1209       while (ss[-1] != 0) ss--;
1210       }
1211     fieldtext = ss;
1212     break;
1213     }
1214
1215   /* Previous field was not last in the string; save its start and put a
1216   zero at its end. */
1217
1218   fieldtext = ss;
1219   len = Ustrcspn(ss, separators);
1220   sep = ss[len];
1221   ss[len] = 0;
1222   ss += len + 1;
1223   }
1224
1225 return fieldtext;
1226 }
1227
1228
1229 static uschar *
1230 expand_getlistele(int field, const uschar * list)
1231 {
1232 const uschar * tlist= list;
1233 int sep= 0;
1234 uschar dummy;
1235
1236 if(field<0)
1237   {
1238   for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1239   sep= 0;
1240   }
1241 if(field==0) return NULL;
1242 while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1243 return string_nextinlist(&list, &sep, NULL, 0);
1244 }
1245
1246
1247 /* Certificate fields, by name.  Worry about by-OID later */
1248 /* Names are chosen to not have common prefixes */
1249
1250 #ifdef SUPPORT_TLS
1251 typedef struct
1252 {
1253 uschar * name;
1254 int      namelen;
1255 uschar * (*getfn)(void * cert, uschar * mod);
1256 } certfield;
1257 static certfield certfields[] =
1258 {                       /* linear search; no special order */
1259   { US"version",         7,  &tls_cert_version },
1260   { US"serial_number",   13, &tls_cert_serial_number },
1261   { US"subject",         7,  &tls_cert_subject },
1262   { US"notbefore",       9,  &tls_cert_not_before },
1263   { US"notafter",        8,  &tls_cert_not_after },
1264   { US"issuer",          6,  &tls_cert_issuer },
1265   { US"signature",       9,  &tls_cert_signature },
1266   { US"sig_algorithm",   13, &tls_cert_signature_algorithm },
1267   { US"subj_altname",    12, &tls_cert_subject_altname },
1268   { US"ocsp_uri",        8,  &tls_cert_ocsp_uri },
1269   { US"crl_uri",         7,  &tls_cert_crl_uri },
1270 };
1271
1272 static uschar *
1273 expand_getcertele(uschar * field, uschar * certvar)
1274 {
1275 var_entry * vp;
1276 certfield * cp;
1277
1278 if (!(vp = find_var_ent(certvar)))
1279   {
1280   expand_string_message =
1281     string_sprintf("no variable named \"%s\"", certvar);
1282   return NULL;          /* Unknown variable name */
1283   }
1284 /* NB this stops us passing certs around in variable.  Might
1285 want to do that in future */
1286 if (vp->type != vtype_cert)
1287   {
1288   expand_string_message =
1289     string_sprintf("\"%s\" is not a certificate", certvar);
1290   return NULL;          /* Unknown variable name */
1291   }
1292 if (!*(void **)vp->value)
1293   return NULL;
1294
1295 if (*field >= '0' && *field <= '9')
1296   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1297
1298 for(cp = certfields;
1299     cp < certfields + nelem(certfields);
1300     cp++)
1301   if (Ustrncmp(cp->name, field, cp->namelen) == 0)
1302     {
1303     uschar * modifier = *(field += cp->namelen) == ','
1304       ? ++field : NULL;
1305     return (*cp->getfn)( *(void **)vp->value, modifier );
1306     }
1307
1308 expand_string_message =
1309   string_sprintf("bad field selector \"%s\" for certextract", field);
1310 return NULL;
1311 }
1312 #endif  /*SUPPORT_TLS*/
1313
1314 /*************************************************
1315 *        Extract a substring from a string       *
1316 *************************************************/
1317
1318 /* Perform the ${substr or ${length expansion operations.
1319
1320 Arguments:
1321   subject     the input string
1322   value1      the offset from the start of the input string to the start of
1323                 the output string; if negative, count from the right.
1324   value2      the length of the output string, or negative (-1) for unset
1325                 if value1 is positive, unset means "all after"
1326                 if value1 is negative, unset means "all before"
1327   len         set to the length of the returned string
1328
1329 Returns:      pointer to the output string, or NULL if there is an error
1330 */
1331
1332 static uschar *
1333 extract_substr(uschar *subject, int value1, int value2, int *len)
1334 {
1335 int sublen = Ustrlen(subject);
1336
1337 if (value1 < 0)    /* count from right */
1338   {
1339   value1 += sublen;
1340
1341   /* If the position is before the start, skip to the start, and adjust the
1342   length. If the length ends up negative, the substring is null because nothing
1343   can precede. This falls out naturally when the length is unset, meaning "all
1344   to the left". */
1345
1346   if (value1 < 0)
1347     {
1348     value2 += value1;
1349     if (value2 < 0) value2 = 0;
1350     value1 = 0;
1351     }
1352
1353   /* Otherwise an unset length => characters before value1 */
1354
1355   else if (value2 < 0)
1356     {
1357     value2 = value1;
1358     value1 = 0;
1359     }
1360   }
1361
1362 /* For a non-negative offset, if the starting position is past the end of the
1363 string, the result will be the null string. Otherwise, an unset length means
1364 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1365
1366 else
1367   {
1368   if (value1 > sublen)
1369     {
1370     value1 = sublen;
1371     value2 = 0;
1372     }
1373   else if (value2 < 0) value2 = sublen;
1374   }
1375
1376 /* Cut the length down to the maximum possible for the offset value, and get
1377 the required characters. */
1378
1379 if (value1 + value2 > sublen) value2 = sublen - value1;
1380 *len = value2;
1381 return subject + value1;
1382 }
1383
1384
1385
1386
1387 /*************************************************
1388 *            Old-style hash of a string          *
1389 *************************************************/
1390
1391 /* Perform the ${hash expansion operation.
1392
1393 Arguments:
1394   subject     the input string (an expanded substring)
1395   value1      the length of the output string; if greater or equal to the
1396                 length of the input string, the input string is returned
1397   value2      the number of hash characters to use, or 26 if negative
1398   len         set to the length of the returned string
1399
1400 Returns:      pointer to the output string, or NULL if there is an error
1401 */
1402
1403 static uschar *
1404 compute_hash(uschar *subject, int value1, int value2, int *len)
1405 {
1406 int sublen = Ustrlen(subject);
1407
1408 if (value2 < 0) value2 = 26;
1409 else if (value2 > Ustrlen(hashcodes))
1410   {
1411   expand_string_message =
1412     string_sprintf("hash count \"%d\" too big", value2);
1413   return NULL;
1414   }
1415
1416 /* Calculate the hash text. We know it is shorter than the original string, so
1417 can safely place it in subject[] (we know that subject is always itself an
1418 expanded substring). */
1419
1420 if (value1 < sublen)
1421   {
1422   int c;
1423   int i = 0;
1424   int j = value1;
1425   while ((c = (subject[j])) != 0)
1426     {
1427     int shift = (c + j++) & 7;
1428     subject[i] ^= (c << shift) | (c >> (8-shift));
1429     if (++i >= value1) i = 0;
1430     }
1431   for (i = 0; i < value1; i++)
1432     subject[i] = hashcodes[(subject[i]) % value2];
1433   }
1434 else value1 = sublen;
1435
1436 *len = value1;
1437 return subject;
1438 }
1439
1440
1441
1442
1443 /*************************************************
1444 *             Numeric hash of a string           *
1445 *************************************************/
1446
1447 /* Perform the ${nhash expansion operation. The first characters of the
1448 string are treated as most important, and get the highest prime numbers.
1449
1450 Arguments:
1451   subject     the input string
1452   value1      the maximum value of the first part of the result
1453   value2      the maximum value of the second part of the result,
1454                 or negative to produce only a one-part result
1455   len         set to the length of the returned string
1456
1457 Returns:  pointer to the output string, or NULL if there is an error.
1458 */
1459
1460 static uschar *
1461 compute_nhash (uschar *subject, int value1, int value2, int *len)
1462 {
1463 uschar *s = subject;
1464 int i = 0;
1465 unsigned long int total = 0; /* no overflow */
1466
1467 while (*s != 0)
1468   {
1469   if (i == 0) i = nelem(prime) - 1;
1470   total += prime[i--] * (unsigned int)(*s++);
1471   }
1472
1473 /* If value2 is unset, just compute one number */
1474
1475 if (value2 < 0)
1476   {
1477   s = string_sprintf("%d", total % value1);
1478   }
1479
1480 /* Otherwise do a div/mod hash */
1481
1482 else
1483   {
1484   total = total % (value1 * value2);
1485   s = string_sprintf("%d/%d", total/value2, total % value2);
1486   }
1487
1488 *len = Ustrlen(s);
1489 return s;
1490 }
1491
1492
1493
1494
1495
1496 /*************************************************
1497 *     Find the value of a header or headers      *
1498 *************************************************/
1499
1500 /* Multiple instances of the same header get concatenated, and this function
1501 can also return a concatenation of all the header lines. When concatenating
1502 specific headers that contain lists of addresses, a comma is inserted between
1503 them. Otherwise we use a straight concatenation. Because some messages can have
1504 pathologically large number of lines, there is a limit on the length that is
1505 returned. Also, to avoid massive store use which would result from using
1506 string_cat() as it copies and extends strings, we do a preliminary pass to find
1507 out exactly how much store will be needed. On "normal" messages this will be
1508 pretty trivial.
1509
1510 Arguments:
1511   name          the name of the header, without the leading $header_ or $h_,
1512                 or NULL if a concatenation of all headers is required
1513   exists_only   TRUE if called from a def: test; don't need to build a string;
1514                 just return a string that is not "" and not "0" if the header
1515                 exists
1516   newsize       return the size of memory block that was obtained; may be NULL
1517                 if exists_only is TRUE
1518   want_raw      TRUE if called for $rh_ or $rheader_ variables; no processing,
1519                 other than concatenating, will be done on the header. Also used
1520                 for $message_headers_raw.
1521   charset       name of charset to translate MIME words to; used only if
1522                 want_raw is false; if NULL, no translation is done (this is
1523                 used for $bh_ and $bheader_)
1524
1525 Returns:        NULL if the header does not exist, else a pointer to a new
1526                 store block
1527 */
1528
1529 static uschar *
1530 find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw,
1531   uschar *charset)
1532 {
1533 BOOL found = name == NULL;
1534 int comma = 0;
1535 int len = found? 0 : Ustrlen(name);
1536 int i;
1537 uschar *yield = NULL;
1538 uschar *ptr = NULL;
1539
1540 /* Loop for two passes - saves code repetition */
1541
1542 for (i = 0; i < 2; i++)
1543   {
1544   int size = 0;
1545   header_line *h;
1546
1547   for (h = header_list; size < header_insert_maxlen && h != NULL; h = h->next)
1548     {
1549     if (h->type != htype_old && h->text != NULL)  /* NULL => Received: placeholder */
1550       {
1551       if (name == NULL || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1552         {
1553         int ilen;
1554         uschar *t;
1555
1556         if (exists_only) return US"1";      /* don't need actual string */
1557         found = TRUE;
1558         t = h->text + len;                  /* text to insert */
1559         if (!want_raw)                      /* unless wanted raw, */
1560           while (isspace(*t)) t++;          /* remove leading white space */
1561         ilen = h->slen - (t - h->text);     /* length to insert */
1562
1563         /* Unless wanted raw, remove trailing whitespace, including the
1564         newline. */
1565
1566         if (!want_raw)
1567           while (ilen > 0 && isspace(t[ilen-1])) ilen--;
1568
1569         /* Set comma = 1 if handling a single header and it's one of those
1570         that contains an address list, except when asked for raw headers. Only
1571         need to do this once. */
1572
1573         if (!want_raw && name != NULL && comma == 0 &&
1574             Ustrchr("BCFRST", h->type) != NULL)
1575           comma = 1;
1576
1577         /* First pass - compute total store needed; second pass - compute
1578         total store used, including this header. */
1579
1580         size += ilen + comma + 1;  /* +1 for the newline */
1581
1582         /* Second pass - concatentate the data, up to a maximum. Note that
1583         the loop stops when size hits the limit. */
1584
1585         if (i != 0)
1586           {
1587           if (size > header_insert_maxlen)
1588             {
1589             ilen -= size - header_insert_maxlen - 1;
1590             comma = 0;
1591             }
1592           Ustrncpy(ptr, t, ilen);
1593           ptr += ilen;
1594
1595           /* For a non-raw header, put in the comma if needed, then add
1596           back the newline we removed above, provided there was some text in
1597           the header. */
1598
1599           if (!want_raw && ilen > 0)
1600             {
1601             if (comma != 0) *ptr++ = ',';
1602             *ptr++ = '\n';
1603             }
1604           }
1605         }
1606       }
1607     }
1608
1609   /* At end of first pass, return NULL if no header found. Then truncate size
1610   if necessary, and get the buffer to hold the data, returning the buffer size.
1611   */
1612
1613   if (i == 0)
1614     {
1615     if (!found) return NULL;
1616     if (size > header_insert_maxlen) size = header_insert_maxlen;
1617     *newsize = size + 1;
1618     ptr = yield = store_get(*newsize);
1619     }
1620   }
1621
1622 /* That's all we do for raw header expansion. */
1623
1624 if (want_raw)
1625   {
1626   *ptr = 0;
1627   }
1628
1629 /* Otherwise, remove a final newline and a redundant added comma. Then we do
1630 RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2()
1631 function can return an error with decoded data if the charset translation
1632 fails. If decoding fails, it returns NULL. */
1633
1634 else
1635   {
1636   uschar *decoded, *error;
1637   if (ptr > yield && ptr[-1] == '\n') ptr--;
1638   if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--;
1639   *ptr = 0;
1640   decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL,
1641     newsize, &error);
1642   if (error != NULL)
1643     {
1644     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1645       "    input was: %s\n", error, yield);
1646     }
1647   if (decoded != NULL) yield = decoded;
1648   }
1649
1650 return yield;
1651 }
1652
1653
1654
1655
1656 /*************************************************
1657 *               Return list of recipients        *
1658 *************************************************/
1659 /* A recipients list is available only during system message filtering,
1660 during ACL processing after DATA, and while expanding pipe commands
1661 generated from a system filter, but not elsewhere. */
1662
1663 static uschar *
1664 fn_recipients(void)
1665 {
1666 if (!enable_dollar_recipients) return NULL; else
1667   {
1668   int size = 128;
1669   int ptr = 0;
1670   int i;
1671   uschar * s = store_get(size);
1672   for (i = 0; i < recipients_count; i++)
1673     {
1674     if (i != 0) s = string_catn(s, &size, &ptr, US", ", 2);
1675     s = string_cat(s, &size, &ptr, recipients_list[i].address);
1676     }
1677   s[ptr] = 0;     /* string_cat() leaves room */
1678   return s;
1679   }
1680 }
1681
1682
1683 /*************************************************
1684 *               Find value of a variable         *
1685 *************************************************/
1686
1687 /* The table of variables is kept in alphabetic order, so we can search it
1688 using a binary chop. The "choplen" variable is nothing to do with the binary
1689 chop.
1690
1691 Arguments:
1692   name          the name of the variable being sought
1693   exists_only   TRUE if this is a def: test; passed on to find_header()
1694   skipping      TRUE => skip any processing evaluation; this is not the same as
1695                   exists_only because def: may test for values that are first
1696                   evaluated here
1697   newsize       pointer to an int which is initially zero; if the answer is in
1698                 a new memory buffer, *newsize is set to its size
1699
1700 Returns:        NULL if the variable does not exist, or
1701                 a pointer to the variable's contents, or
1702                 something non-NULL if exists_only is TRUE
1703 */
1704
1705 static uschar *
1706 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1707 {
1708 var_entry * vp;
1709 uschar *s, *domain;
1710 uschar **ss;
1711 void * val;
1712
1713 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1714 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1715 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1716 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1717 (this gave backwards compatibility at the changeover). There may be built-in
1718 variables whose names start acl_ but they should never start in this way. This
1719 slightly messy specification is a consequence of the history, needless to say.
1720
1721 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1722 set, in which case give an error. */
1723
1724 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1725      !isalpha(name[5]))
1726   {
1727   tree_node *node =
1728     tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4);
1729   return node ? node->data.ptr : strict_acl_vars ? NULL : US"";
1730   }
1731
1732 /* Handle $auth<n> variables. */
1733
1734 if (Ustrncmp(name, "auth", 4) == 0)
1735   {
1736   uschar *endptr;
1737   int n = Ustrtoul(name + 4, &endptr, 10);
1738   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1739     return !auth_vars[n-1] ? US"" : auth_vars[n-1];
1740   }
1741 else if (Ustrncmp(name, "regex", 5) == 0)
1742   {
1743   uschar *endptr;
1744   int n = Ustrtoul(name + 5, &endptr, 10);
1745   if (*endptr == 0 && n != 0 && n <= REGEX_VARS)
1746     return !regex_vars[n-1] ? US"" : regex_vars[n-1];
1747   }
1748
1749 /* For all other variables, search the table */
1750
1751 if (!(vp = find_var_ent(name)))
1752   return NULL;          /* Unknown variable name */
1753
1754 /* Found an existing variable. If in skipping state, the value isn't needed,
1755 and we want to avoid processing (such as looking up the host name). */
1756
1757 if (skipping)
1758   return US"";
1759
1760 val = vp->value;
1761 switch (vp->type)
1762   {
1763   case vtype_filter_int:
1764     if (!filter_running) return NULL;
1765     /* Fall through */
1766     /* VVVVVVVVVVVV */
1767   case vtype_int:
1768     sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1769     return var_buffer;
1770
1771   case vtype_ino:
1772     sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1773     return var_buffer;
1774
1775   case vtype_gid:
1776     sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1777     return var_buffer;
1778
1779   case vtype_uid:
1780     sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1781     return var_buffer;
1782
1783   case vtype_bool:
1784     sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1785     return var_buffer;
1786
1787   case vtype_stringptr:                      /* Pointer to string */
1788     return (s = *((uschar **)(val))) ? s : US"";
1789
1790   case vtype_pid:
1791     sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1792     return var_buffer;
1793
1794   case vtype_load_avg:
1795     sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1796     return var_buffer;
1797
1798   case vtype_host_lookup:                    /* Lookup if not done so */
1799     if (sender_host_name == NULL && sender_host_address != NULL &&
1800         !host_lookup_failed && host_name_lookup() == OK)
1801       host_build_sender_fullhost();
1802     return (sender_host_name == NULL)? US"" : sender_host_name;
1803
1804   case vtype_localpart:                      /* Get local part from address */
1805     s = *((uschar **)(val));
1806     if (s == NULL) return US"";
1807     domain = Ustrrchr(s, '@');
1808     if (domain == NULL) return s;
1809     if (domain - s > sizeof(var_buffer) - 1)
1810       log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1811           " in string expansion", sizeof(var_buffer));
1812     Ustrncpy(var_buffer, s, domain - s);
1813     var_buffer[domain - s] = 0;
1814     return var_buffer;
1815
1816   case vtype_domain:                         /* Get domain from address */
1817     s = *((uschar **)(val));
1818     if (s == NULL) return US"";
1819     domain = Ustrrchr(s, '@');
1820     return (domain == NULL)? US"" : domain + 1;
1821
1822   case vtype_msgheaders:
1823     return find_header(NULL, exists_only, newsize, FALSE, NULL);
1824
1825   case vtype_msgheaders_raw:
1826     return find_header(NULL, exists_only, newsize, TRUE, NULL);
1827
1828   case vtype_msgbody:                        /* Pointer to msgbody string */
1829   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1830     ss = (uschar **)(val);
1831     if (*ss == NULL && deliver_datafile >= 0)  /* Read body when needed */
1832       {
1833       uschar *body;
1834       off_t start_offset = SPOOL_DATA_START_OFFSET;
1835       int len = message_body_visible;
1836       if (len > message_size) len = message_size;
1837       *ss = body = store_malloc(len+1);
1838       body[0] = 0;
1839       if (vp->type == vtype_msgbody_end)
1840         {
1841         struct stat statbuf;
1842         if (fstat(deliver_datafile, &statbuf) == 0)
1843           {
1844           start_offset = statbuf.st_size - len;
1845           if (start_offset < SPOOL_DATA_START_OFFSET)
1846             start_offset = SPOOL_DATA_START_OFFSET;
1847           }
1848         }
1849       if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0)
1850         log_write(0, LOG_MAIN|LOG_PANIC_DIE, "deliver_datafile lseek: %s",
1851           strerror(errno));
1852       len = read(deliver_datafile, body, len);
1853       if (len > 0)
1854         {
1855         body[len] = 0;
1856         if (message_body_newlines)   /* Separate loops for efficiency */
1857           while (len > 0)
1858             { if (body[--len] == 0) body[len] = ' '; }
1859         else
1860           while (len > 0)
1861             { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1862         }
1863       }
1864     return (*ss == NULL)? US"" : *ss;
1865
1866   case vtype_todbsdin:                       /* BSD inbox time of day */
1867     return tod_stamp(tod_bsdin);
1868
1869   case vtype_tode:                           /* Unix epoch time of day */
1870     return tod_stamp(tod_epoch);
1871
1872   case vtype_todel:                          /* Unix epoch/usec time of day */
1873     return tod_stamp(tod_epoch_l);
1874
1875   case vtype_todf:                           /* Full time of day */
1876     return tod_stamp(tod_full);
1877
1878   case vtype_todl:                           /* Log format time of day */
1879     return tod_stamp(tod_log_bare);            /* (without timezone) */
1880
1881   case vtype_todzone:                        /* Time zone offset only */
1882     return tod_stamp(tod_zone);
1883
1884   case vtype_todzulu:                        /* Zulu time */
1885     return tod_stamp(tod_zulu);
1886
1887   case vtype_todlf:                          /* Log file datestamp tod */
1888     return tod_stamp(tod_log_datestamp_daily);
1889
1890   case vtype_reply:                          /* Get reply address */
1891     s = find_header(US"reply-to:", exists_only, newsize, TRUE,
1892       headers_charset);
1893     if (s != NULL) while (isspace(*s)) s++;
1894     if (s == NULL || *s == 0)
1895       {
1896       *newsize = 0;                            /* For the *s==0 case */
1897       s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset);
1898       }
1899     if (s != NULL)
1900       {
1901       uschar *t;
1902       while (isspace(*s)) s++;
1903       for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1904       while (t > s && isspace(t[-1])) t--;
1905       *t = 0;
1906       }
1907     return (s == NULL)? US"" : s;
1908
1909   case vtype_string_func:
1910     {
1911     uschar * (*fn)() = val;
1912     return fn();
1913     }
1914
1915   case vtype_pspace:
1916     {
1917     int inodes;
1918     sprintf(CS var_buffer, "%d",
1919       receive_statvfs(val == (void *)TRUE, &inodes));
1920     }
1921   return var_buffer;
1922
1923   case vtype_pinodes:
1924     {
1925     int inodes;
1926     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1927     sprintf(CS var_buffer, "%d", inodes);
1928     }
1929   return var_buffer;
1930
1931   case vtype_cert:
1932     return *(void **)val ? US"<cert>" : US"";
1933
1934 #ifndef DISABLE_DKIM
1935   case vtype_dkim:
1936     return dkim_exim_expand_query((int)(long)val);
1937 #endif
1938
1939   }
1940
1941 return NULL;  /* Unknown variable. Silences static checkers. */
1942 }
1943
1944
1945
1946
1947 void
1948 modify_variable(uschar *name, void * value)
1949 {
1950 var_entry * vp;
1951 if ((vp = find_var_ent(name))) vp->value = value;
1952 return;          /* Unknown variable name, fail silently */
1953 }
1954
1955
1956
1957
1958
1959 /*************************************************
1960 *           Read and expand substrings           *
1961 *************************************************/
1962
1963 /* This function is called to read and expand argument substrings for various
1964 expansion items. Some have a minimum requirement that is less than the maximum;
1965 in these cases, the first non-present one is set to NULL.
1966
1967 Arguments:
1968   sub        points to vector of pointers to set
1969   n          maximum number of substrings
1970   m          minimum required
1971   sptr       points to current string pointer
1972   skipping   the skipping flag
1973   check_end  if TRUE, check for final '}'
1974   name       name of item, for error message
1975   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
1976              the store.
1977
1978 Returns:     0 OK; string pointer updated
1979              1 curly bracketing error (too few arguments)
1980              2 too many arguments (only if check_end is set); message set
1981              3 other error (expansion failure)
1982 */
1983
1984 static int
1985 read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping,
1986   BOOL check_end, uschar *name, BOOL *resetok)
1987 {
1988 int i;
1989 const uschar *s = *sptr;
1990
1991 while (isspace(*s)) s++;
1992 for (i = 0; i < n; i++)
1993   {
1994   if (*s != '{')
1995     {
1996     if (i < m)
1997       {
1998       expand_string_message = string_sprintf("Not enough arguments for '%s' "
1999         "(min is %d)", name, m);
2000       return 1;
2001       }
2002     sub[i] = NULL;
2003     break;
2004     }
2005   if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok)))
2006     return 3;
2007   if (*s++ != '}') return 1;
2008   while (isspace(*s)) s++;
2009   }
2010 if (check_end && *s++ != '}')
2011   {
2012   if (s[-1] == '{')
2013     {
2014     expand_string_message = string_sprintf("Too many arguments for '%s' "
2015       "(max is %d)", name, n);
2016     return 2;
2017     }
2018   expand_string_message = string_sprintf("missing '}' after '%s'", name);
2019   return 1;
2020   }
2021
2022 *sptr = s;
2023 return 0;
2024 }
2025
2026
2027
2028
2029 /*************************************************
2030 *     Elaborate message for bad variable         *
2031 *************************************************/
2032
2033 /* For the "unknown variable" message, take a look at the variable's name, and
2034 give additional information about possible ACL variables. The extra information
2035 is added on to expand_string_message.
2036
2037 Argument:   the name of the variable
2038 Returns:    nothing
2039 */
2040
2041 static void
2042 check_variable_error_message(uschar *name)
2043 {
2044 if (Ustrncmp(name, "acl_", 4) == 0)
2045   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
2046     (name[4] == 'c' || name[4] == 'm')?
2047       (isalpha(name[5])?
2048         US"6th character of a user-defined ACL variable must be a digit or underscore" :
2049         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
2050       ) :
2051       US"user-defined ACL variables must start acl_c or acl_m");
2052 }
2053
2054
2055
2056 /*
2057 Load args from sub array to globals, and call acl_check().
2058 Sub array will be corrupted on return.
2059
2060 Returns:       OK         access is granted by an ACCEPT verb
2061                DISCARD    access is (apparently) granted by a DISCARD verb
2062                FAIL       access is denied
2063                FAIL_DROP  access is denied; drop the connection
2064                DEFER      can't tell at the moment
2065                ERROR      disaster
2066 */
2067 static int
2068 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
2069 {
2070 int i;
2071 int sav_narg = acl_narg;
2072 int ret;
2073 uschar * dummy_logmsg;
2074 extern int acl_where;
2075
2076 if(--nsub > nelem(acl_arg)) nsub = nelem(acl_arg);
2077 for (i = 0; i < nsub && sub[i+1]; i++)
2078   {
2079   uschar * tmp = acl_arg[i];
2080   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2081   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2082   }
2083 acl_narg = i;
2084 while (i < nsub)
2085   {
2086   sub[i+1] = acl_arg[i];
2087   acl_arg[i++] = NULL;
2088   }
2089
2090 DEBUG(D_expand)
2091   debug_printf("expanding: acl: %s  arg: %s%s\n",
2092     sub[0],
2093     acl_narg>0 ? acl_arg[0] : US"<none>",
2094     acl_narg>1 ? " +more"   : "");
2095
2096 ret = acl_eval(acl_where, sub[0], user_msgp, &dummy_logmsg);
2097
2098 for (i = 0; i < nsub; i++)
2099   acl_arg[i] = sub[i+1];        /* restore old args */
2100 acl_narg = sav_narg;
2101
2102 return ret;
2103 }
2104
2105
2106
2107
2108 /*************************************************
2109 *        Read and evaluate a condition           *
2110 *************************************************/
2111
2112 /*
2113 Arguments:
2114   s        points to the start of the condition text
2115   resetok  points to a BOOL which is written false if it is unsafe to
2116            free memory. Certain condition types (acl) may have side-effect
2117            allocation which must be preserved.
2118   yield    points to a BOOL to hold the result of the condition test;
2119            if NULL, we are just reading through a condition that is
2120            part of an "or" combination to check syntax, or in a state
2121            where the answer isn't required
2122
2123 Returns:   a pointer to the first character after the condition, or
2124            NULL after an error
2125 */
2126
2127 static const uschar *
2128 eval_condition(const uschar *s, BOOL *resetok, BOOL *yield)
2129 {
2130 BOOL testfor = TRUE;
2131 BOOL tempcond, combined_cond;
2132 BOOL *subcondptr;
2133 BOOL sub2_honour_dollar = TRUE;
2134 int i, rc, cond_type, roffset;
2135 int_eximarith_t num[2];
2136 struct stat statbuf;
2137 uschar name[256];
2138 const uschar *sub[10];
2139
2140 const pcre *re;
2141 const uschar *rerror;
2142
2143 for (;;)
2144   {
2145   while (isspace(*s)) s++;
2146   if (*s == '!') { testfor = !testfor; s++; } else break;
2147   }
2148
2149 /* Numeric comparisons are symbolic */
2150
2151 if (*s == '=' || *s == '>' || *s == '<')
2152   {
2153   int p = 0;
2154   name[p++] = *s++;
2155   if (*s == '=')
2156     {
2157     name[p++] = '=';
2158     s++;
2159     }
2160   name[p] = 0;
2161   }
2162
2163 /* All other conditions are named */
2164
2165 else s = read_name(name, 256, s, US"_");
2166
2167 /* If we haven't read a name, it means some non-alpha character is first. */
2168
2169 if (name[0] == 0)
2170   {
2171   expand_string_message = string_sprintf("condition name expected, "
2172     "but found \"%.16s\"", s);
2173   return NULL;
2174   }
2175
2176 /* Find which condition we are dealing with, and switch on it */
2177
2178 cond_type = chop_match(name, cond_table, nelem(cond_table));
2179 switch(cond_type)
2180   {
2181   /* def: tests for a non-empty variable, or for the existence of a header. If
2182   yield == NULL we are in a skipping state, and don't care about the answer. */
2183
2184   case ECOND_DEF:
2185   if (*s != ':')
2186     {
2187     expand_string_message = US"\":\" expected after \"def\"";
2188     return NULL;
2189     }
2190
2191   s = read_name(name, 256, s+1, US"_");
2192
2193   /* Test for a header's existence. If the name contains a closing brace
2194   character, this may be a user error where the terminating colon has been
2195   omitted. Set a flag to adjust a subsequent error message in this case. */
2196
2197   if (Ustrncmp(name, "h_", 2) == 0 ||
2198       Ustrncmp(name, "rh_", 3) == 0 ||
2199       Ustrncmp(name, "bh_", 3) == 0 ||
2200       Ustrncmp(name, "header_", 7) == 0 ||
2201       Ustrncmp(name, "rheader_", 8) == 0 ||
2202       Ustrncmp(name, "bheader_", 8) == 0)
2203     {
2204     s = read_header_name(name, 256, s);
2205     /* {-for-text-editors */
2206     if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2207     if (yield != NULL) *yield =
2208       (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor;
2209     }
2210
2211   /* Test for a variable's having a non-empty value. A non-existent variable
2212   causes an expansion failure. */
2213
2214   else
2215     {
2216     uschar *value = find_variable(name, TRUE, yield == NULL, NULL);
2217     if (value == NULL)
2218       {
2219       expand_string_message = (name[0] == 0)?
2220         string_sprintf("variable name omitted after \"def:\"") :
2221         string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2222       check_variable_error_message(name);
2223       return NULL;
2224       }
2225     if (yield != NULL) *yield = (value[0] != 0) == testfor;
2226     }
2227
2228   return s;
2229
2230
2231   /* first_delivery tests for first delivery attempt */
2232
2233   case ECOND_FIRST_DELIVERY:
2234   if (yield != NULL) *yield = deliver_firsttime == testfor;
2235   return s;
2236
2237
2238   /* queue_running tests for any process started by a queue runner */
2239
2240   case ECOND_QUEUE_RUNNING:
2241   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2242   return s;
2243
2244
2245   /* exists:  tests for file existence
2246        isip:  tests for any IP address
2247       isip4:  tests for an IPv4 address
2248       isip6:  tests for an IPv6 address
2249         pam:  does PAM authentication
2250      radius:  does RADIUS authentication
2251    ldapauth:  does LDAP authentication
2252     pwcheck:  does Cyrus SASL pwcheck authentication
2253   */
2254
2255   case ECOND_EXISTS:
2256   case ECOND_ISIP:
2257   case ECOND_ISIP4:
2258   case ECOND_ISIP6:
2259   case ECOND_PAM:
2260   case ECOND_RADIUS:
2261   case ECOND_LDAPAUTH:
2262   case ECOND_PWCHECK:
2263
2264   while (isspace(*s)) s++;
2265   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2266
2267   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2268   if (sub[0] == NULL) return NULL;
2269   /* {-for-text-editors */
2270   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2271
2272   if (yield == NULL) return s;   /* No need to run the test if skipping */
2273
2274   switch(cond_type)
2275     {
2276     case ECOND_EXISTS:
2277     if ((expand_forbid & RDO_EXISTS) != 0)
2278       {
2279       expand_string_message = US"File existence tests are not permitted";
2280       return NULL;
2281       }
2282     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2283     break;
2284
2285     case ECOND_ISIP:
2286     case ECOND_ISIP4:
2287     case ECOND_ISIP6:
2288     rc = string_is_ip_address(sub[0], NULL);
2289     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2290              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2291     break;
2292
2293     /* Various authentication tests - all optionally compiled */
2294
2295     case ECOND_PAM:
2296     #ifdef SUPPORT_PAM
2297     rc = auth_call_pam(sub[0], &expand_string_message);
2298     goto END_AUTH;
2299     #else
2300     goto COND_FAILED_NOT_COMPILED;
2301     #endif  /* SUPPORT_PAM */
2302
2303     case ECOND_RADIUS:
2304     #ifdef RADIUS_CONFIG_FILE
2305     rc = auth_call_radius(sub[0], &expand_string_message);
2306     goto END_AUTH;
2307     #else
2308     goto COND_FAILED_NOT_COMPILED;
2309     #endif  /* RADIUS_CONFIG_FILE */
2310
2311     case ECOND_LDAPAUTH:
2312     #ifdef LOOKUP_LDAP
2313       {
2314       /* Just to keep the interface the same */
2315       BOOL do_cache;
2316       int old_pool = store_pool;
2317       store_pool = POOL_SEARCH;
2318       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2319         &expand_string_message, &do_cache);
2320       store_pool = old_pool;
2321       }
2322     goto END_AUTH;
2323     #else
2324     goto COND_FAILED_NOT_COMPILED;
2325     #endif  /* LOOKUP_LDAP */
2326
2327     case ECOND_PWCHECK:
2328     #ifdef CYRUS_PWCHECK_SOCKET
2329     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2330     goto END_AUTH;
2331     #else
2332     goto COND_FAILED_NOT_COMPILED;
2333     #endif  /* CYRUS_PWCHECK_SOCKET */
2334
2335     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2336         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2337     END_AUTH:
2338     if (rc == ERROR || rc == DEFER) return NULL;
2339     *yield = (rc == OK) == testfor;
2340     #endif
2341     }
2342   return s;
2343
2344
2345   /* call ACL (in a conditional context).  Accept true, deny false.
2346   Defer is a forced-fail.  Anything set by message= goes to $value.
2347   Up to ten parameters are used; we use the braces round the name+args
2348   like the saslauthd condition does, to permit a variable number of args.
2349   See also the expansion-item version EITEM_ACL and the traditional
2350   acl modifier ACLC_ACL.
2351   Since the ACL may allocate new global variables, tell our caller to not
2352   reclaim memory.
2353   */
2354
2355   case ECOND_ACL:
2356     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2357     {
2358     uschar *sub[10];
2359     uschar *user_msg;
2360     BOOL cond = FALSE;
2361     int size = 0;
2362     int ptr = 0;
2363
2364     while (isspace(*s)) s++;
2365     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2366
2367     switch(read_subs(sub, nelem(sub), 1,
2368       &s, yield == NULL, TRUE, US"acl", resetok))
2369       {
2370       case 1: expand_string_message = US"too few arguments or bracketing "
2371         "error for acl";
2372       case 2:
2373       case 3: return NULL;
2374       }
2375
2376     *resetok = FALSE;   /* eval_acl() might allocate; do not reclaim */
2377     if (yield != NULL) switch(eval_acl(sub, nelem(sub), &user_msg))
2378         {
2379         case OK:
2380           cond = TRUE;
2381         case FAIL:
2382           lookup_value = NULL;
2383           if (user_msg)
2384             {
2385             lookup_value = string_cat(NULL, &size, &ptr, user_msg);
2386             lookup_value[ptr] = '\0';
2387             }
2388           *yield = cond == testfor;
2389           break;
2390
2391         case DEFER:
2392           expand_string_forcedfail = TRUE;
2393           /*FALLTHROUGH*/
2394         default:
2395           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
2396           return NULL;
2397         }
2398     return s;
2399     }
2400
2401
2402   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2403
2404      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2405
2406   However, the last two are optional. That is why the whole set is enclosed
2407   in their own set of braces. */
2408
2409   case ECOND_SASLAUTHD:
2410 #ifndef CYRUS_SASLAUTHD_SOCKET
2411     goto COND_FAILED_NOT_COMPILED;
2412 #else
2413     {
2414     uschar *sub[4];
2415     while (isspace(*s)) s++;
2416     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2417     switch(read_subs(sub, nelem(sub), 2, &s, yield == NULL, TRUE, US"saslauthd",
2418                     resetok))
2419       {
2420       case 1: expand_string_message = US"too few arguments or bracketing "
2421         "error for saslauthd";
2422       case 2:
2423       case 3: return NULL;
2424       }
2425     if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2426     if (yield != NULL)
2427       {
2428       int rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2429         &expand_string_message);
2430       if (rc == ERROR || rc == DEFER) return NULL;
2431       *yield = (rc == OK) == testfor;
2432       }
2433     return s;
2434     }
2435 #endif /* CYRUS_SASLAUTHD_SOCKET */
2436
2437
2438   /* symbolic operators for numeric and string comparison, and a number of
2439   other operators, all requiring two arguments.
2440
2441   crypteq:           encrypts plaintext and compares against an encrypted text,
2442                        using crypt(), crypt16(), MD5 or SHA-1
2443   inlist/inlisti:    checks if first argument is in the list of the second
2444   match:             does a regular expression match and sets up the numerical
2445                        variables if it succeeds
2446   match_address:     matches in an address list
2447   match_domain:      matches in a domain list
2448   match_ip:          matches a host list that is restricted to IP addresses
2449   match_local_part:  matches in a local part list
2450   */
2451
2452   case ECOND_MATCH_ADDRESS:
2453   case ECOND_MATCH_DOMAIN:
2454   case ECOND_MATCH_IP:
2455   case ECOND_MATCH_LOCAL_PART:
2456 #ifndef EXPAND_LISTMATCH_RHS
2457     sub2_honour_dollar = FALSE;
2458 #endif
2459     /* FALLTHROUGH */
2460
2461   case ECOND_CRYPTEQ:
2462   case ECOND_INLIST:
2463   case ECOND_INLISTI:
2464   case ECOND_MATCH:
2465
2466   case ECOND_NUM_L:     /* Numerical comparisons */
2467   case ECOND_NUM_LE:
2468   case ECOND_NUM_E:
2469   case ECOND_NUM_EE:
2470   case ECOND_NUM_G:
2471   case ECOND_NUM_GE:
2472
2473   case ECOND_STR_LT:    /* String comparisons */
2474   case ECOND_STR_LTI:
2475   case ECOND_STR_LE:
2476   case ECOND_STR_LEI:
2477   case ECOND_STR_EQ:
2478   case ECOND_STR_EQI:
2479   case ECOND_STR_GT:
2480   case ECOND_STR_GTI:
2481   case ECOND_STR_GE:
2482   case ECOND_STR_GEI:
2483
2484   for (i = 0; i < 2; i++)
2485     {
2486     /* Sometimes, we don't expand substrings; too many insecure configurations
2487     created using match_address{}{} and friends, where the second param
2488     includes information from untrustworthy sources. */
2489     BOOL honour_dollar = TRUE;
2490     if ((i > 0) && !sub2_honour_dollar)
2491       honour_dollar = FALSE;
2492
2493     while (isspace(*s)) s++;
2494     if (*s != '{')
2495       {
2496       if (i == 0) goto COND_FAILED_CURLY_START;
2497       expand_string_message = string_sprintf("missing 2nd string in {} "
2498         "after \"%s\"", name);
2499       return NULL;
2500       }
2501     sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2502         honour_dollar, resetok);
2503     if (sub[i] == NULL) return NULL;
2504     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2505
2506     /* Convert to numerical if required; we know that the names of all the
2507     conditions that compare numbers do not start with a letter. This just saves
2508     checking for them individually. */
2509
2510     if (!isalpha(name[0]) && yield != NULL)
2511       if (sub[i][0] == 0)
2512         {
2513         num[i] = 0;
2514         DEBUG(D_expand)
2515           debug_printf("empty string cast to zero for numerical comparison\n");
2516         }
2517       else
2518         {
2519         num[i] = expanded_string_integer(sub[i], FALSE);
2520         if (expand_string_message != NULL) return NULL;
2521         }
2522     }
2523
2524   /* Result not required */
2525
2526   if (yield == NULL) return s;
2527
2528   /* Do an appropriate comparison */
2529
2530   switch(cond_type)
2531     {
2532     case ECOND_NUM_E:
2533     case ECOND_NUM_EE:
2534     tempcond = (num[0] == num[1]);
2535     break;
2536
2537     case ECOND_NUM_G:
2538     tempcond = (num[0] > num[1]);
2539     break;
2540
2541     case ECOND_NUM_GE:
2542     tempcond = (num[0] >= num[1]);
2543     break;
2544
2545     case ECOND_NUM_L:
2546     tempcond = (num[0] < num[1]);
2547     break;
2548
2549     case ECOND_NUM_LE:
2550     tempcond = (num[0] <= num[1]);
2551     break;
2552
2553     case ECOND_STR_LT:
2554     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2555     break;
2556
2557     case ECOND_STR_LTI:
2558     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2559     break;
2560
2561     case ECOND_STR_LE:
2562     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2563     break;
2564
2565     case ECOND_STR_LEI:
2566     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2567     break;
2568
2569     case ECOND_STR_EQ:
2570     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2571     break;
2572
2573     case ECOND_STR_EQI:
2574     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2575     break;
2576
2577     case ECOND_STR_GT:
2578     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2579     break;
2580
2581     case ECOND_STR_GTI:
2582     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2583     break;
2584
2585     case ECOND_STR_GE:
2586     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2587     break;
2588
2589     case ECOND_STR_GEI:
2590     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2591     break;
2592
2593     case ECOND_MATCH:   /* Regular expression match */
2594     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2595       NULL);
2596     if (re == NULL)
2597       {
2598       expand_string_message = string_sprintf("regular expression error in "
2599         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2600       return NULL;
2601       }
2602     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2603     break;
2604
2605     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2606     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2607     goto MATCHED_SOMETHING;
2608
2609     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2610     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2611       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2612     goto MATCHED_SOMETHING;
2613
2614     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2615     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2616       {
2617       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2618         sub[0]);
2619       return NULL;
2620       }
2621     else
2622       {
2623       unsigned int *nullcache = NULL;
2624       check_host_block cb;
2625
2626       cb.host_name = US"";
2627       cb.host_address = sub[0];
2628
2629       /* If the host address starts off ::ffff: it is an IPv6 address in
2630       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2631       addresses. */
2632
2633       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2634         cb.host_address + 7 : cb.host_address;
2635
2636       rc = match_check_list(
2637              &sub[1],                   /* the list */
2638              0,                         /* separator character */
2639              &hostlist_anchor,          /* anchor pointer */
2640              &nullcache,                /* cache pointer */
2641              check_host,                /* function for testing */
2642              &cb,                       /* argument for function */
2643              MCL_HOST,                  /* type of check */
2644              sub[0],                    /* text for debugging */
2645              NULL);                     /* where to pass back data */
2646       }
2647     goto MATCHED_SOMETHING;
2648
2649     case ECOND_MATCH_LOCAL_PART:
2650     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2651       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2652     /* Fall through */
2653     /* VVVVVVVVVVVV */
2654     MATCHED_SOMETHING:
2655     switch(rc)
2656       {
2657       case OK:
2658       tempcond = TRUE;
2659       break;
2660
2661       case FAIL:
2662       tempcond = FALSE;
2663       break;
2664
2665       case DEFER:
2666       expand_string_message = string_sprintf("unable to complete match "
2667         "against \"%s\": %s", sub[1], search_error_message);
2668       return NULL;
2669       }
2670
2671     break;
2672
2673     /* Various "encrypted" comparisons. If the second string starts with
2674     "{" then an encryption type is given. Default to crypt() or crypt16()
2675     (build-time choice). */
2676     /* }-for-text-editors */
2677
2678     case ECOND_CRYPTEQ:
2679     #ifndef SUPPORT_CRYPTEQ
2680     goto COND_FAILED_NOT_COMPILED;
2681     #else
2682     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2683       {
2684       int sublen = Ustrlen(sub[1]+5);
2685       md5 base;
2686       uschar digest[16];
2687
2688       md5_start(&base);
2689       md5_end(&base, sub[0], Ustrlen(sub[0]), digest);
2690
2691       /* If the length that we are comparing against is 24, the MD5 digest
2692       is expressed as a base64 string. This is the way LDAP does it. However,
2693       some other software uses a straightforward hex representation. We assume
2694       this if the length is 32. Other lengths fail. */
2695
2696       if (sublen == 24)
2697         {
2698         uschar *coded = b64encode(digest, 16);
2699         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2700           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2701         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2702         }
2703       else if (sublen == 32)
2704         {
2705         int i;
2706         uschar coded[36];
2707         for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2708         coded[32] = 0;
2709         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2710           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2711         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2712         }
2713       else
2714         {
2715         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2716           "fail\n  crypted=%s\n", sub[1]+5);
2717         tempcond = FALSE;
2718         }
2719       }
2720
2721     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2722       {
2723       int sublen = Ustrlen(sub[1]+6);
2724       hctx h;
2725       uschar digest[20];
2726
2727       sha1_start(&h);
2728       sha1_end(&h, sub[0], Ustrlen(sub[0]), digest);
2729
2730       /* If the length that we are comparing against is 28, assume the SHA1
2731       digest is expressed as a base64 string. If the length is 40, assume a
2732       straightforward hex representation. Other lengths fail. */
2733
2734       if (sublen == 28)
2735         {
2736         uschar *coded = b64encode(digest, 20);
2737         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2738           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2739         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2740         }
2741       else if (sublen == 40)
2742         {
2743         int i;
2744         uschar coded[44];
2745         for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2746         coded[40] = 0;
2747         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2748           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2749         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2750         }
2751       else
2752         {
2753         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2754           "fail\n  crypted=%s\n", sub[1]+6);
2755         tempcond = FALSE;
2756         }
2757       }
2758
2759     else   /* {crypt} or {crypt16} and non-{ at start */
2760            /* }-for-text-editors */
2761       {
2762       int which = 0;
2763       uschar *coded;
2764
2765       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2766         {
2767         sub[1] += 7;
2768         which = 1;
2769         }
2770       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2771         {
2772         sub[1] += 9;
2773         which = 2;
2774         }
2775       else if (sub[1][0] == '{')                /* }-for-text-editors */
2776         {
2777         expand_string_message = string_sprintf("unknown encryption mechanism "
2778           "in \"%s\"", sub[1]);
2779         return NULL;
2780         }
2781
2782       switch(which)
2783         {
2784         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2785         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2786         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2787         }
2788
2789       #define STR(s) # s
2790       #define XSTR(s) STR(s)
2791       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2792         "  subject=%s\n  crypted=%s\n",
2793         which == 0 ? XSTR(DEFAULT_CRYPT) : which == 1 ? "crypt" : "crypt16",
2794         coded, sub[1]);
2795       #undef STR
2796       #undef XSTR
2797
2798       /* If the encrypted string contains fewer than two characters (for the
2799       salt), force failure. Otherwise we get false positives: with an empty
2800       string the yield of crypt() is an empty string! */
2801
2802       if (coded)
2803         tempcond = Ustrlen(sub[1]) < 2 ? FALSE : Ustrcmp(coded, sub[1]) == 0;
2804       else if (errno == EINVAL)
2805         tempcond = FALSE;
2806       else
2807         {
2808         expand_string_message = string_sprintf("crypt error: %s\n",
2809           US strerror(errno));
2810         return NULL;
2811         }
2812       }
2813     break;
2814     #endif  /* SUPPORT_CRYPTEQ */
2815
2816     case ECOND_INLIST:
2817     case ECOND_INLISTI:
2818       {
2819       const uschar * list = sub[1];
2820       int sep = 0;
2821       uschar *save_iterate_item = iterate_item;
2822       int (*compare)(const uschar *, const uschar *);
2823
2824       DEBUG(D_expand) debug_printf("condition: %s\n", name);
2825
2826       tempcond = FALSE;
2827       compare = cond_type == ECOND_INLISTI
2828         ? strcmpic : (int (*)(const uschar *, const uschar *)) strcmp;
2829
2830       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
2831         if (compare(sub[0], iterate_item) == 0)
2832           {
2833           tempcond = TRUE;
2834           break;
2835           }
2836       iterate_item = save_iterate_item;
2837       }
2838
2839     }   /* Switch for comparison conditions */
2840
2841   *yield = tempcond == testfor;
2842   return s;    /* End of comparison conditions */
2843
2844
2845   /* and/or: computes logical and/or of several conditions */
2846
2847   case ECOND_AND:
2848   case ECOND_OR:
2849   subcondptr = (yield == NULL)? NULL : &tempcond;
2850   combined_cond = (cond_type == ECOND_AND);
2851
2852   while (isspace(*s)) s++;
2853   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2854
2855   for (;;)
2856     {
2857     while (isspace(*s)) s++;
2858     /* {-for-text-editors */
2859     if (*s == '}') break;
2860     if (*s != '{')                                      /* }-for-text-editors */
2861       {
2862       expand_string_message = string_sprintf("each subcondition "
2863         "inside an \"%s{...}\" condition must be in its own {}", name);
2864       return NULL;
2865       }
2866
2867     if (!(s = eval_condition(s+1, resetok, subcondptr)))
2868       {
2869       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
2870         expand_string_message, name);
2871       return NULL;
2872       }
2873     while (isspace(*s)) s++;
2874
2875     /* {-for-text-editors */
2876     if (*s++ != '}')
2877       {
2878       /* {-for-text-editors */
2879       expand_string_message = string_sprintf("missing } at end of condition "
2880         "inside \"%s\" group", name);
2881       return NULL;
2882       }
2883
2884     if (yield != NULL)
2885       {
2886       if (cond_type == ECOND_AND)
2887         {
2888         combined_cond &= tempcond;
2889         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
2890         }                                       /* evaluate any more */
2891       else
2892         {
2893         combined_cond |= tempcond;
2894         if (combined_cond) subcondptr = NULL;   /* once true, don't */
2895         }                                       /* evaluate any more */
2896       }
2897     }
2898
2899   if (yield != NULL) *yield = (combined_cond == testfor);
2900   return ++s;
2901
2902
2903   /* forall/forany: iterates a condition with different values */
2904
2905   case ECOND_FORALL:
2906   case ECOND_FORANY:
2907     {
2908     const uschar * list;
2909     int sep = 0;
2910     uschar *save_iterate_item = iterate_item;
2911
2912     DEBUG(D_expand) debug_printf("condition: %s\n", name);
2913
2914     while (isspace(*s)) s++;
2915     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2916     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
2917     if (sub[0] == NULL) return NULL;
2918     /* {-for-text-editors */
2919     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2920
2921     while (isspace(*s)) s++;
2922     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2923
2924     sub[1] = s;
2925
2926     /* Call eval_condition once, with result discarded (as if scanning a
2927     "false" part). This allows us to find the end of the condition, because if
2928     the list it empty, we won't actually evaluate the condition for real. */
2929
2930     if (!(s = eval_condition(sub[1], resetok, NULL)))
2931       {
2932       expand_string_message = string_sprintf("%s inside \"%s\" condition",
2933         expand_string_message, name);
2934       return NULL;
2935       }
2936     while (isspace(*s)) s++;
2937
2938     /* {-for-text-editors */
2939     if (*s++ != '}')
2940       {
2941       /* {-for-text-editors */
2942       expand_string_message = string_sprintf("missing } at end of condition "
2943         "inside \"%s\"", name);
2944       return NULL;
2945       }
2946
2947     if (yield != NULL) *yield = !testfor;
2948     list = sub[0];
2949     while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
2950       {
2951       DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
2952       if (!eval_condition(sub[1], resetok, &tempcond))
2953         {
2954         expand_string_message = string_sprintf("%s inside \"%s\" condition",
2955           expand_string_message, name);
2956         iterate_item = save_iterate_item;
2957         return NULL;
2958         }
2959       DEBUG(D_expand) debug_printf("%s: condition evaluated to %s\n", name,
2960         tempcond? "true":"false");
2961
2962       if (yield != NULL) *yield = (tempcond == testfor);
2963       if (tempcond == (cond_type == ECOND_FORANY)) break;
2964       }
2965
2966     iterate_item = save_iterate_item;
2967     return s;
2968     }
2969
2970
2971   /* The bool{} expansion condition maps a string to boolean.
2972   The values supported should match those supported by the ACL condition
2973   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
2974   of true/false.  Note that Router "condition" rules have a different
2975   interpretation, where general data can be used and only a few values
2976   map to FALSE.
2977   Note that readconf.c boolean matching, for boolean configuration options,
2978   only matches true/yes/false/no.
2979   The bool_lax{} condition matches the Router logic, which is much more
2980   liberal. */
2981   case ECOND_BOOL:
2982   case ECOND_BOOL_LAX:
2983     {
2984     uschar *sub_arg[1];
2985     uschar *t, *t2;
2986     uschar *ourname;
2987     size_t len;
2988     BOOL boolvalue = FALSE;
2989     while (isspace(*s)) s++;
2990     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2991     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
2992     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
2993       {
2994       case 1: expand_string_message = string_sprintf(
2995                   "too few arguments or bracketing error for %s",
2996                   ourname);
2997       /*FALLTHROUGH*/
2998       case 2:
2999       case 3: return NULL;
3000       }
3001     t = sub_arg[0];
3002     while (isspace(*t)) t++;
3003     len = Ustrlen(t);
3004     if (len)
3005       {
3006       /* trailing whitespace: seems like a good idea to ignore it too */
3007       t2 = t + len - 1;
3008       while (isspace(*t2)) t2--;
3009       if (t2 != (t + len))
3010         {
3011         *++t2 = '\0';
3012         len = t2 - t;
3013         }
3014       }
3015     DEBUG(D_expand)
3016       debug_printf("considering %s: %s\n", ourname, len ? t : US"<empty>");
3017     /* logic for the lax case from expand_check_condition(), which also does
3018     expands, and the logic is both short and stable enough that there should
3019     be no maintenance burden from replicating it. */
3020     if (len == 0)
3021       boolvalue = FALSE;
3022     else if (*t == '-'
3023              ? Ustrspn(t+1, "0123456789") == len-1
3024              : Ustrspn(t,   "0123456789") == len)
3025       {
3026       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
3027       /* expand_check_condition only does a literal string "0" check */
3028       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
3029         boolvalue = TRUE;
3030       }
3031     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
3032       boolvalue = TRUE;
3033     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
3034       boolvalue = FALSE;
3035     else if (cond_type == ECOND_BOOL_LAX)
3036       boolvalue = TRUE;
3037     else
3038       {
3039       expand_string_message = string_sprintf("unrecognised boolean "
3040        "value \"%s\"", t);
3041       return NULL;
3042       }
3043     if (yield != NULL) *yield = (boolvalue == testfor);
3044     return s;
3045     }
3046
3047   /* Unknown condition */
3048
3049   default:
3050   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
3051   return NULL;
3052   }   /* End switch on condition type */
3053
3054 /* Missing braces at start and end of data */
3055
3056 COND_FAILED_CURLY_START:
3057 expand_string_message = string_sprintf("missing { after \"%s\"", name);
3058 return NULL;
3059
3060 COND_FAILED_CURLY_END:
3061 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
3062   name);
3063 return NULL;
3064
3065 /* A condition requires code that is not compiled */
3066
3067 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
3068     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
3069     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
3070 COND_FAILED_NOT_COMPILED:
3071 expand_string_message = string_sprintf("support for \"%s\" not compiled",
3072   name);
3073 return NULL;
3074 #endif
3075 }
3076
3077
3078
3079
3080 /*************************************************
3081 *          Save numerical variables              *
3082 *************************************************/
3083
3084 /* This function is called from items such as "if" that want to preserve and
3085 restore the numbered variables.
3086
3087 Arguments:
3088   save_expand_string    points to an array of pointers to set
3089   save_expand_nlength   points to an array of ints for the lengths
3090
3091 Returns:                the value of expand max to save
3092 */
3093
3094 static int
3095 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3096 {
3097 int i;
3098 for (i = 0; i <= expand_nmax; i++)
3099   {
3100   save_expand_nstring[i] = expand_nstring[i];
3101   save_expand_nlength[i] = expand_nlength[i];
3102   }
3103 return expand_nmax;
3104 }
3105
3106
3107
3108 /*************************************************
3109 *           Restore numerical variables          *
3110 *************************************************/
3111
3112 /* This function restored saved values of numerical strings.
3113
3114 Arguments:
3115   save_expand_nmax      the number of strings to restore
3116   save_expand_string    points to an array of pointers
3117   save_expand_nlength   points to an array of ints
3118
3119 Returns:                nothing
3120 */
3121
3122 static void
3123 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3124   int *save_expand_nlength)
3125 {
3126 int i;
3127 expand_nmax = save_expand_nmax;
3128 for (i = 0; i <= expand_nmax; i++)
3129   {
3130   expand_nstring[i] = save_expand_nstring[i];
3131   expand_nlength[i] = save_expand_nlength[i];
3132   }
3133 }
3134
3135
3136
3137
3138
3139 /*************************************************
3140 *            Handle yes/no substrings            *
3141 *************************************************/
3142
3143 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3144 alternative substrings that depend on whether or not the condition was true,
3145 or the lookup or extraction succeeded. The substrings always have to be
3146 expanded, to check their syntax, but "skipping" is set when the result is not
3147 needed - this avoids unnecessary nested lookups.
3148
3149 Arguments:
3150   skipping       TRUE if we were skipping when this item was reached
3151   yes            TRUE if the first string is to be used, else use the second
3152   save_lookup    a value to put back into lookup_value before the 2nd expansion
3153   sptr           points to the input string pointer
3154   yieldptr       points to the output string pointer
3155   sizeptr        points to the output string size
3156   ptrptr         points to the output string pointer
3157   type           "lookup", "if", "extract", "run", "env", "listextract" or
3158                  "certextract" for error message
3159   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3160                 the store.
3161
3162 Returns:         0 OK; lookup_value has been reset to save_lookup
3163                  1 expansion failed
3164                  2 expansion failed because of bracketing error
3165 */
3166
3167 static int
3168 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr,
3169   uschar **yieldptr, int *sizeptr, int *ptrptr, uschar *type, BOOL *resetok)
3170 {
3171 int rc = 0;
3172 const uschar *s = *sptr;    /* Local value */
3173 uschar *sub1, *sub2;
3174 const uschar * errwhere;
3175
3176 /* If there are no following strings, we substitute the contents of $value for
3177 lookups and for extractions in the success case. For the ${if item, the string
3178 "true" is substituted. In the fail case, nothing is substituted for all three
3179 items. */
3180
3181 while (isspace(*s)) s++;
3182 if (*s == '}')
3183   {
3184   if (type[0] == 'i')
3185     {
3186     if (yes) *yieldptr = string_catn(*yieldptr, sizeptr, ptrptr, US"true", 4);
3187     }
3188   else
3189     {
3190     if (yes && lookup_value)
3191       *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value);
3192     lookup_value = save_lookup;
3193     }
3194   s++;
3195   goto RETURN;
3196   }
3197
3198 /* The first following string must be braced. */
3199
3200 if (*s++ != '{')
3201   {
3202   errwhere = US"'yes' part did not start with '{'";
3203   goto FAILED_CURLY;
3204   }
3205
3206 /* Expand the first substring. Forced failures are noticed only if we actually
3207 want this string. Set skipping in the call in the fail case (this will always
3208 be the case if we were already skipping). */
3209
3210 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3211 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
3212 expand_string_forcedfail = FALSE;
3213 if (*s++ != '}')
3214   {
3215   errwhere = US"'yes' part did not end with '}'";
3216   goto FAILED_CURLY;
3217   }
3218
3219 /* If we want the first string, add it to the output */
3220
3221 if (yes)
3222   *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub1);
3223
3224 /* If this is called from a lookup/env or a (cert)extract, we want to restore
3225 $value to what it was at the start of the item, so that it has this value
3226 during the second string expansion. For the call from "if" or "run" to this
3227 function, save_lookup is set to lookup_value, so that this statement does
3228 nothing. */
3229
3230 lookup_value = save_lookup;
3231
3232 /* There now follows either another substring, or "fail", or nothing. This
3233 time, forced failures are noticed only if we want the second string. We must
3234 set skipping in the nested call if we don't want this string, or if we were
3235 already skipping. */
3236
3237 while (isspace(*s)) s++;
3238 if (*s == '{')
3239   {
3240   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3241   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
3242   expand_string_forcedfail = FALSE;
3243   if (*s++ != '}')
3244     {
3245     errwhere = US"'no' part did not start with '{'";
3246     goto FAILED_CURLY;
3247     }
3248
3249   /* If we want the second string, add it to the output */
3250
3251   if (!yes)
3252     *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub2);
3253   }
3254
3255 /* If there is no second string, but the word "fail" is present when the use of
3256 the second string is wanted, set a flag indicating it was a forced failure
3257 rather than a syntactic error. Swallow the terminating } in case this is nested
3258 inside another lookup or if or extract. */
3259
3260 else if (*s != '}')
3261   {
3262   uschar name[256];
3263   /* deconst cast ok here as source is s anyway */
3264   s = US read_name(name, sizeof(name), s, US"_");
3265   if (Ustrcmp(name, "fail") == 0)
3266     {
3267     if (!yes && !skipping)
3268       {
3269       while (isspace(*s)) s++;
3270       if (*s++ != '}')
3271         {
3272         errwhere = US"did not close with '}' after forcedfail";
3273         goto FAILED_CURLY;
3274         }
3275       expand_string_message =
3276         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3277       expand_string_forcedfail = TRUE;
3278       goto FAILED;
3279       }
3280     }
3281   else
3282     {
3283     expand_string_message =
3284       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3285     goto FAILED;
3286     }
3287   }
3288
3289 /* All we have to do now is to check on the final closing brace. */
3290
3291 while (isspace(*s)) s++;
3292 if (*s++ != '}')
3293   {
3294   errwhere = US"did not close with '}'";
3295   goto FAILED_CURLY;
3296   }
3297
3298
3299 RETURN:
3300 /* Update the input pointer value before returning */
3301 *sptr = s;
3302 return rc;
3303
3304 FAILED_CURLY:
3305   /* Get here if there is a bracketing failure */
3306   expand_string_message = string_sprintf(
3307     "curly-bracket problem in conditional yes/no parsing: %s\n"
3308     " remaining string is '%s'", errwhere, --s);
3309   rc = 2;
3310   goto RETURN;
3311
3312 FAILED:
3313   /* Get here for other failures */
3314   rc = 1;
3315   goto RETURN;
3316 }
3317
3318
3319
3320
3321 /*************************************************
3322 *    Handle MD5 or SHA-1 computation for HMAC    *
3323 *************************************************/
3324
3325 /* These are some wrapping functions that enable the HMAC code to be a bit
3326 cleaner. A good compiler will spot the tail recursion.
3327
3328 Arguments:
3329   type         HMAC_MD5 or HMAC_SHA1
3330   remaining    are as for the cryptographic hash functions
3331
3332 Returns:       nothing
3333 */
3334
3335 static void
3336 chash_start(int type, void *base)
3337 {
3338 if (type == HMAC_MD5)
3339   md5_start((md5 *)base);
3340 else
3341   sha1_start((hctx *)base);
3342 }
3343
3344 static void
3345 chash_mid(int type, void *base, uschar *string)
3346 {
3347 if (type == HMAC_MD5)
3348   md5_mid((md5 *)base, string);
3349 else
3350   sha1_mid((hctx *)base, string);
3351 }
3352
3353 static void
3354 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3355 {
3356 if (type == HMAC_MD5)
3357   md5_end((md5 *)base, string, length, digest);
3358 else
3359   sha1_end((hctx *)base, string, length, digest);
3360 }
3361
3362
3363
3364
3365
3366 /********************************************************
3367 * prvs: Get last three digits of days since Jan 1, 1970 *
3368 ********************************************************/
3369
3370 /* This is needed to implement the "prvs" BATV reverse
3371    path signing scheme
3372
3373 Argument: integer "days" offset to add or substract to
3374           or from the current number of days.
3375
3376 Returns:  pointer to string containing the last three
3377           digits of the number of days since Jan 1, 1970,
3378           modified by the offset argument, NULL if there
3379           was an error in the conversion.
3380
3381 */
3382
3383 static uschar *
3384 prvs_daystamp(int day_offset)
3385 {
3386 uschar *days = store_get(32);                /* Need at least 24 for cases */
3387 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3388   (time(NULL) + day_offset*86400)/86400);
3389 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3390 }
3391
3392
3393
3394 /********************************************************
3395 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3396 ********************************************************/
3397
3398 /* This is needed to implement the "prvs" BATV reverse
3399    path signing scheme
3400
3401 Arguments:
3402   address RFC2821 Address to use
3403       key The key to use (must be less than 64 characters
3404           in size)
3405   key_num Single-digit key number to use. Defaults to
3406           '0' when NULL.
3407
3408 Returns:  pointer to string containing the first three
3409           bytes of the final hash in hex format, NULL if
3410           there was an error in the process.
3411 */
3412
3413 static uschar *
3414 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3415 {
3416 uschar *hash_source, *p;
3417 int size = 0,offset = 0,i;
3418 hctx h;
3419 uschar innerhash[20];
3420 uschar finalhash[20];
3421 uschar innerkey[64];
3422 uschar outerkey[64];
3423 uschar *finalhash_hex = store_get(40);
3424
3425 if (key_num == NULL)
3426   key_num = US"0";
3427
3428 if (Ustrlen(key) > 64)
3429   return NULL;
3430
3431 hash_source = string_catn(NULL, &size, &offset, key_num, 1);
3432 hash_source = string_catn(hash_source, &size, &offset, daystamp, 3);
3433 hash_source = string_cat(hash_source, &size, &offset, address);
3434 hash_source[offset] = '\0';
3435
3436 DEBUG(D_expand) debug_printf("prvs: hash source is '%s'\n", hash_source);
3437
3438 memset(innerkey, 0x36, 64);
3439 memset(outerkey, 0x5c, 64);
3440
3441 for (i = 0; i < Ustrlen(key); i++)
3442   {
3443   innerkey[i] ^= key[i];
3444   outerkey[i] ^= key[i];
3445   }
3446
3447 chash_start(HMAC_SHA1, &h);
3448 chash_mid(HMAC_SHA1, &h, innerkey);
3449 chash_end(HMAC_SHA1, &h, hash_source, offset, innerhash);
3450
3451 chash_start(HMAC_SHA1, &h);
3452 chash_mid(HMAC_SHA1, &h, outerkey);
3453 chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash);
3454
3455 p = finalhash_hex;
3456 for (i = 0; i < 3; i++)
3457   {
3458   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3459   *p++ = hex_digits[finalhash[i] & 0x0f];
3460   }
3461 *p = '\0';
3462
3463 return finalhash_hex;
3464 }
3465
3466
3467
3468
3469 /*************************************************
3470 *        Join a file onto the output string      *
3471 *************************************************/
3472
3473 /* This is used for readfile/readsock and after a run expansion.
3474 It joins the contents of a file onto the output string, globally replacing
3475 newlines with a given string (optionally).
3476
3477 Arguments:
3478   f            the FILE
3479   yield        pointer to the expandable string
3480   sizep        pointer to the current size
3481   ptrp         pointer to the current position
3482   eol          newline replacement string, or NULL
3483
3484 Returns:       new value of string pointer
3485 */
3486
3487 static uschar *
3488 cat_file(FILE *f, uschar *yield, int *sizep, int *ptrp, uschar *eol)
3489 {
3490 int eollen = eol ? Ustrlen(eol) : 0;
3491 uschar buffer[1024];
3492
3493 while (Ufgets(buffer, sizeof(buffer), f))
3494   {
3495   int len = Ustrlen(buffer);
3496   if (eol && buffer[len-1] == '\n') len--;
3497   yield = string_catn(yield, sizep, ptrp, buffer, len);
3498   if (buffer[len] != 0)
3499     yield = string_catn(yield, sizep, ptrp, eol, eollen);
3500   }
3501
3502 if (yield) yield[*ptrp] = 0;
3503
3504 return yield;
3505 }
3506
3507
3508
3509
3510 /*************************************************
3511 *          Evaluate numeric expression           *
3512 *************************************************/
3513
3514 /* This is a set of mutually recursive functions that evaluate an arithmetic
3515 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3516 these functions that is called from elsewhere is eval_expr, whose interface is:
3517
3518 Arguments:
3519   sptr        pointer to the pointer to the string - gets updated
3520   decimal     TRUE if numbers are to be assumed decimal
3521   error       pointer to where to put an error message - must be NULL on input
3522   endket      TRUE if ')' must terminate - FALSE for external call
3523
3524 Returns:      on success: the value of the expression, with *error still NULL
3525               on failure: an undefined value, with *error = a message
3526 */
3527
3528 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3529
3530
3531 static int_eximarith_t
3532 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3533 {
3534 uschar *s = *sptr;
3535 int_eximarith_t x = eval_op_or(&s, decimal, error);
3536 if (*error == NULL)
3537   {
3538   if (endket)
3539     {
3540     if (*s != ')')
3541       *error = US"expecting closing parenthesis";
3542     else
3543       while (isspace(*(++s)));
3544     }
3545   else if (*s != 0) *error = US"expecting operator";
3546   }
3547 *sptr = s;
3548 return x;
3549 }
3550
3551
3552 static int_eximarith_t
3553 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3554 {
3555 register int c;
3556 int_eximarith_t n;
3557 uschar *s = *sptr;
3558 while (isspace(*s)) s++;
3559 c = *s;
3560 if (isdigit(c))
3561   {
3562   int count;
3563   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3564   s += count;
3565   switch (tolower(*s))
3566     {
3567     default: break;
3568     case 'k': n *= 1024; s++; break;
3569     case 'm': n *= 1024*1024; s++; break;
3570     case 'g': n *= 1024*1024*1024; s++; break;
3571     }
3572   while (isspace (*s)) s++;
3573   }
3574 else if (c == '(')
3575   {
3576   s++;
3577   n = eval_expr(&s, decimal, error, 1);
3578   }
3579 else
3580   {
3581   *error = US"expecting number or opening parenthesis";
3582   n = 0;
3583   }
3584 *sptr = s;
3585 return n;
3586 }
3587
3588
3589 static int_eximarith_t
3590 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3591 {
3592 uschar *s = *sptr;
3593 int_eximarith_t x;
3594 while (isspace(*s)) s++;
3595 if (*s == '+' || *s == '-' || *s == '~')
3596   {
3597   int op = *s++;
3598   x = eval_op_unary(&s, decimal, error);
3599   if (op == '-') x = -x;
3600     else if (op == '~') x = ~x;
3601   }
3602 else
3603   {
3604   x = eval_number(&s, decimal, error);
3605   }
3606 *sptr = s;
3607 return x;
3608 }
3609
3610
3611 static int_eximarith_t
3612 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3613 {
3614 uschar *s = *sptr;
3615 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3616 if (*error == NULL)
3617   {
3618   while (*s == '*' || *s == '/' || *s == '%')
3619     {
3620     int op = *s++;
3621     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3622     if (*error != NULL) break;
3623     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3624      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3625      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3626      * -N*M is INT_MIN will yielf INT_MIN.
3627      * Since we don't support floating point, this is somewhat simpler.
3628      * Ideally, we'd return an error, but since we overflow for all other
3629      * arithmetic, consistency suggests otherwise, but what's the correct value
3630      * to use?  There is none.
3631      * The C standard guarantees overflow for unsigned arithmetic but signed
3632      * overflow invokes undefined behaviour; in practice, this is overflow
3633      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3634      * that long/longlong larger than int are available, or we could just work
3635      * with larger types.  We should consider whether to guarantee 32bit eval
3636      * and 64-bit working variables, with errors returned.  For now ...
3637      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3638      * can just let the other invalid results occur otherwise, as they have
3639      * until now.  For this one case, we can coerce.
3640      */
3641     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3642       {
3643       DEBUG(D_expand)
3644         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3645             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3646       x = EXIM_ARITH_MAX;
3647       continue;
3648       }
3649     if (op == '*')
3650       x *= y;
3651     else
3652       {
3653       if (y == 0)
3654         {
3655         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3656         x = 0;
3657         break;
3658         }
3659       if (op == '/')
3660         x /= y;
3661       else
3662         x %= y;
3663       }
3664     }
3665   }
3666 *sptr = s;
3667 return x;
3668 }
3669
3670
3671 static int_eximarith_t
3672 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3673 {
3674 uschar *s = *sptr;
3675 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3676 if (!*error)
3677   {
3678   while (*s == '+' || *s == '-')
3679     {
3680     int op = *s++;
3681     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3682     if (*error) break;
3683     if (  (x >=   EXIM_ARITH_MAX/2  && x >=   EXIM_ARITH_MAX/2)
3684        || (x <= -(EXIM_ARITH_MAX/2) && y <= -(EXIM_ARITH_MAX/2)))
3685       {                 /* over-conservative check */
3686       *error = op == '+'
3687         ? US"overflow in sum" : US"overflow in difference";
3688       break;
3689       }
3690     if (op == '+') x += y; else x -= y;
3691     }
3692   }
3693 *sptr = s;
3694 return x;
3695 }
3696
3697
3698 static int_eximarith_t
3699 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3700 {
3701 uschar *s = *sptr;
3702 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3703 if (*error == NULL)
3704   {
3705   while ((*s == '<' || *s == '>') && s[1] == s[0])
3706     {
3707     int_eximarith_t y;
3708     int op = *s++;
3709     s++;
3710     y = eval_op_sum(&s, decimal, error);
3711     if (*error != NULL) break;
3712     if (op == '<') x <<= y; else x >>= y;
3713     }
3714   }
3715 *sptr = s;
3716 return x;
3717 }
3718
3719
3720 static int_eximarith_t
3721 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3722 {
3723 uschar *s = *sptr;
3724 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3725 if (*error == NULL)
3726   {
3727   while (*s == '&')
3728     {
3729     int_eximarith_t y;
3730     s++;
3731     y = eval_op_shift(&s, decimal, error);
3732     if (*error != NULL) break;
3733     x &= y;
3734     }
3735   }
3736 *sptr = s;
3737 return x;
3738 }
3739
3740
3741 static int_eximarith_t
3742 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3743 {
3744 uschar *s = *sptr;
3745 int_eximarith_t x = eval_op_and(&s, decimal, error);
3746 if (*error == NULL)
3747   {
3748   while (*s == '^')
3749     {
3750     int_eximarith_t y;
3751     s++;
3752     y = eval_op_and(&s, decimal, error);
3753     if (*error != NULL) break;
3754     x ^= y;
3755     }
3756   }
3757 *sptr = s;
3758 return x;
3759 }
3760
3761
3762 static int_eximarith_t
3763 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3764 {
3765 uschar *s = *sptr;
3766 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3767 if (*error == NULL)
3768   {
3769   while (*s == '|')
3770     {
3771     int_eximarith_t y;
3772     s++;
3773     y = eval_op_xor(&s, decimal, error);
3774     if (*error != NULL) break;
3775     x |= y;
3776     }
3777   }
3778 *sptr = s;
3779 return x;
3780 }
3781
3782
3783
3784 /*************************************************
3785 *                 Expand string                  *
3786 *************************************************/
3787
3788 /* Returns either an unchanged string, or the expanded string in stacking pool
3789 store. Interpreted sequences are:
3790
3791    \...                    normal escaping rules
3792    $name                   substitutes the variable
3793    ${name}                 ditto
3794    ${op:string}            operates on the expanded string value
3795    ${item{arg1}{arg2}...}  expands the args and then does the business
3796                              some literal args are not enclosed in {}
3797
3798 There are now far too many operators and item types to make it worth listing
3799 them here in detail any more.
3800
3801 We use an internal routine recursively to handle embedded substrings. The
3802 external function follows. The yield is NULL if the expansion failed, and there
3803 are two cases: if something collapsed syntactically, or if "fail" was given
3804 as the action on a lookup failure. These can be distinguised by looking at the
3805 variable expand_string_forcedfail, which is TRUE in the latter case.
3806
3807 The skipping flag is set true when expanding a substring that isn't actually
3808 going to be used (after "if" or "lookup") and it prevents lookups from
3809 happening lower down.
3810
3811 Store usage: At start, a store block of the length of the input plus 64
3812 is obtained. This is expanded as necessary by string_cat(), which might have to
3813 get a new block, or might be able to expand the original. At the end of the
3814 function we can release any store above that portion of the yield block that
3815 was actually used. In many cases this will be optimal.
3816
3817 However: if the first item in the expansion is a variable name or header name,
3818 we reset the store before processing it; if the result is in fresh store, we
3819 use that without copying. This is helpful for expanding strings like
3820 $message_headers which can get very long.
3821
3822 There's a problem if a ${dlfunc item has side-effects that cause allocation,
3823 since resetting the store at the end of the expansion will free store that was
3824 allocated by the plugin code as well as the slop after the expanded string. So
3825 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
3826 and, given the acl condition, ${if }. This is an unfortunate consequence of
3827 string expansion becoming too powerful.
3828
3829 Arguments:
3830   string         the string to be expanded
3831   ket_ends       true if expansion is to stop at }
3832   left           if not NULL, a pointer to the first character after the
3833                  expansion is placed here (typically used with ket_ends)
3834   skipping       TRUE for recursive calls when the value isn't actually going
3835                  to be used (to allow for optimisation)
3836   honour_dollar  TRUE if $ is to be expanded,
3837                  FALSE if it's just another character
3838   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
3839                  the store.
3840
3841 Returns:         NULL if expansion fails:
3842                    expand_string_forcedfail is set TRUE if failure was forced
3843                    expand_string_message contains a textual error message
3844                  a pointer to the expanded string on success
3845 */
3846
3847 static uschar *
3848 expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left,
3849   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
3850 {
3851 int ptr = 0;
3852 int size = Ustrlen(string)+ 64;
3853 uschar *yield = store_get(size);
3854 int item_type;
3855 const uschar *s = string;
3856 uschar *save_expand_nstring[EXPAND_MAXN+1];
3857 int save_expand_nlength[EXPAND_MAXN+1];
3858 BOOL resetok = TRUE;
3859
3860 DEBUG(D_expand)
3861   debug_printf("%s: %s\n", skipping ? "   scanning" : "considering", string);
3862
3863 expand_string_forcedfail = FALSE;
3864 expand_string_message = US"";
3865
3866 while (*s != 0)
3867   {
3868   uschar *value;
3869   uschar name[256];
3870
3871   /* \ escapes the next character, which must exist, or else
3872   the expansion fails. There's a special escape, \N, which causes
3873   copying of the subject verbatim up to the next \N. Otherwise,
3874   the escapes are the standard set. */
3875
3876   if (*s == '\\')
3877     {
3878     if (s[1] == 0)
3879       {
3880       expand_string_message = US"\\ at end of string";
3881       goto EXPAND_FAILED;
3882       }
3883
3884     if (s[1] == 'N')
3885       {
3886       const uschar * t = s + 2;
3887       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
3888       yield = string_catn(yield, &size, &ptr, t, s - t);
3889       if (*s != 0) s += 2;
3890       }
3891
3892     else
3893       {
3894       uschar ch[1];
3895       ch[0] = string_interpret_escape(&s);
3896       s++;
3897       yield = string_catn(yield, &size, &ptr, ch, 1);
3898       }
3899
3900     continue;
3901     }
3902
3903   /*{*/
3904   /* Anything other than $ is just copied verbatim, unless we are
3905   looking for a terminating } character. */
3906
3907   /*{*/
3908   if (ket_ends && *s == '}') break;
3909
3910   if (*s != '$' || !honour_dollar)
3911     {
3912     yield = string_catn(yield, &size, &ptr, s++, 1);
3913     continue;
3914     }
3915
3916   /* No { after the $ - must be a plain name or a number for string
3917   match variable. There has to be a fudge for variables that are the
3918   names of header fields preceded by "$header_" because header field
3919   names can contain any printing characters except space and colon.
3920   For those that don't like typing this much, "$h_" is a synonym for
3921   "$header_". A non-existent header yields a NULL value; nothing is
3922   inserted. */  /*}*/
3923
3924   if (isalpha((*(++s))))
3925     {
3926     int len;
3927     int newsize = 0;
3928
3929     s = read_name(name, sizeof(name), s, US"_");
3930
3931     /* If this is the first thing to be expanded, release the pre-allocated
3932     buffer. */
3933
3934     if (ptr == 0 && yield != NULL)
3935       {
3936       if (resetok) store_reset(yield);
3937       yield = NULL;
3938       size = 0;
3939       }
3940
3941     /* Header */
3942
3943     if (Ustrncmp(name, "h_", 2) == 0 ||
3944         Ustrncmp(name, "rh_", 3) == 0 ||
3945         Ustrncmp(name, "bh_", 3) == 0 ||
3946         Ustrncmp(name, "header_", 7) == 0 ||
3947         Ustrncmp(name, "rheader_", 8) == 0 ||
3948         Ustrncmp(name, "bheader_", 8) == 0)
3949       {
3950       BOOL want_raw = (name[0] == 'r')? TRUE : FALSE;
3951       uschar *charset = (name[0] == 'b')? NULL : headers_charset;
3952       s = read_header_name(name, sizeof(name), s);
3953       value = find_header(name, FALSE, &newsize, want_raw, charset);
3954
3955       /* If we didn't find the header, and the header contains a closing brace
3956       character, this may be a user error where the terminating colon
3957       has been omitted. Set a flag to adjust the error message in this case.
3958       But there is no error here - nothing gets inserted. */
3959
3960       if (value == NULL)
3961         {
3962         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
3963         continue;
3964         }
3965       }
3966
3967     /* Variable */
3968
3969     else if (!(value = find_variable(name, FALSE, skipping, &newsize)))
3970       {
3971       expand_string_message =
3972         string_sprintf("unknown variable name \"%s\"", name);
3973         check_variable_error_message(name);
3974       goto EXPAND_FAILED;
3975       }
3976
3977     /* If the data is known to be in a new buffer, newsize will be set to the
3978     size of that buffer. If this is the first thing in an expansion string,
3979     yield will be NULL; just point it at the new store instead of copying. Many
3980     expansion strings contain just one reference, so this is a useful
3981     optimization, especially for humungous headers. */
3982
3983     len = Ustrlen(value);
3984     if (yield == NULL && newsize != 0)
3985       {
3986       yield = value;
3987       size = newsize;
3988       ptr = len;
3989       }
3990     else yield = string_catn(yield, &size, &ptr, value, len);
3991
3992     continue;
3993     }
3994
3995   if (isdigit(*s))
3996     {
3997     int n;
3998     s = read_cnumber(&n, s);
3999     if (n >= 0 && n <= expand_nmax)
4000       yield = string_catn(yield, &size, &ptr, expand_nstring[n],
4001         expand_nlength[n]);
4002     continue;
4003     }
4004
4005   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
4006
4007   if (*s != '{')                                                        /*}*/
4008     {
4009     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
4010     goto EXPAND_FAILED;
4011     }
4012
4013   /* After { there can be various things, but they all start with
4014   an initial word, except for a number for a string match variable. */
4015
4016   if (isdigit((*(++s))))
4017     {
4018     int n;
4019     s = read_cnumber(&n, s);            /*{*/
4020     if (*s++ != '}')
4021       {                                 /*{*/
4022       expand_string_message = US"} expected after number";
4023       goto EXPAND_FAILED;
4024       }
4025     if (n >= 0 && n <= expand_nmax)
4026       yield = string_catn(yield, &size, &ptr, expand_nstring[n],
4027         expand_nlength[n]);
4028     continue;
4029     }
4030
4031   if (!isalpha(*s))
4032     {
4033     expand_string_message = US"letter or digit expected after ${";      /*}*/
4034     goto EXPAND_FAILED;
4035     }
4036
4037   /* Allow "-" in names to cater for substrings with negative
4038   arguments. Since we are checking for known names after { this is
4039   OK. */
4040
4041   s = read_name(name, sizeof(name), s, US"_-");
4042   item_type = chop_match(name, item_table, nelem(item_table));
4043
4044   switch(item_type)
4045     {
4046     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
4047     If the ACL returns accept or reject we return content set by "message ="
4048     There is currently no limit on recursion; this would have us call
4049     acl_check_internal() directly and get a current level from somewhere.
4050     See also the acl expansion condition ECOND_ACL and the traditional
4051     acl modifier ACLC_ACL.
4052     Assume that the function has side-effects on the store that must be preserved.
4053     */
4054
4055     case EITEM_ACL:
4056       /* ${acl {name} {arg1}{arg2}...} */
4057       {
4058       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
4059       uschar *user_msg;
4060
4061       switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, US"acl",
4062                       &resetok))
4063         {
4064         case 1: goto EXPAND_FAILED_CURLY;
4065         case 2:
4066         case 3: goto EXPAND_FAILED;
4067         }
4068       if (skipping) continue;
4069
4070       resetok = FALSE;
4071       switch(eval_acl(sub, nelem(sub), &user_msg))
4072         {
4073         case OK:
4074         case FAIL:
4075           DEBUG(D_expand)
4076             debug_printf("acl expansion yield: %s\n", user_msg);
4077           if (user_msg)
4078             yield = string_cat(yield, &size, &ptr, user_msg);
4079           continue;
4080
4081         case DEFER:
4082           expand_string_forcedfail = TRUE;
4083           /*FALLTHROUGH*/
4084         default:
4085           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
4086           goto EXPAND_FAILED;
4087         }
4088       }
4089
4090     /* Handle conditionals - preserve the values of the numerical expansion
4091     variables in case they get changed by a regular expression match in the
4092     condition. If not, they retain their external settings. At the end
4093     of this "if" section, they get restored to their previous values. */
4094
4095     case EITEM_IF:
4096       {
4097       BOOL cond = FALSE;
4098       const uschar *next_s;
4099       int save_expand_nmax =
4100         save_expand_strings(save_expand_nstring, save_expand_nlength);
4101
4102       while (isspace(*s)) s++;
4103       next_s = eval_condition(s, &resetok, skipping? NULL : &cond);
4104       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
4105
4106       DEBUG(D_expand)
4107         debug_printf("condition: %.*s\n   result: %s\n", (int)(next_s - s), s,
4108           cond? "true" : "false");
4109
4110       s = next_s;
4111
4112       /* The handling of "yes" and "no" result strings is now in a separate
4113       function that is also used by ${lookup} and ${extract} and ${run}. */
4114
4115       switch(process_yesno(
4116                skipping,                     /* were previously skipping */
4117                cond,                         /* success/failure indicator */
4118                lookup_value,                 /* value to reset for string2 */
4119                &s,                           /* input pointer */
4120                &yield,                       /* output pointer */
4121                &size,                        /* output size */
4122                &ptr,                         /* output current point */
4123                US"if",                       /* condition type */
4124                &resetok))
4125         {
4126         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4127         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4128         }
4129
4130       /* Restore external setting of expansion variables for continuation
4131       at this level. */
4132
4133       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4134         save_expand_nlength);
4135       continue;
4136       }
4137
4138 #ifdef SUPPORT_I18N
4139     case EITEM_IMAPFOLDER:
4140       {                         /* ${imapfolder {name}{sep]{specials}} */
4141       uschar *sub_arg[3];
4142       uschar *encoded;
4143
4144       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4145                       &resetok))
4146         {
4147         case 1: goto EXPAND_FAILED_CURLY;
4148         case 2:
4149         case 3: goto EXPAND_FAILED;
4150         }
4151
4152       if (sub_arg[1] == NULL)           /* One argument */
4153         {
4154         sub_arg[1] = US"/";             /* default separator */
4155         sub_arg[2] = NULL;
4156         }
4157       else if (Ustrlen(sub_arg[1]) != 1)
4158         {
4159         expand_string_message =
4160           string_sprintf(
4161                 "IMAP folder separator must be one character, found \"%s\"",
4162                 sub_arg[1]);
4163         goto EXPAND_FAILED;
4164         }
4165
4166       if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
4167                           sub_arg[1][0], sub_arg[2], &expand_string_message)))
4168         goto EXPAND_FAILED;
4169       if (!skipping)
4170         yield = string_cat(yield, &size, &ptr, encoded);
4171       continue;
4172       }
4173 #endif
4174
4175     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4176     expanding an internal string that isn't actually going to be used. All we
4177     need to do is check the syntax, so don't do a lookup at all. Preserve the
4178     values of the numerical expansion variables in case they get changed by a
4179     partial lookup. If not, they retain their external settings. At the end
4180     of this "lookup" section, they get restored to their previous values. */
4181
4182     case EITEM_LOOKUP:
4183       {
4184       int stype, partial, affixlen, starflags;
4185       int expand_setup = 0;
4186       int nameptr = 0;
4187       uschar *key, *filename;
4188       const uschar *affix;
4189       uschar *save_lookup_value = lookup_value;
4190       int save_expand_nmax =
4191         save_expand_strings(save_expand_nstring, save_expand_nlength);
4192
4193       if ((expand_forbid & RDO_LOOKUP) != 0)
4194         {
4195         expand_string_message = US"lookup expansions are not permitted";
4196         goto EXPAND_FAILED;
4197         }
4198
4199       /* Get the key we are to look up for single-key+file style lookups.
4200       Otherwise set the key NULL pro-tem. */
4201
4202       while (isspace(*s)) s++;
4203       if (*s == '{')                                    /*}*/
4204         {
4205         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4206         if (!key) goto EXPAND_FAILED;                   /*{{*/
4207         if (*s++ != '}')
4208           {
4209           expand_string_message = US"missing '}' after lookup key";
4210           goto EXPAND_FAILED_CURLY;
4211           }
4212         while (isspace(*s)) s++;
4213         }
4214       else key = NULL;
4215
4216       /* Find out the type of database */
4217
4218       if (!isalpha(*s))
4219         {
4220         expand_string_message = US"missing lookup type";
4221         goto EXPAND_FAILED;
4222         }
4223
4224       /* The type is a string that may contain special characters of various
4225       kinds. Allow everything except space or { to appear; the actual content
4226       is checked by search_findtype_partial. */         /*}*/
4227
4228       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4229         {
4230         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4231         s++;
4232         }
4233       name[nameptr] = 0;
4234       while (isspace(*s)) s++;
4235
4236       /* Now check for the individual search type and any partial or default
4237       options. Only those types that are actually in the binary are valid. */
4238
4239       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4240         &starflags);
4241       if (stype < 0)
4242         {
4243         expand_string_message = search_error_message;
4244         goto EXPAND_FAILED;
4245         }
4246
4247       /* Check that a key was provided for those lookup types that need it,
4248       and was not supplied for those that use the query style. */
4249
4250       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4251         {
4252         if (key == NULL)
4253           {
4254           expand_string_message = string_sprintf("missing {key} for single-"
4255             "key \"%s\" lookup", name);
4256           goto EXPAND_FAILED;
4257           }
4258         }
4259       else
4260         {
4261         if (key != NULL)
4262           {
4263           expand_string_message = string_sprintf("a single key was given for "
4264             "lookup type \"%s\", which is not a single-key lookup type", name);
4265           goto EXPAND_FAILED;
4266           }
4267         }
4268
4269       /* Get the next string in brackets and expand it. It is the file name for
4270       single-key+file lookups, and the whole query otherwise. In the case of
4271       queries that also require a file name (e.g. sqlite), the file name comes
4272       first. */
4273
4274       if (*s != '{')
4275         {
4276         expand_string_message = US"missing '{' for lookup file-or-query arg";
4277         goto EXPAND_FAILED_CURLY;
4278         }
4279       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4280       if (filename == NULL) goto EXPAND_FAILED;
4281       if (*s++ != '}')
4282         {
4283         expand_string_message = US"missing '}' closing lookup file-or-query arg";
4284         goto EXPAND_FAILED_CURLY;
4285         }
4286       while (isspace(*s)) s++;
4287
4288       /* If this isn't a single-key+file lookup, re-arrange the variables
4289       to be appropriate for the search_ functions. For query-style lookups,
4290       there is just a "key", and no file name. For the special query-style +
4291       file types, the query (i.e. "key") starts with a file name. */
4292
4293       if (!key)
4294         {
4295         while (isspace(*filename)) filename++;
4296         key = filename;
4297
4298         if (mac_islookup(stype, lookup_querystyle))
4299           filename = NULL;
4300         else
4301           {
4302           if (*filename != '/')
4303             {
4304             expand_string_message = string_sprintf(
4305               "absolute file name expected for \"%s\" lookup", name);
4306             goto EXPAND_FAILED;
4307             }
4308           while (*key != 0 && !isspace(*key)) key++;
4309           if (*key != 0) *key++ = 0;
4310           }
4311         }
4312
4313       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4314       the entry was not found. Note that there is no search_close() function.
4315       Files are left open in case of re-use. At suitable places in higher logic,
4316       search_tidyup() is called to tidy all open files. This can save opening
4317       the same file several times. However, files may also get closed when
4318       others are opened, if too many are open at once. The rule is that a
4319       handle should not be used after a second search_open().
4320
4321       Request that a partial search sets up $1 and maybe $2 by passing
4322       expand_setup containing zero. If its value changes, reset expand_nmax,
4323       since new variables will have been set. Note that at the end of this
4324       "lookup" section, the old numeric variables are restored. */
4325
4326       if (skipping)
4327         lookup_value = NULL;
4328       else
4329         {
4330         void *handle = search_open(filename, stype, 0, NULL, NULL);
4331         if (handle == NULL)
4332           {
4333           expand_string_message = search_error_message;
4334           goto EXPAND_FAILED;
4335           }
4336         lookup_value = search_find(handle, filename, key, partial, affix,
4337           affixlen, starflags, &expand_setup);
4338         if (search_find_defer)
4339           {
4340           expand_string_message =
4341             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4342               string_printing2(key, FALSE), search_error_message);
4343           goto EXPAND_FAILED;
4344           }
4345         if (expand_setup > 0) expand_nmax = expand_setup;
4346         }
4347
4348       /* The handling of "yes" and "no" result strings is now in a separate
4349       function that is also used by ${if} and ${extract}. */
4350
4351       switch(process_yesno(
4352                skipping,                     /* were previously skipping */
4353                lookup_value != NULL,         /* success/failure indicator */
4354                save_lookup_value,            /* value to reset for string2 */
4355                &s,                           /* input pointer */
4356                &yield,                       /* output pointer */
4357                &size,                        /* output size */
4358                &ptr,                         /* output current point */
4359                US"lookup",                   /* condition type */
4360                &resetok))
4361         {
4362         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4363         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4364         }
4365
4366       /* Restore external setting of expansion variables for carrying on
4367       at this level, and continue. */
4368
4369       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4370         save_expand_nlength);
4371       continue;
4372       }
4373
4374     /* If Perl support is configured, handle calling embedded perl subroutines,
4375     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4376     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4377     arguments (defined below). */
4378
4379     #define EXIM_PERL_MAX_ARGS 8
4380
4381     case EITEM_PERL:
4382     #ifndef EXIM_PERL
4383     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4384       "is not included in this binary";
4385     goto EXPAND_FAILED;
4386
4387     #else   /* EXIM_PERL */
4388       {
4389       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4390       uschar *new_yield;
4391
4392       if ((expand_forbid & RDO_PERL) != 0)
4393         {
4394         expand_string_message = US"Perl calls are not permitted";
4395         goto EXPAND_FAILED;
4396         }
4397
4398       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4399            US"perl", &resetok))
4400         {
4401         case 1: goto EXPAND_FAILED_CURLY;
4402         case 2:
4403         case 3: goto EXPAND_FAILED;
4404         }
4405
4406       /* If skipping, we don't actually do anything */
4407
4408       if (skipping) continue;
4409
4410       /* Start the interpreter if necessary */
4411
4412       if (!opt_perl_started)
4413         {
4414         uschar *initerror;
4415         if (opt_perl_startup == NULL)
4416           {
4417           expand_string_message = US"A setting of perl_startup is needed when "
4418             "using the Perl interpreter";
4419           goto EXPAND_FAILED;
4420           }
4421         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4422         initerror = init_perl(opt_perl_startup);
4423         if (initerror != NULL)
4424           {
4425           expand_string_message =
4426             string_sprintf("error in perl_startup code: %s\n", initerror);
4427           goto EXPAND_FAILED;
4428           }
4429         opt_perl_started = TRUE;
4430         }
4431
4432       /* Call the function */
4433
4434       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4435       new_yield = call_perl_cat(yield, &size, &ptr, &expand_string_message,
4436         sub_arg[0], sub_arg + 1);
4437
4438       /* NULL yield indicates failure; if the message pointer has been set to
4439       NULL, the yield was undef, indicating a forced failure. Otherwise the
4440       message will indicate some kind of Perl error. */
4441
4442       if (new_yield == NULL)
4443         {
4444         if (expand_string_message == NULL)
4445           {
4446           expand_string_message =
4447             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4448               "failure", sub_arg[0]);
4449           expand_string_forcedfail = TRUE;
4450           }
4451         goto EXPAND_FAILED;
4452         }
4453
4454       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4455       set during a callback from Perl. */
4456
4457       expand_string_forcedfail = FALSE;
4458       yield = new_yield;
4459       continue;
4460       }
4461     #endif /* EXIM_PERL */
4462
4463     /* Transform email address to "prvs" scheme to use
4464        as BATV-signed return path */
4465
4466     case EITEM_PRVS:
4467       {
4468       uschar *sub_arg[3];
4469       uschar *p,*domain;
4470
4471       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4472         {
4473         case 1: goto EXPAND_FAILED_CURLY;
4474         case 2:
4475         case 3: goto EXPAND_FAILED;
4476         }
4477
4478       /* If skipping, we don't actually do anything */
4479       if (skipping) continue;
4480
4481       /* sub_arg[0] is the address */
4482       domain = Ustrrchr(sub_arg[0],'@');
4483       if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) )
4484         {
4485         expand_string_message = US"prvs first argument must be a qualified email address";
4486         goto EXPAND_FAILED;
4487         }
4488
4489       /* Calculate the hash. The second argument must be a single-digit
4490       key number, or unset. */
4491
4492       if (sub_arg[2] != NULL &&
4493           (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4494         {
4495         expand_string_message = US"prvs second argument must be a single digit";
4496         goto EXPAND_FAILED;
4497         }
4498
4499       p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7));
4500       if (p == NULL)
4501         {
4502         expand_string_message = US"prvs hmac-sha1 conversion failed";
4503         goto EXPAND_FAILED;
4504         }
4505
4506       /* Now separate the domain from the local part */
4507       *domain++ = '\0';
4508
4509       yield = string_catn(yield, &size, &ptr, US"prvs=", 5);
4510       yield = string_catn(yield, &size, &ptr, sub_arg[2] ? sub_arg[2] : US"0", 1);
4511       yield = string_catn(yield, &size, &ptr, prvs_daystamp(7), 3);
4512       yield = string_catn(yield, &size, &ptr, p, 6);
4513       yield = string_catn(yield, &size, &ptr, US"=", 1);
4514       yield = string_cat (yield, &size, &ptr, sub_arg[0]);
4515       yield = string_catn(yield, &size, &ptr, US"@", 1);
4516       yield = string_cat (yield, &size, &ptr, domain);
4517
4518       continue;
4519       }
4520
4521     /* Check a prvs-encoded address for validity */
4522
4523     case EITEM_PRVSCHECK:
4524       {
4525       uschar *sub_arg[3];
4526       int mysize = 0, myptr = 0;
4527       const pcre *re;
4528       uschar *p;
4529
4530       /* TF: Ugliness: We want to expand parameter 1 first, then set
4531          up expansion variables that are used in the expansion of
4532          parameter 2. So we clone the string for the first
4533          expansion, where we only expand parameter 1.
4534
4535          PH: Actually, that isn't necessary. The read_subs() function is
4536          designed to work this way for the ${if and ${lookup expansions. I've
4537          tidied the code.
4538       */
4539
4540       /* Reset expansion variables */
4541       prvscheck_result = NULL;
4542       prvscheck_address = NULL;
4543       prvscheck_keynum = NULL;
4544
4545       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4546         {
4547         case 1: goto EXPAND_FAILED_CURLY;
4548         case 2:
4549         case 3: goto EXPAND_FAILED;
4550         }
4551
4552       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4553                               TRUE,FALSE);
4554
4555       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4556         {
4557         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4558         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4559         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4560         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4561         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4562
4563         DEBUG(D_expand) debug_printf("prvscheck localpart: %s\n", local_part);
4564         DEBUG(D_expand) debug_printf("prvscheck key number: %s\n", key_num);
4565         DEBUG(D_expand) debug_printf("prvscheck daystamp: %s\n", daystamp);
4566         DEBUG(D_expand) debug_printf("prvscheck hash: %s\n", hash);
4567         DEBUG(D_expand) debug_printf("prvscheck domain: %s\n", domain);
4568
4569         /* Set up expansion variables */
4570         prvscheck_address = string_cat (NULL, &mysize, &myptr, local_part);
4571         prvscheck_address = string_catn(prvscheck_address, &mysize, &myptr, US"@", 1);
4572         prvscheck_address = string_cat (prvscheck_address, &mysize, &myptr, domain);
4573         prvscheck_address[myptr] = '\0';
4574         prvscheck_keynum = string_copy(key_num);
4575
4576         /* Now expand the second argument */
4577         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4578           {
4579           case 1: goto EXPAND_FAILED_CURLY;
4580           case 2:
4581           case 3: goto EXPAND_FAILED;
4582           }
4583
4584         /* Now we have the key and can check the address. */
4585
4586         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4587           daystamp);
4588
4589         if (p == NULL)
4590           {
4591           expand_string_message = US"hmac-sha1 conversion failed";
4592           goto EXPAND_FAILED;
4593           }
4594
4595         DEBUG(D_expand) debug_printf("prvscheck: received hash is %s\n", hash);
4596         DEBUG(D_expand) debug_printf("prvscheck:      own hash is %s\n", p);
4597
4598         if (Ustrcmp(p,hash) == 0)
4599           {
4600           /* Success, valid BATV address. Now check the expiry date. */
4601           uschar *now = prvs_daystamp(0);
4602           unsigned int inow = 0,iexpire = 1;
4603
4604           (void)sscanf(CS now,"%u",&inow);
4605           (void)sscanf(CS daystamp,"%u",&iexpire);
4606
4607           /* When "iexpire" is < 7, a "flip" has occured.
4608              Adjust "inow" accordingly. */
4609           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4610
4611           if (iexpire >= inow)
4612             {
4613             prvscheck_result = US"1";
4614             DEBUG(D_expand) debug_printf("prvscheck: success, $pvrs_result set to 1\n");
4615             }
4616             else
4617             {
4618             prvscheck_result = NULL;
4619             DEBUG(D_expand) debug_printf("prvscheck: signature expired, $pvrs_result unset\n");
4620             }
4621           }
4622         else
4623           {
4624           prvscheck_result = NULL;
4625           DEBUG(D_expand) debug_printf("prvscheck: hash failure, $pvrs_result unset\n");
4626           }
4627
4628         /* Now expand the final argument. We leave this till now so that
4629         it can include $prvscheck_result. */
4630
4631         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4632           {
4633           case 1: goto EXPAND_FAILED_CURLY;
4634           case 2:
4635           case 3: goto EXPAND_FAILED;
4636           }
4637
4638         yield = string_cat(yield, &size, &ptr,
4639           !sub_arg[0] || !*sub_arg[0] ? prvscheck_address : sub_arg[0]);
4640
4641         /* Reset the "internal" variables afterwards, because they are in
4642         dynamic store that will be reclaimed if the expansion succeeded. */
4643
4644         prvscheck_address = NULL;
4645         prvscheck_keynum = NULL;
4646         }
4647       else
4648         {
4649         /* Does not look like a prvs encoded address, return the empty string.
4650            We need to make sure all subs are expanded first, so as to skip over
4651            the entire item. */
4652
4653         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4654           {
4655           case 1: goto EXPAND_FAILED_CURLY;
4656           case 2:
4657           case 3: goto EXPAND_FAILED;
4658           }
4659         }
4660
4661       continue;
4662       }
4663
4664     /* Handle "readfile" to insert an entire file */
4665
4666     case EITEM_READFILE:
4667       {
4668       FILE *f;
4669       uschar *sub_arg[2];
4670
4671       if ((expand_forbid & RDO_READFILE) != 0)
4672         {
4673         expand_string_message = US"file insertions are not permitted";
4674         goto EXPAND_FAILED;
4675         }
4676
4677       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4678         {
4679         case 1: goto EXPAND_FAILED_CURLY;
4680         case 2:
4681         case 3: goto EXPAND_FAILED;
4682         }
4683
4684       /* If skipping, we don't actually do anything */
4685
4686       if (skipping) continue;
4687
4688       /* Open the file and read it */
4689
4690       f = Ufopen(sub_arg[0], "rb");
4691       if (f == NULL)
4692         {
4693         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4694         goto EXPAND_FAILED;
4695         }
4696
4697       yield = cat_file(f, yield, &size, &ptr, sub_arg[1]);
4698       (void)fclose(f);
4699       continue;
4700       }
4701
4702     /* Handle "readsocket" to insert data from a Unix domain socket */
4703
4704     case EITEM_READSOCK:
4705       {
4706       int fd;
4707       int timeout = 5;
4708       int save_ptr = ptr;
4709       FILE *f;
4710       struct sockaddr_un sockun;         /* don't call this "sun" ! */
4711       uschar *arg;
4712       uschar *sub_arg[4];
4713
4714       if ((expand_forbid & RDO_READSOCK) != 0)
4715         {
4716         expand_string_message = US"socket insertions are not permitted";
4717         goto EXPAND_FAILED;
4718         }
4719
4720       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4721       because there may be a string for expansion on failure. */
4722
4723       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4724         {
4725         case 1: goto EXPAND_FAILED_CURLY;
4726         case 2:                             /* Won't occur: no end check */
4727         case 3: goto EXPAND_FAILED;
4728         }
4729
4730       /* Sort out timeout, if given */
4731
4732       if (sub_arg[2] != NULL)
4733         {
4734         timeout = readconf_readtime(sub_arg[2], 0, FALSE);
4735         if (timeout < 0)
4736           {
4737           expand_string_message = string_sprintf("bad time value %s",
4738             sub_arg[2]);
4739           goto EXPAND_FAILED;
4740           }
4741         }
4742       else sub_arg[3] = NULL;                     /* No eol if no timeout */
4743
4744       /* If skipping, we don't actually do anything. Otherwise, arrange to
4745       connect to either an IP or a Unix socket. */
4746
4747       if (!skipping)
4748         {
4749         /* Handle an IP (internet) domain */
4750
4751         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
4752           {
4753           int port;
4754           uschar *server_name = sub_arg[0] + 5;
4755           uschar *port_name = Ustrrchr(server_name, ':');
4756
4757           /* Sort out the port */
4758
4759           if (port_name == NULL)
4760             {
4761             expand_string_message =
4762               string_sprintf("missing port for readsocket %s", sub_arg[0]);
4763             goto EXPAND_FAILED;
4764             }
4765           *port_name++ = 0;           /* Terminate server name */
4766
4767           if (isdigit(*port_name))
4768             {
4769             uschar *end;
4770             port = Ustrtol(port_name, &end, 0);
4771             if (end != port_name + Ustrlen(port_name))
4772               {
4773               expand_string_message =
4774                 string_sprintf("invalid port number %s", port_name);
4775               goto EXPAND_FAILED;
4776               }
4777             }
4778           else
4779             {
4780             struct servent *service_info = getservbyname(CS port_name, "tcp");
4781             if (service_info == NULL)
4782               {
4783               expand_string_message = string_sprintf("unknown port \"%s\"",
4784                 port_name);
4785               goto EXPAND_FAILED;
4786               }
4787             port = ntohs(service_info->s_port);
4788             }
4789
4790           if ((fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
4791                   timeout, NULL, &expand_string_message)) < 0)
4792               goto SOCK_FAIL;
4793           }
4794
4795         /* Handle a Unix domain socket */
4796
4797         else
4798           {
4799           int rc;
4800           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
4801             {
4802             expand_string_message = string_sprintf("failed to create socket: %s",
4803               strerror(errno));
4804             goto SOCK_FAIL;
4805             }
4806
4807           sockun.sun_family = AF_UNIX;
4808           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
4809             sub_arg[0]);
4810
4811           sigalrm_seen = FALSE;
4812           alarm(timeout);
4813           rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun));
4814           alarm(0);
4815           if (sigalrm_seen)
4816             {
4817             expand_string_message = US "socket connect timed out";
4818             goto SOCK_FAIL;
4819             }
4820           if (rc < 0)
4821             {
4822             expand_string_message = string_sprintf("failed to connect to socket "
4823               "%s: %s", sub_arg[0], strerror(errno));
4824             goto SOCK_FAIL;
4825             }
4826           }
4827
4828         DEBUG(D_expand) debug_printf("connected to socket %s\n", sub_arg[0]);
4829
4830         /* Allow sequencing of test actions */
4831         if (running_in_test_harness) millisleep(100);
4832
4833         /* Write the request string, if not empty */
4834
4835         if (sub_arg[1][0] != 0)
4836           {
4837           int len = Ustrlen(sub_arg[1]);
4838           DEBUG(D_expand) debug_printf("writing \"%s\" to socket\n",
4839             sub_arg[1]);
4840           if (write(fd, sub_arg[1], len) != len)
4841             {
4842             expand_string_message = string_sprintf("request write to socket "
4843               "failed: %s", strerror(errno));
4844             goto SOCK_FAIL;
4845             }
4846           }
4847
4848         /* Shut down the sending side of the socket. This helps some servers to
4849         recognise that it is their turn to do some work. Just in case some
4850         system doesn't have this function, make it conditional. */
4851
4852         #ifdef SHUT_WR
4853         shutdown(fd, SHUT_WR);
4854         #endif
4855
4856         if (running_in_test_harness) millisleep(100);
4857
4858         /* Now we need to read from the socket, under a timeout. The function
4859         that reads a file can be used. */
4860
4861         f = fdopen(fd, "rb");
4862         sigalrm_seen = FALSE;
4863         alarm(timeout);
4864         yield = cat_file(f, yield, &size, &ptr, sub_arg[3]);
4865         alarm(0);
4866         (void)fclose(f);
4867
4868         /* After a timeout, we restore the pointer in the result, that is,
4869         make sure we add nothing from the socket. */
4870
4871         if (sigalrm_seen)
4872           {
4873           ptr = save_ptr;
4874           expand_string_message = US "socket read timed out";
4875           goto SOCK_FAIL;
4876           }
4877         }
4878
4879       /* The whole thing has worked (or we were skipping). If there is a
4880       failure string following, we need to skip it. */
4881
4882       if (*s == '{')
4883         {
4884         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
4885           goto EXPAND_FAILED;
4886         if (*s++ != '}')
4887           {
4888           expand_string_message = US"missing '}' closing failstring for readsocket";
4889           goto EXPAND_FAILED_CURLY;
4890           }
4891         while (isspace(*s)) s++;
4892         }
4893
4894     readsock_done:
4895       if (*s++ != '}')
4896         {
4897         expand_string_message = US"missing '}' closing readsocket";
4898         goto EXPAND_FAILED_CURLY;
4899         }
4900       continue;
4901
4902       /* Come here on failure to create socket, connect socket, write to the
4903       socket, or timeout on reading. If another substring follows, expand and
4904       use it. Otherwise, those conditions give expand errors. */
4905
4906       SOCK_FAIL:
4907       if (*s != '{') goto EXPAND_FAILED;
4908       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
4909       if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok)))
4910         goto EXPAND_FAILED;
4911       yield = string_cat(yield, &size, &ptr, arg);
4912       if (*s++ != '}')
4913         {
4914         expand_string_message = US"missing '}' closing failstring for readsocket";
4915         goto EXPAND_FAILED_CURLY;
4916         }
4917       while (isspace(*s)) s++;
4918       goto readsock_done;
4919       }
4920
4921     /* Handle "run" to execute a program. */
4922
4923     case EITEM_RUN:
4924       {
4925       FILE *f;
4926       uschar *arg;
4927       const uschar **argv;
4928       pid_t pid;
4929       int fd_in, fd_out;
4930       int lsize = 0, lptr = 0;
4931
4932       if ((expand_forbid & RDO_RUN) != 0)
4933         {
4934         expand_string_message = US"running a command is not permitted";
4935         goto EXPAND_FAILED;
4936         }
4937
4938       while (isspace(*s)) s++;
4939       if (*s != '{')
4940         {
4941         expand_string_message = US"missing '{' for command arg of run";
4942         goto EXPAND_FAILED_CURLY;
4943         }
4944       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4945       if (arg == NULL) goto EXPAND_FAILED;
4946       while (isspace(*s)) s++;
4947       if (*s++ != '}')
4948         {
4949         expand_string_message = US"missing '}' closing command arg of run";
4950         goto EXPAND_FAILED_CURLY;
4951         }
4952
4953       if (skipping)   /* Just pretend it worked when we're skipping */
4954         runrc = 0;
4955       else
4956         {
4957         if (!transport_set_up_command(&argv,    /* anchor for arg list */
4958             arg,                                /* raw command */
4959             FALSE,                              /* don't expand the arguments */
4960             0,                                  /* not relevant when... */
4961             NULL,                               /* no transporting address */
4962             US"${run} expansion",               /* for error messages */
4963             &expand_string_message))            /* where to put error message */
4964           goto EXPAND_FAILED;
4965
4966         /* Create the child process, making it a group leader. */
4967
4968         if ((pid = child_open(USS argv, NULL, 0077, &fd_in, &fd_out, TRUE)) < 0)
4969           {
4970           expand_string_message =
4971             string_sprintf("couldn't create child process: %s", strerror(errno));
4972           goto EXPAND_FAILED;
4973           }
4974
4975         /* Nothing is written to the standard input. */
4976
4977         (void)close(fd_in);
4978
4979         /* Read the pipe to get the command's output into $value (which is kept
4980         in lookup_value). Read during execution, so that if the output exceeds
4981         the OS pipe buffer limit, we don't block forever. Remember to not release
4982         memory just allocated for $value. */
4983
4984         resetok = FALSE;
4985         f = fdopen(fd_out, "rb");
4986         sigalrm_seen = FALSE;
4987         alarm(60);
4988         lookup_value = cat_file(f, NULL, &lsize, &lptr, NULL);
4989         alarm(0);
4990         (void)fclose(f);
4991
4992         /* Wait for the process to finish, applying the timeout, and inspect its
4993         return code for serious disasters. Simple non-zero returns are passed on.
4994         */
4995
4996         if (sigalrm_seen == TRUE || (runrc = child_close(pid, 30)) < 0)
4997           {
4998           if (sigalrm_seen == TRUE || runrc == -256)
4999             {
5000             expand_string_message = string_sprintf("command timed out");
5001             killpg(pid, SIGKILL);       /* Kill the whole process group */
5002             }
5003
5004           else if (runrc == -257)
5005             expand_string_message = string_sprintf("wait() failed: %s",
5006               strerror(errno));
5007
5008           else
5009             expand_string_message = string_sprintf("command killed by signal %d",
5010               -runrc);
5011
5012           goto EXPAND_FAILED;
5013           }
5014         }
5015
5016       /* Process the yes/no strings; $value may be useful in both cases */
5017
5018       switch(process_yesno(
5019                skipping,                     /* were previously skipping */
5020                runrc == 0,                   /* success/failure indicator */
5021                lookup_value,                 /* value to reset for string2 */
5022                &s,                           /* input pointer */
5023                &yield,                       /* output pointer */
5024                &size,                        /* output size */
5025                &ptr,                         /* output current point */
5026                US"run",                      /* condition type */
5027                &resetok))
5028         {
5029         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5030         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5031         }
5032
5033       continue;
5034       }
5035
5036     /* Handle character translation for "tr" */
5037
5038     case EITEM_TR:
5039       {
5040       int oldptr = ptr;
5041       int o2m;
5042       uschar *sub[3];
5043
5044       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
5045         {
5046         case 1: goto EXPAND_FAILED_CURLY;
5047         case 2:
5048         case 3: goto EXPAND_FAILED;
5049         }
5050
5051       yield = string_cat(yield, &size, &ptr, sub[0]);
5052       o2m = Ustrlen(sub[2]) - 1;
5053
5054       if (o2m >= 0) for (; oldptr < ptr; oldptr++)
5055         {
5056         uschar *m = Ustrrchr(sub[1], yield[oldptr]);
5057         if (m != NULL)
5058           {
5059           int o = m - sub[1];
5060           yield[oldptr] = sub[2][(o < o2m)? o : o2m];
5061           }
5062         }
5063
5064       continue;
5065       }
5066
5067     /* Handle "hash", "length", "nhash", and "substr" when they are given with
5068     expanded arguments. */
5069
5070     case EITEM_HASH:
5071     case EITEM_LENGTH:
5072     case EITEM_NHASH:
5073     case EITEM_SUBSTR:
5074       {
5075       int i;
5076       int len;
5077       uschar *ret;
5078       int val[2] = { 0, -1 };
5079       uschar *sub[3];
5080
5081       /* "length" takes only 2 arguments whereas the others take 2 or 3.
5082       Ensure that sub[2] is set in the ${length } case. */
5083
5084       sub[2] = NULL;
5085       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
5086              TRUE, name, &resetok))
5087         {
5088         case 1: goto EXPAND_FAILED_CURLY;
5089         case 2:
5090         case 3: goto EXPAND_FAILED;
5091         }
5092
5093       /* Juggle the arguments if there are only two of them: always move the
5094       string to the last position and make ${length{n}{str}} equivalent to
5095       ${substr{0}{n}{str}}. See the defaults for val[] above. */
5096
5097       if (sub[2] == NULL)
5098         {
5099         sub[2] = sub[1];
5100         sub[1] = NULL;
5101         if (item_type == EITEM_LENGTH)
5102           {
5103           sub[1] = sub[0];
5104           sub[0] = NULL;
5105           }
5106         }
5107
5108       for (i = 0; i < 2; i++)
5109         {
5110         if (sub[i] == NULL) continue;
5111         val[i] = (int)Ustrtol(sub[i], &ret, 10);
5112         if (*ret != 0 || (i != 0 && val[i] < 0))
5113           {
5114           expand_string_message = string_sprintf("\"%s\" is not a%s number "
5115             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
5116           goto EXPAND_FAILED;
5117           }
5118         }
5119
5120       ret =
5121         (item_type == EITEM_HASH)?
5122           compute_hash(sub[2], val[0], val[1], &len) :
5123         (item_type == EITEM_NHASH)?
5124           compute_nhash(sub[2], val[0], val[1], &len) :
5125           extract_substr(sub[2], val[0], val[1], &len);
5126
5127       if (ret == NULL) goto EXPAND_FAILED;
5128       yield = string_catn(yield, &size, &ptr, ret, len);
5129       continue;
5130       }
5131
5132     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
5133     This code originally contributed by Steve Haslam. It currently supports
5134     the use of MD5 and SHA-1 hashes.
5135
5136     We need some workspace that is large enough to handle all the supported
5137     hash types. Use macros to set the sizes rather than be too elaborate. */
5138
5139     #define MAX_HASHLEN      20
5140     #define MAX_HASHBLOCKLEN 64
5141
5142     case EITEM_HMAC:
5143       {
5144       uschar *sub[3];
5145       md5 md5_base;
5146       hctx sha1_ctx;
5147       void *use_base;
5148       int type, i;
5149       int hashlen;      /* Number of octets for the hash algorithm's output */
5150       int hashblocklen; /* Number of octets the hash algorithm processes */
5151       uschar *keyptr, *p;
5152       unsigned int keylen;
5153
5154       uschar keyhash[MAX_HASHLEN];
5155       uschar innerhash[MAX_HASHLEN];
5156       uschar finalhash[MAX_HASHLEN];
5157       uschar finalhash_hex[2*MAX_HASHLEN];
5158       uschar innerkey[MAX_HASHBLOCKLEN];
5159       uschar outerkey[MAX_HASHBLOCKLEN];
5160
5161       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
5162         {
5163         case 1: goto EXPAND_FAILED_CURLY;
5164         case 2:
5165         case 3: goto EXPAND_FAILED;
5166         }
5167
5168       if (Ustrcmp(sub[0], "md5") == 0)
5169         {
5170         type = HMAC_MD5;
5171         use_base = &md5_base;
5172         hashlen = 16;
5173         hashblocklen = 64;
5174         }
5175       else if (Ustrcmp(sub[0], "sha1") == 0)
5176         {
5177         type = HMAC_SHA1;
5178         use_base = &sha1_ctx;
5179         hashlen = 20;
5180         hashblocklen = 64;
5181         }
5182       else
5183         {
5184         expand_string_message =
5185           string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
5186         goto EXPAND_FAILED;
5187         }
5188
5189       keyptr = sub[1];
5190       keylen = Ustrlen(keyptr);
5191
5192       /* If the key is longer than the hash block length, then hash the key
5193       first */
5194
5195       if (keylen > hashblocklen)
5196         {
5197         chash_start(type, use_base);
5198         chash_end(type, use_base, keyptr, keylen, keyhash);
5199         keyptr = keyhash;
5200         keylen = hashlen;
5201         }
5202
5203       /* Now make the inner and outer key values */
5204
5205       memset(innerkey, 0x36, hashblocklen);
5206       memset(outerkey, 0x5c, hashblocklen);
5207
5208       for (i = 0; i < keylen; i++)
5209         {
5210         innerkey[i] ^= keyptr[i];
5211         outerkey[i] ^= keyptr[i];
5212         }
5213
5214       /* Now do the hashes */
5215
5216       chash_start(type, use_base);
5217       chash_mid(type, use_base, innerkey);
5218       chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5219
5220       chash_start(type, use_base);
5221       chash_mid(type, use_base, outerkey);
5222       chash_end(type, use_base, innerhash, hashlen, finalhash);
5223
5224       /* Encode the final hash as a hex string */
5225
5226       p = finalhash_hex;
5227       for (i = 0; i < hashlen; i++)
5228         {
5229         *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5230         *p++ = hex_digits[finalhash[i] & 0x0f];
5231         }
5232
5233       DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%.*s)=%.*s\n", sub[0],
5234         (int)keylen, keyptr, Ustrlen(sub[2]), sub[2], hashlen*2, finalhash_hex);
5235
5236       yield = string_catn(yield, &size, &ptr, finalhash_hex, hashlen*2);
5237       }
5238
5239     continue;
5240
5241     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5242     We have to save the numerical variables and restore them afterwards. */
5243
5244     case EITEM_SG:
5245       {
5246       const pcre *re;
5247       int moffset, moffsetextra, slen;
5248       int roffset;
5249       int emptyopt;
5250       const uschar *rerror;
5251       uschar *subject;
5252       uschar *sub[3];
5253       int save_expand_nmax =
5254         save_expand_strings(save_expand_nstring, save_expand_nlength);
5255
5256       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5257         {
5258         case 1: goto EXPAND_FAILED_CURLY;
5259         case 2:
5260         case 3: goto EXPAND_FAILED;
5261         }
5262
5263       /* Compile the regular expression */
5264
5265       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5266         NULL);
5267
5268       if (re == NULL)
5269         {
5270         expand_string_message = string_sprintf("regular expression error in "
5271           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5272         goto EXPAND_FAILED;
5273         }
5274
5275       /* Now run a loop to do the substitutions as often as necessary. It ends
5276       when there are no more matches. Take care over matches of the null string;
5277       do the same thing as Perl does. */
5278
5279       subject = sub[0];
5280       slen = Ustrlen(sub[0]);
5281       moffset = moffsetextra = 0;
5282       emptyopt = 0;
5283
5284       for (;;)
5285         {
5286         int ovector[3*(EXPAND_MAXN+1)];
5287         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5288           PCRE_EOPT | emptyopt, ovector, nelem(ovector));
5289         int nn;
5290         uschar *insert;
5291
5292         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5293         is not necessarily the end. We want to repeat the match from one
5294         character further along, but leaving the basic offset the same (for
5295         copying below). We can't be at the end of the string - that was checked
5296         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5297         finished; copy the remaining string and end the loop. */
5298
5299         if (n < 0)
5300           {
5301           if (emptyopt != 0)
5302             {
5303             moffsetextra = 1;
5304             emptyopt = 0;
5305             continue;
5306             }
5307           yield = string_catn(yield, &size, &ptr, subject+moffset, slen-moffset);
5308           break;
5309           }
5310
5311         /* Match - set up for expanding the replacement. */
5312
5313         if (n == 0) n = EXPAND_MAXN + 1;
5314         expand_nmax = 0;
5315         for (nn = 0; nn < n*2; nn += 2)
5316           {
5317           expand_nstring[expand_nmax] = subject + ovector[nn];
5318           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5319           }
5320         expand_nmax--;
5321
5322         /* Copy the characters before the match, plus the expanded insertion. */
5323
5324         yield = string_catn(yield, &size, &ptr, subject + moffset,
5325           ovector[0] - moffset);
5326         insert = expand_string(sub[2]);
5327         if (insert == NULL) goto EXPAND_FAILED;
5328         yield = string_cat(yield, &size, &ptr, insert);
5329
5330         moffset = ovector[1];
5331         moffsetextra = 0;
5332         emptyopt = 0;
5333
5334         /* If we have matched an empty string, first check to see if we are at
5335         the end of the subject. If so, the loop is over. Otherwise, mimic
5336         what Perl's /g options does. This turns out to be rather cunning. First
5337         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5338         string at the same point. If this fails (picked up above) we advance to
5339         the next character. */
5340
5341         if (ovector[0] == ovector[1])
5342           {
5343           if (ovector[0] == slen) break;
5344           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5345           }
5346         }
5347
5348       /* All done - restore numerical variables. */
5349
5350       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5351         save_expand_nlength);
5352       continue;
5353       }
5354
5355     /* Handle keyed and numbered substring extraction. If the first argument
5356     consists entirely of digits, then a numerical extraction is assumed. */
5357
5358     case EITEM_EXTRACT:
5359       {
5360       int i;
5361       int j;
5362       int field_number = 1;
5363       BOOL field_number_set = FALSE;
5364       uschar *save_lookup_value = lookup_value;
5365       uschar *sub[3];
5366       int save_expand_nmax =
5367         save_expand_strings(save_expand_nstring, save_expand_nlength);
5368
5369       /* While skipping we cannot rely on the data for expansions being
5370       available (eg. $item) hence cannot decide on numeric vs. keyed.
5371       Read a maximum of 5 arguments (inclding the yes/no) */
5372
5373       if (skipping)
5374         {
5375         while (isspace(*s)) s++;
5376         for (j = 5; j > 0 && *s == '{'; j--)
5377           {
5378           if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))
5379             goto EXPAND_FAILED;                                 /*{*/
5380           if (*s++ != '}')
5381             {
5382             expand_string_message = US"missing '{' for arg of extract";
5383             goto EXPAND_FAILED_CURLY;
5384             }
5385           while (isspace(*s)) s++;
5386           }
5387         if (  Ustrncmp(s, "fail", 4) == 0
5388            && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4])
5389            )
5390           {
5391           s += 4;
5392           while (isspace(*s)) s++;
5393           }
5394         if (*s != '}')
5395           {
5396           expand_string_message = US"missing '}' closing extract";
5397           goto EXPAND_FAILED_CURLY;
5398           }
5399         }
5400
5401       else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */
5402         {
5403         while (isspace(*s)) s++;
5404         if (*s == '{')                                          /*}*/
5405           {
5406           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5407           if (sub[i] == NULL) goto EXPAND_FAILED;               /*{*/
5408           if (*s++ != '}')
5409             {
5410             expand_string_message = string_sprintf(
5411               "missing '}' closing arg %d of extract", i+1);
5412             goto EXPAND_FAILED_CURLY;
5413             }
5414
5415           /* After removal of leading and trailing white space, the first
5416           argument must not be empty; if it consists entirely of digits
5417           (optionally preceded by a minus sign), this is a numerical
5418           extraction, and we expect 3 arguments. */
5419
5420           if (i == 0)
5421             {
5422             int len;
5423             int x = 0;
5424             uschar *p = sub[0];
5425
5426             while (isspace(*p)) p++;
5427             sub[0] = p;
5428
5429             len = Ustrlen(p);
5430             while (len > 0 && isspace(p[len-1])) len--;
5431             p[len] = 0;
5432
5433             if (*p == 0)
5434               {
5435               expand_string_message = US"first argument of \"extract\" must "
5436                 "not be empty";
5437               goto EXPAND_FAILED;
5438               }
5439
5440             if (*p == '-')
5441               {
5442               field_number = -1;
5443               p++;
5444               }
5445             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5446             if (*p == 0)
5447               {
5448               field_number *= x;
5449               j = 3;               /* Need 3 args */
5450               field_number_set = TRUE;
5451               }
5452             }
5453           }
5454         else
5455           {
5456           expand_string_message = string_sprintf(
5457             "missing '{' for arg %d of extract", i+1);
5458           goto EXPAND_FAILED_CURLY;
5459           }
5460         }
5461
5462       /* Extract either the numbered or the keyed substring into $value. If
5463       skipping, just pretend the extraction failed. */
5464
5465       lookup_value = skipping? NULL : field_number_set?
5466         expand_gettokened(field_number, sub[1], sub[2]) :
5467         expand_getkeyed(sub[0], sub[1]);
5468
5469       /* If no string follows, $value gets substituted; otherwise there can
5470       be yes/no strings, as for lookup or if. */
5471
5472       switch(process_yesno(
5473                skipping,                     /* were previously skipping */
5474                lookup_value != NULL,         /* success/failure indicator */
5475                save_lookup_value,            /* value to reset for string2 */
5476                &s,                           /* input pointer */
5477                &yield,                       /* output pointer */
5478                &size,                        /* output size */
5479                &ptr,                         /* output current point */
5480                US"extract",                  /* condition type */
5481                &resetok))
5482         {
5483         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5484         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5485         }
5486
5487       /* All done - restore numerical variables. */
5488
5489       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5490         save_expand_nlength);
5491
5492       continue;
5493       }
5494
5495     /* return the Nth item from a list */
5496
5497     case EITEM_LISTEXTRACT:
5498       {
5499       int i;
5500       int field_number = 1;
5501       uschar *save_lookup_value = lookup_value;
5502       uschar *sub[2];
5503       int save_expand_nmax =
5504         save_expand_strings(save_expand_nstring, save_expand_nlength);
5505
5506       /* Read the field & list arguments */
5507
5508       for (i = 0; i < 2; i++)
5509         {
5510         while (isspace(*s)) s++;
5511         if (*s != '{')                                  /*}*/
5512           {
5513           expand_string_message = string_sprintf(
5514             "missing '{' for arg %d of listextract", i+1);
5515           goto EXPAND_FAILED_CURLY;
5516           }
5517
5518         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5519         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5520         if (*s++ != '}')
5521           {
5522           expand_string_message = string_sprintf(
5523             "missing '}' closing arg %d of listextract", i+1);
5524           goto EXPAND_FAILED_CURLY;
5525           }
5526
5527         /* After removal of leading and trailing white space, the first
5528         argument must be numeric and nonempty. */
5529
5530         if (i == 0)
5531           {
5532           int len;
5533           int x = 0;
5534           uschar *p = sub[0];
5535
5536           while (isspace(*p)) p++;
5537           sub[0] = p;
5538
5539           len = Ustrlen(p);
5540           while (len > 0 && isspace(p[len-1])) len--;
5541           p[len] = 0;
5542
5543           if (!*p && !skipping)
5544             {
5545             expand_string_message = US"first argument of \"listextract\" must "
5546               "not be empty";
5547             goto EXPAND_FAILED;
5548             }
5549
5550           if (*p == '-')
5551             {
5552             field_number = -1;
5553             p++;
5554             }
5555           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5556           if (*p)
5557             {
5558             expand_string_message = US"first argument of \"listextract\" must "
5559               "be numeric";
5560             goto EXPAND_FAILED;
5561             }
5562           field_number *= x;
5563           }
5564         }
5565
5566       /* Extract the numbered element into $value. If
5567       skipping, just pretend the extraction failed. */
5568
5569       lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]);
5570
5571       /* If no string follows, $value gets substituted; otherwise there can
5572       be yes/no strings, as for lookup or if. */
5573
5574       switch(process_yesno(
5575                skipping,                     /* were previously skipping */
5576                lookup_value != NULL,         /* success/failure indicator */
5577                save_lookup_value,            /* value to reset for string2 */
5578                &s,                           /* input pointer */
5579                &yield,                       /* output pointer */
5580                &size,                        /* output size */
5581                &ptr,                         /* output current point */
5582                US"listextract",              /* condition type */
5583                &resetok))
5584         {
5585         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5586         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5587         }
5588
5589       /* All done - restore numerical variables. */
5590
5591       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5592         save_expand_nlength);
5593
5594       continue;
5595       }
5596
5597 #ifdef SUPPORT_TLS
5598     case EITEM_CERTEXTRACT:
5599       {
5600       uschar *save_lookup_value = lookup_value;
5601       uschar *sub[2];
5602       int save_expand_nmax =
5603         save_expand_strings(save_expand_nstring, save_expand_nlength);
5604
5605       /* Read the field argument */
5606       while (isspace(*s)) s++;
5607       if (*s != '{')                                    /*}*/
5608         {
5609         expand_string_message = US"missing '{' for field arg of certextract";
5610         goto EXPAND_FAILED_CURLY;
5611         }
5612       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5613       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
5614       if (*s++ != '}')
5615         {
5616         expand_string_message = US"missing '}' closing field arg of certextract";
5617         goto EXPAND_FAILED_CURLY;
5618         }
5619       /* strip spaces fore & aft */
5620       {
5621       int len;
5622       uschar *p = sub[0];
5623
5624       while (isspace(*p)) p++;
5625       sub[0] = p;
5626
5627       len = Ustrlen(p);
5628       while (len > 0 && isspace(p[len-1])) len--;
5629       p[len] = 0;
5630       }
5631
5632       /* inspect the cert argument */
5633       while (isspace(*s)) s++;
5634       if (*s != '{')                                    /*}*/
5635         {
5636         expand_string_message = US"missing '{' for cert variable arg of certextract";
5637         goto EXPAND_FAILED_CURLY;
5638         }
5639       if (*++s != '$')
5640         {
5641         expand_string_message = US"second argument of \"certextract\" must "
5642           "be a certificate variable";
5643         goto EXPAND_FAILED;
5644         }
5645       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
5646       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
5647       if (*s++ != '}')
5648         {
5649         expand_string_message = US"missing '}' closing cert variable arg of certextract";
5650         goto EXPAND_FAILED_CURLY;
5651         }
5652
5653       if (skipping)
5654         lookup_value = NULL;
5655       else
5656         {
5657         lookup_value = expand_getcertele(sub[0], sub[1]);
5658         if (*expand_string_message) goto EXPAND_FAILED;
5659         }
5660       switch(process_yesno(
5661                skipping,                     /* were previously skipping */
5662                lookup_value != NULL,         /* success/failure indicator */
5663                save_lookup_value,            /* value to reset for string2 */
5664                &s,                           /* input pointer */
5665                &yield,                       /* output pointer */
5666                &size,                        /* output size */
5667                &ptr,                         /* output current point */
5668                US"certextract",              /* condition type */
5669                &resetok))
5670         {
5671         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5672         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5673         }
5674
5675       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5676         save_expand_nlength);
5677       continue;
5678       }
5679 #endif  /*SUPPORT_TLS*/
5680
5681     /* Handle list operations */
5682
5683     case EITEM_FILTER:
5684     case EITEM_MAP:
5685     case EITEM_REDUCE:
5686       {
5687       int sep = 0;
5688       int save_ptr = ptr;
5689       uschar outsep[2] = { '\0', '\0' };
5690       const uschar *list, *expr, *temp;
5691       uschar *save_iterate_item = iterate_item;
5692       uschar *save_lookup_value = lookup_value;
5693
5694       while (isspace(*s)) s++;
5695       if (*s++ != '{')
5696         {
5697         expand_string_message =
5698           string_sprintf("missing '{' for first arg of %s", name);
5699         goto EXPAND_FAILED_CURLY;
5700         }
5701
5702       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5703       if (list == NULL) goto EXPAND_FAILED;
5704       if (*s++ != '}')
5705         {
5706         expand_string_message =
5707           string_sprintf("missing '}' closing first arg of %s", name);
5708         goto EXPAND_FAILED_CURLY;
5709         }
5710
5711       if (item_type == EITEM_REDUCE)
5712         {
5713         uschar * t;
5714         while (isspace(*s)) s++;
5715         if (*s++ != '{')
5716           {
5717           expand_string_message = US"missing '{' for second arg of reduce";
5718           goto EXPAND_FAILED_CURLY;
5719           }
5720         t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5721         if (!t) goto EXPAND_FAILED;
5722         lookup_value = t;
5723         if (*s++ != '}')
5724           {
5725           expand_string_message = US"missing '}' closing second arg of reduce";
5726           goto EXPAND_FAILED_CURLY;
5727           }
5728         }
5729
5730       while (isspace(*s)) s++;
5731       if (*s++ != '{')
5732         {
5733         expand_string_message =
5734           string_sprintf("missing '{' for last arg of %s", name);
5735         goto EXPAND_FAILED_CURLY;
5736         }
5737
5738       expr = s;
5739
5740       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
5741       if scanning a "false" part). This allows us to find the end of the
5742       condition, because if the list is empty, we won't actually evaluate the
5743       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
5744       the normal internal expansion function. */
5745
5746       if (item_type == EITEM_FILTER)
5747         {
5748         temp = eval_condition(expr, &resetok, NULL);
5749         if (temp != NULL) s = temp;
5750         }
5751       else
5752         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5753
5754       if (temp == NULL)
5755         {
5756         expand_string_message = string_sprintf("%s inside \"%s\" item",
5757           expand_string_message, name);
5758         goto EXPAND_FAILED;
5759         }
5760
5761       while (isspace(*s)) s++;
5762       if (*s++ != '}')
5763         {                                               /*{*/
5764         expand_string_message = string_sprintf("missing } at end of condition "
5765           "or expression inside \"%s\"", name);
5766         goto EXPAND_FAILED;
5767         }
5768
5769       while (isspace(*s)) s++;                          /*{*/
5770       if (*s++ != '}')
5771         {                                               /*{*/
5772         expand_string_message = string_sprintf("missing } at end of \"%s\"",
5773           name);
5774         goto EXPAND_FAILED;
5775         }
5776
5777       /* If we are skipping, we can now just move on to the next item. When
5778       processing for real, we perform the iteration. */
5779
5780       if (skipping) continue;
5781       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
5782         {
5783         *outsep = (uschar)sep;      /* Separator as a string */
5784
5785         DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
5786
5787         if (item_type == EITEM_FILTER)
5788           {
5789           BOOL condresult;
5790           if (eval_condition(expr, &resetok, &condresult) == NULL)
5791             {
5792             iterate_item = save_iterate_item;
5793             lookup_value = save_lookup_value;
5794             expand_string_message = string_sprintf("%s inside \"%s\" condition",
5795               expand_string_message, name);
5796             goto EXPAND_FAILED;
5797             }
5798           DEBUG(D_expand) debug_printf("%s: condition is %s\n", name,
5799             condresult? "true":"false");
5800           if (condresult)
5801             temp = iterate_item;    /* TRUE => include this item */
5802           else
5803             continue;               /* FALSE => skip this item */
5804           }
5805
5806         /* EITEM_MAP and EITEM_REDUCE */
5807
5808         else
5809           {
5810           uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
5811           temp = t;
5812           if (temp == NULL)
5813             {
5814             iterate_item = save_iterate_item;
5815             expand_string_message = string_sprintf("%s inside \"%s\" item",
5816               expand_string_message, name);
5817             goto EXPAND_FAILED;
5818             }
5819           if (item_type == EITEM_REDUCE)
5820             {
5821             lookup_value = t;         /* Update the value of $value */
5822             continue;                 /* and continue the iteration */
5823             }
5824           }
5825
5826         /* We reach here for FILTER if the condition is true, always for MAP,
5827         and never for REDUCE. The value in "temp" is to be added to the output
5828         list that is being created, ensuring that any occurrences of the
5829         separator character are doubled. Unless we are dealing with the first
5830         item of the output list, add in a space if the new item begins with the
5831         separator character, or is an empty string. */
5832
5833         if (ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
5834           yield = string_catn(yield, &size, &ptr, US" ", 1);
5835
5836         /* Add the string in "temp" to the output list that we are building,
5837         This is done in chunks by searching for the separator character. */
5838
5839         for (;;)
5840           {
5841           size_t seglen = Ustrcspn(temp, outsep);
5842
5843           yield = string_catn(yield, &size, &ptr, temp, seglen + 1);
5844
5845           /* If we got to the end of the string we output one character
5846           too many; backup and end the loop. Otherwise arrange to double the
5847           separator. */
5848
5849           if (temp[seglen] == '\0') { ptr--; break; }
5850           yield = string_catn(yield, &size, &ptr, outsep, 1);
5851           temp += seglen + 1;
5852           }
5853
5854         /* Output a separator after the string: we will remove the redundant
5855         final one at the end. */
5856
5857         yield = string_catn(yield, &size, &ptr, outsep, 1);
5858         }   /* End of iteration over the list loop */
5859
5860       /* REDUCE has generated no output above: output the final value of
5861       $value. */
5862
5863       if (item_type == EITEM_REDUCE)
5864         {
5865         yield = string_cat(yield, &size, &ptr, lookup_value);
5866         lookup_value = save_lookup_value;  /* Restore $value */
5867         }
5868
5869       /* FILTER and MAP generate lists: if they have generated anything, remove
5870       the redundant final separator. Even though an empty item at the end of a
5871       list does not count, this is tidier. */
5872
5873       else if (ptr != save_ptr) ptr--;
5874
5875       /* Restore preserved $item */
5876
5877       iterate_item = save_iterate_item;
5878       continue;
5879       }
5880
5881     case EITEM_SORT:
5882       {
5883       int sep = 0;
5884       const uschar *srclist, *cmp, *xtract;
5885       uschar *srcitem;
5886       const uschar *dstlist = NULL, *dstkeylist = NULL;
5887       uschar * tmp;
5888       uschar *save_iterate_item = iterate_item;
5889
5890       while (isspace(*s)) s++;
5891       if (*s++ != '{')
5892         {
5893         expand_string_message = US"missing '{' for list arg of sort";
5894         goto EXPAND_FAILED_CURLY;
5895         }
5896
5897       srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5898       if (!srclist) goto EXPAND_FAILED;
5899       if (*s++ != '}')
5900         {
5901         expand_string_message = US"missing '}' closing list arg of sort";
5902         goto EXPAND_FAILED_CURLY;
5903         }
5904
5905       while (isspace(*s)) s++;
5906       if (*s++ != '{')
5907         {
5908         expand_string_message = US"missing '{' for comparator arg of sort";
5909         goto EXPAND_FAILED_CURLY;
5910         }
5911
5912       cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok);
5913       if (!cmp) goto EXPAND_FAILED;
5914       if (*s++ != '}')
5915         {
5916         expand_string_message = US"missing '}' closing comparator arg of sort";
5917         goto EXPAND_FAILED_CURLY;
5918         }
5919
5920       while (isspace(*s)) s++;
5921       if (*s++ != '{')
5922         {
5923         expand_string_message = US"missing '{' for extractor arg of sort";
5924         goto EXPAND_FAILED_CURLY;
5925         }
5926
5927       xtract = s;
5928       tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5929       if (!tmp) goto EXPAND_FAILED;
5930       xtract = string_copyn(xtract, s - xtract);
5931
5932       if (*s++ != '}')
5933         {
5934         expand_string_message = US"missing '}' closing extractor arg of sort";
5935         goto EXPAND_FAILED_CURLY;
5936         }
5937                                                         /*{*/
5938       if (*s++ != '}')
5939         {                                               /*{*/
5940         expand_string_message = US"missing } at end of \"sort\"";
5941         goto EXPAND_FAILED;
5942         }
5943
5944       if (skipping) continue;
5945
5946       while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0)))
5947         {
5948         uschar * dstitem;
5949         uschar * newlist = NULL;
5950         uschar * newkeylist = NULL;
5951         uschar * srcfield;
5952
5953         DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, srcitem);
5954
5955         /* extract field for comparisons */
5956         iterate_item = srcitem;
5957         if (  !(srcfield = expand_string_internal(xtract, FALSE, NULL, FALSE,
5958                                           TRUE, &resetok))
5959            || !*srcfield)
5960           {
5961           expand_string_message = string_sprintf(
5962               "field-extract in sort: \"%s\"", xtract);
5963           goto EXPAND_FAILED;
5964           }
5965
5966         /* Insertion sort */
5967
5968         /* copy output list until new-item < list-item */
5969         while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
5970           {
5971           uschar * dstfield;
5972           uschar * expr;
5973           BOOL before;
5974
5975           /* field for comparison */
5976           if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
5977             goto sort_mismatch;
5978
5979           /* build and run condition string */
5980           expr = string_sprintf("%s{%s}{%s}", cmp, srcfield, dstfield);
5981
5982           DEBUG(D_expand) debug_printf("%s: cond = \"%s\"\n", name, expr);
5983           if (!eval_condition(expr, &resetok, &before))
5984             {
5985             expand_string_message = string_sprintf("comparison in sort: %s",
5986                 expr);
5987             goto EXPAND_FAILED;
5988             }
5989
5990           if (before)
5991             {
5992             /* New-item sorts before this dst-item.  Append new-item,
5993             then dst-item, then remainder of dst list. */
5994
5995             newlist = string_append_listele(newlist, sep, srcitem);
5996             newkeylist = string_append_listele(newkeylist, sep, srcfield);
5997             srcitem = NULL;
5998
5999             newlist = string_append_listele(newlist, sep, dstitem);
6000             newkeylist = string_append_listele(newkeylist, sep, dstfield);
6001
6002             while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6003               {
6004               if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6005                 goto sort_mismatch;
6006               newlist = string_append_listele(newlist, sep, dstitem);
6007               newkeylist = string_append_listele(newkeylist, sep, dstfield);
6008               }
6009
6010             break;
6011             }
6012
6013           newlist = string_append_listele(newlist, sep, dstitem);
6014           newkeylist = string_append_listele(newkeylist, sep, dstfield);
6015           }
6016
6017         /* If we ran out of dstlist without consuming srcitem, append it */
6018         if (srcitem)
6019           {
6020           newlist = string_append_listele(newlist, sep, srcitem);
6021           newkeylist = string_append_listele(newkeylist, sep, srcfield);
6022           }
6023
6024         dstlist = newlist;
6025         dstkeylist = newkeylist;
6026
6027         DEBUG(D_expand) debug_printf("%s: dstlist = \"%s\"\n", name, dstlist);
6028         DEBUG(D_expand) debug_printf("%s: dstkeylist = \"%s\"\n", name, dstkeylist);
6029         }
6030
6031       if (dstlist)
6032         yield = string_cat(yield, &size, &ptr, dstlist);
6033
6034       /* Restore preserved $item */
6035       iterate_item = save_iterate_item;
6036       continue;
6037
6038       sort_mismatch:
6039         expand_string_message = US"Internal error in sort (list mismatch)";
6040         goto EXPAND_FAILED;
6041       }
6042
6043
6044     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
6045     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
6046     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
6047     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
6048
6049     #define EXPAND_DLFUNC_MAX_ARGS 8
6050
6051     case EITEM_DLFUNC:
6052 #ifndef EXPAND_DLFUNC
6053       expand_string_message = US"\"${dlfunc\" encountered, but this facility "  /*}*/
6054         "is not included in this binary";
6055       goto EXPAND_FAILED;
6056
6057 #else   /* EXPAND_DLFUNC */
6058       {
6059       tree_node *t;
6060       exim_dlfunc_t *func;
6061       uschar *result;
6062       int status, argc;
6063       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
6064
6065       if ((expand_forbid & RDO_DLFUNC) != 0)
6066         {
6067         expand_string_message =
6068           US"dynamically-loaded functions are not permitted";
6069         goto EXPAND_FAILED;
6070         }
6071
6072       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
6073            TRUE, US"dlfunc", &resetok))
6074         {
6075         case 1: goto EXPAND_FAILED_CURLY;
6076         case 2:
6077         case 3: goto EXPAND_FAILED;
6078         }
6079
6080       /* If skipping, we don't actually do anything */
6081
6082       if (skipping) continue;
6083
6084       /* Look up the dynamically loaded object handle in the tree. If it isn't
6085       found, dlopen() the file and put the handle in the tree for next time. */
6086
6087       t = tree_search(dlobj_anchor, argv[0]);
6088       if (t == NULL)
6089         {
6090         void *handle = dlopen(CS argv[0], RTLD_LAZY);
6091         if (handle == NULL)
6092           {
6093           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
6094             argv[0], dlerror());
6095           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6096           goto EXPAND_FAILED;
6097           }
6098         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
6099         Ustrcpy(t->name, argv[0]);
6100         t->data.ptr = handle;
6101         (void)tree_insertnode(&dlobj_anchor, t);
6102         }
6103
6104       /* Having obtained the dynamically loaded object handle, look up the
6105       function pointer. */
6106
6107       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
6108       if (func == NULL)
6109         {
6110         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
6111           "%s", argv[1], argv[0], dlerror());
6112         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6113         goto EXPAND_FAILED;
6114         }
6115
6116       /* Call the function and work out what to do with the result. If it
6117       returns OK, we have a replacement string; if it returns DEFER then
6118       expansion has failed in a non-forced manner; if it returns FAIL then
6119       failure was forced; if it returns ERROR or any other value there's a
6120       problem, so panic slightly. In any case, assume that the function has
6121       side-effects on the store that must be preserved. */
6122
6123       resetok = FALSE;
6124       result = NULL;
6125       for (argc = 0; argv[argc] != NULL; argc++);
6126       status = func(&result, argc - 2, &argv[2]);
6127       if(status == OK)
6128         {
6129         if (result == NULL) result = US"";
6130         yield = string_cat(yield, &size, &ptr, result);
6131         continue;
6132         }
6133       else
6134         {
6135         expand_string_message = result == NULL ? US"(no message)" : result;
6136         if(status == FAIL_FORCED) expand_string_forcedfail = TRUE;
6137           else if(status != FAIL)
6138             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
6139               argv[0], argv[1], status, expand_string_message);
6140         goto EXPAND_FAILED;
6141         }
6142       }
6143 #endif /* EXPAND_DLFUNC */
6144
6145     case EITEM_ENV:     /* ${env {name} {val_if_found} {val_if_unfound}} */
6146       {
6147       uschar * key;
6148       uschar *save_lookup_value = lookup_value;
6149
6150       while (isspace(*s)) s++;
6151       if (*s != '{')                                    /*}*/
6152         goto EXPAND_FAILED;
6153
6154       key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6155       if (!key) goto EXPAND_FAILED;                     /*{*/
6156       if (*s++ != '}')
6157         {
6158         expand_string_message = US"missing '{' for name arg of env";
6159         goto EXPAND_FAILED_CURLY;
6160         }
6161
6162       lookup_value = US getenv(CS key);
6163
6164       switch(process_yesno(
6165                skipping,                     /* were previously skipping */
6166                lookup_value != NULL,         /* success/failure indicator */
6167                save_lookup_value,            /* value to reset for string2 */
6168                &s,                           /* input pointer */
6169                &yield,                       /* output pointer */
6170                &size,                        /* output size */
6171                &ptr,                         /* output current point */
6172                US"env",                      /* condition type */
6173                &resetok))
6174         {
6175         case 1: goto EXPAND_FAILED;          /* when all is well, the */
6176         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
6177         }
6178       continue;
6179       }
6180     }   /* EITEM_* switch */
6181
6182   /* Control reaches here if the name is not recognized as one of the more
6183   complicated expansion items. Check for the "operator" syntax (name terminated
6184   by a colon). Some of the operators have arguments, separated by _ from the
6185   name. */
6186
6187   if (*s == ':')
6188     {
6189     int c;
6190     uschar *arg = NULL;
6191     uschar *sub;
6192     var_entry *vp = NULL;
6193
6194     /* Owing to an historical mis-design, an underscore may be part of the
6195     operator name, or it may introduce arguments.  We therefore first scan the
6196     table of names that contain underscores. If there is no match, we cut off
6197     the arguments and then scan the main table. */
6198
6199     if ((c = chop_match(name, op_table_underscore,
6200                         nelem(op_table_underscore))) < 0)
6201       {
6202       arg = Ustrchr(name, '_');
6203       if (arg != NULL) *arg = 0;
6204       c = chop_match(name, op_table_main, nelem(op_table_main));
6205       if (c >= 0) c += nelem(op_table_underscore);
6206       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
6207       }
6208
6209     /* Deal specially with operators that might take a certificate variable
6210     as we do not want to do the usual expansion. For most, expand the string.*/
6211     switch(c)
6212       {
6213 #ifdef SUPPORT_TLS
6214       case EOP_MD5:
6215       case EOP_SHA1:
6216       case EOP_SHA256:
6217       case EOP_BASE64:
6218         if (s[1] == '$')
6219           {
6220           const uschar * s1 = s;
6221           sub = expand_string_internal(s+2, TRUE, &s1, skipping,
6222                   FALSE, &resetok);
6223           if (!sub)       goto EXPAND_FAILED;           /*{*/
6224           if (*s1 != '}')
6225             {
6226             expand_string_message =
6227               string_sprintf("missing '}' closing cert arg of %s", name);
6228             goto EXPAND_FAILED_CURLY;
6229             }
6230           if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
6231             {
6232             s = s1+1;
6233             break;
6234             }
6235           vp = NULL;
6236           }
6237         /*FALLTHROUGH*/
6238 #endif
6239       default:
6240         sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6241         if (!sub) goto EXPAND_FAILED;
6242         s++;
6243         break;
6244       }
6245
6246     /* If we are skipping, we don't need to perform the operation at all.
6247     This matters for operations like "mask", because the data may not be
6248     in the correct format when skipping. For example, the expression may test
6249     for the existence of $sender_host_address before trying to mask it. For
6250     other operations, doing them may not fail, but it is a waste of time. */
6251
6252     if (skipping && c >= 0) continue;
6253
6254     /* Otherwise, switch on the operator type */
6255
6256     switch(c)
6257       {
6258       case EOP_BASE62:
6259         {
6260         uschar *t;
6261         unsigned long int n = Ustrtoul(sub, &t, 10);
6262         if (*t != 0)
6263           {
6264           expand_string_message = string_sprintf("argument for base62 "
6265             "operator is \"%s\", which is not a decimal number", sub);
6266           goto EXPAND_FAILED;
6267           }
6268         t = string_base62(n);
6269         yield = string_cat(yield, &size, &ptr, t);
6270         continue;
6271         }
6272
6273       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
6274
6275       case EOP_BASE62D:
6276         {
6277         uschar buf[16];
6278         uschar *tt = sub;
6279         unsigned long int n = 0;
6280         while (*tt != 0)
6281           {
6282           uschar *t = Ustrchr(base62_chars, *tt++);
6283           if (t == NULL)
6284             {
6285             expand_string_message = string_sprintf("argument for base62d "
6286               "operator is \"%s\", which is not a base %d number", sub,
6287               BASE_62);
6288             goto EXPAND_FAILED;
6289             }
6290           n = n * BASE_62 + (t - base62_chars);
6291           }
6292         (void)sprintf(CS buf, "%ld", n);
6293         yield = string_cat(yield, &size, &ptr, buf);
6294         continue;
6295         }
6296
6297       case EOP_EXPAND:
6298         {
6299         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
6300         if (expanded == NULL)
6301           {
6302           expand_string_message =
6303             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
6304               expand_string_message);
6305           goto EXPAND_FAILED;
6306           }
6307         yield = string_cat(yield, &size, &ptr, expanded);
6308         continue;
6309         }
6310
6311       case EOP_LC:
6312         {
6313         int count = 0;
6314         uschar *t = sub - 1;
6315         while (*(++t) != 0) { *t = tolower(*t); count++; }
6316         yield = string_catn(yield, &size, &ptr, sub, count);
6317         continue;
6318         }
6319
6320       case EOP_UC:
6321         {
6322         int count = 0;
6323         uschar *t = sub - 1;
6324         while (*(++t) != 0) { *t = toupper(*t); count++; }
6325         yield = string_catn(yield, &size, &ptr, sub, count);
6326         continue;
6327         }
6328
6329       case EOP_MD5:
6330 #ifdef SUPPORT_TLS
6331         if (vp && *(void **)vp->value)
6332           {
6333           uschar * cp = tls_cert_fprt_md5(*(void **)vp->value);
6334           yield = string_cat(yield, &size, &ptr, cp);
6335           }
6336         else
6337 #endif
6338           {
6339           md5 base;
6340           uschar digest[16];
6341           int j;
6342           char st[33];
6343           md5_start(&base);
6344           md5_end(&base, sub, Ustrlen(sub), digest);
6345           for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]);
6346           yield = string_cat(yield, &size, &ptr, US st);
6347           }
6348         continue;
6349
6350       case EOP_SHA1:
6351 #ifdef SUPPORT_TLS
6352         if (vp && *(void **)vp->value)
6353           {
6354           uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value);
6355           yield = string_cat(yield, &size, &ptr, cp);
6356           }
6357         else
6358 #endif
6359           {
6360           hctx h;
6361           uschar digest[20];
6362           int j;
6363           char st[41];
6364           sha1_start(&h);
6365           sha1_end(&h, sub, Ustrlen(sub), digest);
6366           for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
6367           yield = string_catn(yield, &size, &ptr, US st, 40);
6368           }
6369         continue;
6370
6371       case EOP_SHA256:
6372 #ifdef EXIM_HAVE_SHA2
6373         if (vp && *(void **)vp->value)
6374           {
6375           uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value);
6376           yield = string_cat(yield, &size, &ptr, cp);
6377           }
6378         else
6379           {
6380           hctx h;
6381           blob b;
6382           char st[3];
6383
6384           exim_sha_init(&h, HASH_SHA256);
6385           exim_sha_update(&h, sub, Ustrlen(sub));
6386           exim_sha_finish(&h, &b);
6387           while (b.len-- > 0)
6388             {
6389             sprintf(st, "%02X", *b.data++);
6390             yield = string_catn(yield, &size, &ptr, US st, 2);
6391             }
6392           }
6393 #else
6394           expand_string_message = US"sha256 only supported with TLS";
6395 #endif
6396         continue;
6397
6398       case EOP_SHA3:
6399 #ifdef EXIM_HAVE_SHA3
6400         {
6401         hctx h;
6402         blob b;
6403         char st[3];
6404         hashmethod m = !arg ? HASH_SHA3_256
6405           : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224
6406           : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256
6407           : Ustrcmp(arg, "384") == 0 ? HASH_SHA3_384
6408           : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
6409           : HASH_BADTYPE;
6410
6411         if (m == HASH_BADTYPE)
6412           {
6413           expand_string_message = US"unrecognised sha3 variant";
6414           goto EXPAND_FAILED;
6415           }
6416
6417         exim_sha_init(&h, m);
6418         exim_sha_update(&h, sub, Ustrlen(sub));
6419         exim_sha_finish(&h, &b);
6420         while (b.len-- > 0)
6421           {
6422           sprintf(st, "%02X", *b.data++);
6423           yield = string_catn(yield, &size, &ptr, US st, 2);
6424           }
6425         }
6426         continue;
6427 #else
6428         expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 +";
6429         goto EXPAND_FAILED;
6430 #endif
6431
6432       /* Convert hex encoding to base64 encoding */
6433
6434       case EOP_HEX2B64:
6435         {
6436         int c = 0;
6437         int b = -1;
6438         uschar *in = sub;
6439         uschar *out = sub;
6440         uschar *enc;
6441
6442         for (enc = sub; *enc != 0; enc++)
6443           {
6444           if (!isxdigit(*enc))
6445             {
6446             expand_string_message = string_sprintf("\"%s\" is not a hex "
6447               "string", sub);
6448             goto EXPAND_FAILED;
6449             }
6450           c++;
6451           }
6452
6453         if ((c & 1) != 0)
6454           {
6455           expand_string_message = string_sprintf("\"%s\" contains an odd "
6456             "number of characters", sub);
6457           goto EXPAND_FAILED;
6458           }
6459
6460         while ((c = *in++) != 0)
6461           {
6462           if (isdigit(c)) c -= '0';
6463           else c = toupper(c) - 'A' + 10;
6464           if (b == -1)
6465             {
6466             b = c << 4;
6467             }
6468           else
6469             {
6470             *out++ = b | c;
6471             b = -1;
6472             }
6473           }
6474
6475         enc = b64encode(sub, out - sub);
6476         yield = string_cat(yield, &size, &ptr, enc);
6477         continue;
6478         }
6479
6480       /* Convert octets outside 0x21..0x7E to \xXX form */
6481
6482       case EOP_HEXQUOTE:
6483         {
6484         uschar *t = sub - 1;
6485         while (*(++t) != 0)
6486           {
6487           if (*t < 0x21 || 0x7E < *t)
6488             yield = string_catn(yield, &size, &ptr,
6489               string_sprintf("\\x%02x", *t), 4);
6490           else
6491             yield = string_catn(yield, &size, &ptr, t, 1);
6492           }
6493         continue;
6494         }
6495
6496       /* count the number of list elements */
6497
6498       case EOP_LISTCOUNT:
6499         {
6500         int cnt = 0;
6501         int sep = 0;
6502         uschar * cp;
6503         uschar buffer[256];
6504
6505         while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
6506         cp = string_sprintf("%d", cnt);
6507         yield = string_cat(yield, &size, &ptr, cp);
6508         continue;
6509         }
6510
6511       /* expand a named list given the name */
6512       /* handles nested named lists; requotes as colon-sep list */
6513
6514       case EOP_LISTNAMED:
6515         {
6516         tree_node *t = NULL;
6517         const uschar * list;
6518         int sep = 0;
6519         uschar * item;
6520         uschar * suffix = US"";
6521         BOOL needsep = FALSE;
6522         uschar buffer[256];
6523
6524         if (*sub == '+') sub++;
6525         if (arg == NULL)        /* no-argument version */
6526           {
6527           if (!(t = tree_search(addresslist_anchor, sub)) &&
6528               !(t = tree_search(domainlist_anchor,  sub)) &&
6529               !(t = tree_search(hostlist_anchor,    sub)))
6530             t = tree_search(localpartlist_anchor, sub);
6531           }
6532         else switch(*arg)       /* specific list-type version */
6533           {
6534           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
6535           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
6536           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
6537           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
6538           default:
6539             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
6540             goto EXPAND_FAILED;
6541           }
6542
6543         if(!t)
6544           {
6545           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
6546             sub, !arg?""
6547               : *arg=='a'?"address "
6548               : *arg=='d'?"domain "
6549               : *arg=='h'?"host "
6550               : *arg=='l'?"localpart "
6551               : 0);
6552           goto EXPAND_FAILED;
6553           }
6554
6555         list = ((namedlist_block *)(t->data.ptr))->string;
6556
6557         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) != NULL)
6558           {
6559           uschar * buf = US" : ";
6560           if (needsep)
6561             yield = string_catn(yield, &size, &ptr, buf, 3);
6562           else
6563             needsep = TRUE;
6564
6565           if (*item == '+')     /* list item is itself a named list */
6566             {
6567             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
6568             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
6569             }
6570           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
6571             {
6572             char * cp;
6573             char tok[3];
6574             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
6575             while ((cp= strpbrk((const char *)item, tok)))
6576               {
6577               yield = string_catn(yield, &size, &ptr, item, cp-(char *)item);
6578               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
6579                 {
6580                 yield = string_catn(yield, &size, &ptr, US"::", 2);
6581                 item = (uschar *)cp;
6582                 }
6583               else              /* sep in item; should already be doubled; emit once */
6584                 {
6585                 yield = string_catn(yield, &size, &ptr, (uschar *)tok, 1);
6586                 if (*cp == sep) cp++;
6587                 item = (uschar *)cp;
6588                 }
6589               }
6590             }
6591           yield = string_cat(yield, &size, &ptr, item);
6592           }
6593         continue;
6594         }
6595
6596       /* mask applies a mask to an IP address; for example the result of
6597       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
6598
6599       case EOP_MASK:
6600         {
6601         int count;
6602         uschar *endptr;
6603         int binary[4];
6604         int mask, maskoffset;
6605         int type = string_is_ip_address(sub, &maskoffset);
6606         uschar buffer[64];
6607
6608         if (type == 0)
6609           {
6610           expand_string_message = string_sprintf("\"%s\" is not an IP address",
6611            sub);
6612           goto EXPAND_FAILED;
6613           }
6614
6615         if (maskoffset == 0)
6616           {
6617           expand_string_message = string_sprintf("missing mask value in \"%s\"",
6618             sub);
6619           goto EXPAND_FAILED;
6620           }
6621
6622         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
6623
6624         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
6625           {
6626           expand_string_message = string_sprintf("mask value too big in \"%s\"",
6627             sub);
6628           goto EXPAND_FAILED;
6629           }
6630
6631         /* Convert the address to binary integer(s) and apply the mask */
6632
6633         sub[maskoffset] = 0;
6634         count = host_aton(sub, binary);
6635         host_mask(count, binary, mask);
6636
6637         /* Convert to masked textual format and add to output. */
6638
6639         yield = string_catn(yield, &size, &ptr, buffer,
6640           host_nmtoa(count, binary, mask, buffer, '.'));
6641         continue;
6642         }
6643
6644       case EOP_IPV6NORM:
6645       case EOP_IPV6DENORM:
6646         {
6647         int type = string_is_ip_address(sub, NULL);
6648         int binary[4];
6649         uschar buffer[44];
6650
6651         switch (type)
6652           {
6653           case 6:
6654             (void) host_aton(sub, binary);
6655             break;
6656
6657           case 4:       /* convert to IPv4-mapped IPv6 */
6658             binary[0] = binary[1] = 0;
6659             binary[2] = 0x0000ffff;
6660             (void) host_aton(sub, binary+3);
6661             break;
6662
6663           case 0:
6664             expand_string_message =
6665               string_sprintf("\"%s\" is not an IP address", sub);
6666             goto EXPAND_FAILED;
6667           }
6668
6669         yield = string_catn(yield, &size, &ptr, buffer,
6670                   c == EOP_IPV6NORM
6671                     ? ipv6_nmtoa(binary, buffer)
6672                     : host_nmtoa(4, binary, -1, buffer, ':')
6673                   );
6674         continue;
6675         }
6676
6677       case EOP_ADDRESS:
6678       case EOP_LOCAL_PART:
6679       case EOP_DOMAIN:
6680         {
6681         uschar *error;
6682         int start, end, domain;
6683         uschar *t = parse_extract_address(sub, &error, &start, &end, &domain,
6684           FALSE);
6685         if (t != NULL)
6686           {
6687           if (c != EOP_DOMAIN)
6688             {
6689             if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1;
6690             yield = string_catn(yield, &size, &ptr, sub+start, end-start);
6691             }
6692           else if (domain != 0)
6693             {
6694             domain += start;
6695             yield = string_catn(yield, &size, &ptr, sub+domain, end-domain);
6696             }
6697           }
6698         continue;
6699         }
6700
6701       case EOP_ADDRESSES:
6702         {
6703         uschar outsep[2] = { ':', '\0' };
6704         uschar *address, *error;
6705         int save_ptr = ptr;
6706         int start, end, domain;  /* Not really used */
6707
6708         while (isspace(*sub)) sub++;
6709         if (*sub == '>') { *outsep = *++sub; ++sub; }
6710         parse_allow_group = TRUE;
6711
6712         for (;;)
6713           {
6714           uschar *p = parse_find_address_end(sub, FALSE);
6715           uschar saveend = *p;
6716           *p = '\0';
6717           address = parse_extract_address(sub, &error, &start, &end, &domain,
6718             FALSE);
6719           *p = saveend;
6720
6721           /* Add the address to the output list that we are building. This is
6722           done in chunks by searching for the separator character. At the
6723           start, unless we are dealing with the first address of the output
6724           list, add in a space if the new address begins with the separator
6725           character, or is an empty string. */
6726
6727           if (address != NULL)
6728             {
6729             if (ptr != save_ptr && address[0] == *outsep)
6730               yield = string_catn(yield, &size, &ptr, US" ", 1);
6731
6732             for (;;)
6733               {
6734               size_t seglen = Ustrcspn(address, outsep);
6735               yield = string_catn(yield, &size, &ptr, address, seglen + 1);
6736
6737               /* If we got to the end of the string we output one character
6738               too many. */
6739
6740               if (address[seglen] == '\0') { ptr--; break; }
6741               yield = string_catn(yield, &size, &ptr, outsep, 1);
6742               address += seglen + 1;
6743               }
6744
6745             /* Output a separator after the string: we will remove the
6746             redundant final one at the end. */
6747
6748             yield = string_catn(yield, &size, &ptr, outsep, 1);
6749             }
6750
6751           if (saveend == '\0') break;
6752           sub = p + 1;
6753           }
6754
6755         /* If we have generated anything, remove the redundant final
6756         separator. */
6757
6758         if (ptr != save_ptr) ptr--;
6759         parse_allow_group = FALSE;
6760         continue;
6761         }
6762
6763
6764       /* quote puts a string in quotes if it is empty or contains anything
6765       other than alphamerics, underscore, dot, or hyphen.
6766
6767       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
6768       be quoted in order to be a valid local part.
6769
6770       In both cases, newlines and carriage returns are converted into \n and \r
6771       respectively */
6772
6773       case EOP_QUOTE:
6774       case EOP_QUOTE_LOCAL_PART:
6775       if (arg == NULL)
6776         {
6777         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
6778         uschar *t = sub - 1;
6779
6780         if (c == EOP_QUOTE)
6781           {
6782           while (!needs_quote && *(++t) != 0)
6783             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
6784           }
6785         else  /* EOP_QUOTE_LOCAL_PART */
6786           {
6787           while (!needs_quote && *(++t) != 0)
6788             needs_quote = !isalnum(*t) &&
6789               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
6790               (*t != '.' || t == sub || t[1] == 0);
6791           }
6792
6793         if (needs_quote)
6794           {
6795           yield = string_catn(yield, &size, &ptr, US"\"", 1);
6796           t = sub - 1;
6797           while (*(++t) != 0)
6798             {
6799             if (*t == '\n')
6800               yield = string_catn(yield, &size, &ptr, US"\\n", 2);
6801             else if (*t == '\r')
6802               yield = string_catn(yield, &size, &ptr, US"\\r", 2);
6803             else
6804               {
6805               if (*t == '\\' || *t == '"')
6806                 yield = string_catn(yield, &size, &ptr, US"\\", 1);
6807               yield = string_catn(yield, &size, &ptr, t, 1);
6808               }
6809             }
6810           yield = string_catn(yield, &size, &ptr, US"\"", 1);
6811           }
6812         else yield = string_cat(yield, &size, &ptr, sub);
6813         continue;
6814         }
6815
6816       /* quote_lookuptype does lookup-specific quoting */
6817
6818       else
6819         {
6820         int n;
6821         uschar *opt = Ustrchr(arg, '_');
6822
6823         if (opt != NULL) *opt++ = 0;
6824
6825         n = search_findtype(arg, Ustrlen(arg));
6826         if (n < 0)
6827           {
6828           expand_string_message = search_error_message;
6829           goto EXPAND_FAILED;
6830           }
6831
6832         if (lookup_list[n]->quote != NULL)
6833           sub = (lookup_list[n]->quote)(sub, opt);
6834         else if (opt != NULL) sub = NULL;
6835
6836         if (sub == NULL)
6837           {
6838           expand_string_message = string_sprintf(
6839             "\"%s\" unrecognized after \"${quote_%s\"",
6840             opt, arg);
6841           goto EXPAND_FAILED;
6842           }
6843
6844         yield = string_cat(yield, &size, &ptr, sub);
6845         continue;
6846         }
6847
6848       /* rx quote sticks in \ before any non-alphameric character so that
6849       the insertion works in a regular expression. */
6850
6851       case EOP_RXQUOTE:
6852         {
6853         uschar *t = sub - 1;
6854         while (*(++t) != 0)
6855           {
6856           if (!isalnum(*t))
6857             yield = string_catn(yield, &size, &ptr, US"\\", 1);
6858           yield = string_catn(yield, &size, &ptr, t, 1);
6859           }
6860         continue;
6861         }
6862
6863       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
6864       prescribed by the RFC, if there are characters that need to be encoded */
6865
6866       case EOP_RFC2047:
6867         {
6868         uschar buffer[2048];
6869         const uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset,
6870           buffer, sizeof(buffer), FALSE);
6871         yield = string_cat(yield, &size, &ptr, string);
6872         continue;
6873         }
6874
6875       /* RFC 2047 decode */
6876
6877       case EOP_RFC2047D:
6878         {
6879         int len;
6880         uschar *error;
6881         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
6882           headers_charset, '?', &len, &error);
6883         if (error != NULL)
6884           {
6885           expand_string_message = error;
6886           goto EXPAND_FAILED;
6887           }
6888         yield = string_catn(yield, &size, &ptr, decoded, len);
6889         continue;
6890         }
6891
6892       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
6893       underscores */
6894
6895       case EOP_FROM_UTF8:
6896         {
6897         while (*sub != 0)
6898           {
6899           int c;
6900           uschar buff[4];
6901           GETUTF8INC(c, sub);
6902           if (c > 255) c = '_';
6903           buff[0] = c;
6904           yield = string_catn(yield, &size, &ptr, buff, 1);
6905           }
6906         continue;
6907         }
6908
6909           /* replace illegal UTF-8 sequences by replacement character  */
6910
6911       #define UTF8_REPLACEMENT_CHAR US"?"
6912
6913       case EOP_UTF8CLEAN:
6914         {
6915         int seq_len = 0, index = 0;
6916         int bytes_left = 0;
6917         long codepoint = -1;
6918         uschar seq_buff[4];                     /* accumulate utf-8 here */
6919
6920         while (*sub != 0)
6921           {
6922           int complete = 0;
6923           uschar c = *sub++;
6924
6925           if (bytes_left)
6926             {
6927             if ((c & 0xc0) != 0x80)
6928                     /* wrong continuation byte; invalidate all bytes */
6929               complete = 1; /* error */
6930             else
6931               {
6932               codepoint = (codepoint << 6) | (c & 0x3f);
6933               seq_buff[index++] = c;
6934               if (--bytes_left == 0)            /* codepoint complete */
6935                 if(codepoint > 0x10FFFF)        /* is it too large? */
6936                   complete = -1;        /* error (RFC3629 limit) */
6937                 else
6938                   {             /* finished; output utf-8 sequence */
6939                   yield = string_catn(yield, &size, &ptr, seq_buff, seq_len);
6940                   index = 0;
6941                   }
6942               }
6943             }
6944           else  /* no bytes left: new sequence */
6945             {
6946             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
6947               {
6948               yield = string_catn(yield, &size, &ptr, &c, 1);
6949               continue;
6950               }
6951             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
6952               {
6953               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
6954                 complete = -1;
6955               else
6956                 {
6957                   bytes_left = 1;
6958                   codepoint = c & 0x1f;
6959                 }
6960               }
6961             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
6962               {
6963               bytes_left = 2;
6964               codepoint = c & 0x0f;
6965               }
6966             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
6967               {
6968               bytes_left = 3;
6969               codepoint = c & 0x07;
6970               }
6971             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
6972               complete = -1;
6973
6974             seq_buff[index++] = c;
6975             seq_len = bytes_left + 1;
6976             }           /* if(bytes_left) */
6977
6978           if (complete != 0)
6979             {
6980             bytes_left = index = 0;
6981             yield = string_catn(yield, &size, &ptr, UTF8_REPLACEMENT_CHAR, 1);
6982             }
6983           if ((complete == 1) && ((c & 0x80) == 0))
6984                         /* ASCII character follows incomplete sequence */
6985               yield = string_catn(yield, &size, &ptr, &c, 1);
6986           }
6987         continue;
6988         }
6989
6990 #ifdef SUPPORT_I18N
6991       case EOP_UTF8_DOMAIN_TO_ALABEL:
6992         {
6993         uschar * error = NULL;
6994         uschar * s = string_domain_utf8_to_alabel(sub, &error);
6995         if (error)
6996           {
6997           expand_string_message = string_sprintf(
6998             "error converting utf8 (%s) to alabel: %s",
6999             string_printing(sub), error);
7000           goto EXPAND_FAILED;
7001           }
7002         yield = string_cat(yield, &size, &ptr, s);
7003         continue;
7004         }
7005
7006       case EOP_UTF8_DOMAIN_FROM_ALABEL:
7007         {
7008         uschar * error = NULL;
7009         uschar * s = string_domain_alabel_to_utf8(sub, &error);
7010         if (error)
7011           {
7012           expand_string_message = string_sprintf(
7013             "error converting alabel (%s) to utf8: %s",
7014             string_printing(sub), error);
7015           goto EXPAND_FAILED;
7016           }
7017         yield = string_cat(yield, &size, &ptr, s);
7018         continue;
7019         }
7020
7021       case EOP_UTF8_LOCALPART_TO_ALABEL:
7022         {
7023         uschar * error = NULL;
7024         uschar * s = string_localpart_utf8_to_alabel(sub, &error);
7025         if (error)
7026           {
7027           expand_string_message = string_sprintf(
7028             "error converting utf8 (%s) to alabel: %s",
7029             string_printing(sub), error);
7030           goto EXPAND_FAILED;
7031           }
7032         yield = string_cat(yield, &size, &ptr, s);
7033         DEBUG(D_expand) debug_printf("yield: '%s'\n", yield);
7034         continue;
7035         }
7036
7037       case EOP_UTF8_LOCALPART_FROM_ALABEL:
7038         {
7039         uschar * error = NULL;
7040         uschar * s = string_localpart_alabel_to_utf8(sub, &error);
7041         if (error)
7042           {
7043           expand_string_message = string_sprintf(
7044             "error converting alabel (%s) to utf8: %s",
7045             string_printing(sub), error);
7046           goto EXPAND_FAILED;
7047           }
7048         yield = string_cat(yield, &size, &ptr, s);
7049         continue;
7050         }
7051 #endif  /* EXPERIMENTAL_INTERNATIONAL */
7052
7053       /* escape turns all non-printing characters into escape sequences. */
7054
7055       case EOP_ESCAPE:
7056         {
7057         const uschar *t = string_printing(sub);
7058         yield = string_cat(yield, &size, &ptr, t);
7059         continue;
7060         }
7061
7062       /* Handle numeric expression evaluation */
7063
7064       case EOP_EVAL:
7065       case EOP_EVAL10:
7066         {
7067         uschar *save_sub = sub;
7068         uschar *error = NULL;
7069         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
7070         if (error != NULL)
7071           {
7072           expand_string_message = string_sprintf("error in expression "
7073             "evaluation: %s (after processing \"%.*s\")", error, sub-save_sub,
7074               save_sub);
7075           goto EXPAND_FAILED;
7076           }
7077         sprintf(CS var_buffer, PR_EXIM_ARITH, n);
7078         yield = string_cat(yield, &size, &ptr, var_buffer);
7079         continue;
7080         }
7081
7082       /* Handle time period formating */
7083
7084       case EOP_TIME_EVAL:
7085         {
7086         int n = readconf_readtime(sub, 0, FALSE);
7087         if (n < 0)
7088           {
7089           expand_string_message = string_sprintf("string \"%s\" is not an "
7090             "Exim time interval in \"%s\" operator", sub, name);
7091           goto EXPAND_FAILED;
7092           }
7093         sprintf(CS var_buffer, "%d", n);
7094         yield = string_cat(yield, &size, &ptr, var_buffer);
7095         continue;
7096         }
7097
7098       case EOP_TIME_INTERVAL:
7099         {
7100         int n;
7101         uschar *t = read_number(&n, sub);
7102         if (*t != 0) /* Not A Number*/
7103           {
7104           expand_string_message = string_sprintf("string \"%s\" is not a "
7105             "positive number in \"%s\" operator", sub, name);
7106           goto EXPAND_FAILED;
7107           }
7108         t = readconf_printtime(n);
7109         yield = string_cat(yield, &size, &ptr, t);
7110         continue;
7111         }
7112
7113       /* Convert string to base64 encoding */
7114
7115       case EOP_STR2B64:
7116       case EOP_BASE64:
7117         {
7118 #ifdef SUPPORT_TLS
7119         uschar * s = vp && *(void **)vp->value
7120           ? tls_cert_der_b64(*(void **)vp->value)
7121           : b64encode(sub, Ustrlen(sub));
7122 #else
7123         uschar * s = b64encode(sub, Ustrlen(sub));
7124 #endif
7125         yield = string_cat(yield, &size, &ptr, s);
7126         continue;
7127         }
7128
7129       case EOP_BASE64D:
7130         {
7131         uschar * s;
7132         int len = b64decode(sub, &s);
7133         if (len < 0)
7134           {
7135           expand_string_message = string_sprintf("string \"%s\" is not "
7136             "well-formed for \"%s\" operator", sub, name);
7137           goto EXPAND_FAILED;
7138           }
7139         yield = string_cat(yield, &size, &ptr, s);
7140         continue;
7141         }
7142
7143       /* strlen returns the length of the string */
7144
7145       case EOP_STRLEN:
7146         {
7147         uschar buff[24];
7148         (void)sprintf(CS buff, "%d", Ustrlen(sub));
7149         yield = string_cat(yield, &size, &ptr, buff);
7150         continue;
7151         }
7152
7153       /* length_n or l_n takes just the first n characters or the whole string,
7154       whichever is the shorter;
7155
7156       substr_m_n, and s_m_n take n characters from offset m; negative m take
7157       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
7158       takes the rest, either to the right or to the left.
7159
7160       hash_n or h_n makes a hash of length n from the string, yielding n
7161       characters from the set a-z; hash_n_m makes a hash of length n, but
7162       uses m characters from the set a-zA-Z0-9.
7163
7164       nhash_n returns a single number between 0 and n-1 (in text form), while
7165       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
7166       between 0 and n-1 and the second between 0 and m-1. */
7167
7168       case EOP_LENGTH:
7169       case EOP_L:
7170       case EOP_SUBSTR:
7171       case EOP_S:
7172       case EOP_HASH:
7173       case EOP_H:
7174       case EOP_NHASH:
7175       case EOP_NH:
7176         {
7177         int sign = 1;
7178         int value1 = 0;
7179         int value2 = -1;
7180         int *pn;
7181         int len;
7182         uschar *ret;
7183
7184         if (arg == NULL)
7185           {
7186           expand_string_message = string_sprintf("missing values after %s",
7187             name);
7188           goto EXPAND_FAILED;
7189           }
7190
7191         /* "length" has only one argument, effectively being synonymous with
7192         substr_0_n. */
7193
7194         if (c == EOP_LENGTH || c == EOP_L)
7195           {
7196           pn = &value2;
7197           value2 = 0;
7198           }
7199
7200         /* The others have one or two arguments; for "substr" the first may be
7201         negative. The second being negative means "not supplied". */
7202
7203         else
7204           {
7205           pn = &value1;
7206           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
7207           }
7208
7209         /* Read up to two numbers, separated by underscores */
7210
7211         ret = arg;
7212         while (*arg != 0)
7213           {
7214           if (arg != ret && *arg == '_' && pn == &value1)
7215             {
7216             pn = &value2;
7217             value2 = 0;
7218             if (arg[1] != 0) arg++;
7219             }
7220           else if (!isdigit(*arg))
7221             {
7222             expand_string_message =
7223               string_sprintf("non-digit after underscore in \"%s\"", name);
7224             goto EXPAND_FAILED;
7225             }
7226           else *pn = (*pn)*10 + *arg++ - '0';
7227           }
7228         value1 *= sign;
7229
7230         /* Perform the required operation */
7231
7232         ret =
7233           (c == EOP_HASH || c == EOP_H)?
7234              compute_hash(sub, value1, value2, &len) :
7235           (c == EOP_NHASH || c == EOP_NH)?
7236              compute_nhash(sub, value1, value2, &len) :
7237              extract_substr(sub, value1, value2, &len);
7238
7239         if (ret == NULL) goto EXPAND_FAILED;
7240         yield = string_catn(yield, &size, &ptr, ret, len);
7241         continue;
7242         }
7243
7244       /* Stat a path */
7245
7246       case EOP_STAT:
7247         {
7248         uschar *s;
7249         uschar smode[12];
7250         uschar **modetable[3];
7251         int i;
7252         mode_t mode;
7253         struct stat st;
7254
7255         if ((expand_forbid & RDO_EXISTS) != 0)
7256           {
7257           expand_string_message = US"Use of the stat() expansion is not permitted";
7258           goto EXPAND_FAILED;
7259           }
7260
7261         if (stat(CS sub, &st) < 0)
7262           {
7263           expand_string_message = string_sprintf("stat(%s) failed: %s",
7264             sub, strerror(errno));
7265           goto EXPAND_FAILED;
7266           }
7267         mode = st.st_mode;
7268         switch (mode & S_IFMT)
7269           {
7270           case S_IFIFO: smode[0] = 'p'; break;
7271           case S_IFCHR: smode[0] = 'c'; break;
7272           case S_IFDIR: smode[0] = 'd'; break;
7273           case S_IFBLK: smode[0] = 'b'; break;
7274           case S_IFREG: smode[0] = '-'; break;
7275           default: smode[0] = '?'; break;
7276           }
7277
7278         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
7279         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
7280         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
7281
7282         for (i = 0; i < 3; i++)
7283           {
7284           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
7285           mode >>= 3;
7286           }
7287
7288         smode[10] = 0;
7289         s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
7290           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
7291           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
7292           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
7293           (long)st.st_gid, st.st_size, (long)st.st_atime,
7294           (long)st.st_mtime, (long)st.st_ctime);
7295         yield = string_cat(yield, &size, &ptr, s);
7296         continue;
7297         }
7298
7299       /* vaguely random number less than N */
7300
7301       case EOP_RANDINT:
7302         {
7303         int_eximarith_t max;
7304         uschar *s;
7305
7306         max = expanded_string_integer(sub, TRUE);
7307         if (expand_string_message != NULL)
7308           goto EXPAND_FAILED;
7309         s = string_sprintf("%d", vaguely_random_number((int)max));
7310         yield = string_cat(yield, &size, &ptr, s);
7311         continue;
7312         }
7313
7314       /* Reverse IP, including IPv6 to dotted-nibble */
7315
7316       case EOP_REVERSE_IP:
7317         {
7318         int family, maskptr;
7319         uschar reversed[128];
7320
7321         family = string_is_ip_address(sub, &maskptr);
7322         if (family == 0)
7323           {
7324           expand_string_message = string_sprintf(
7325               "reverse_ip() not given an IP address [%s]", sub);
7326           goto EXPAND_FAILED;
7327           }
7328         invert_address(reversed, sub);
7329         yield = string_cat(yield, &size, &ptr, reversed);
7330         continue;
7331         }
7332
7333       /* Unknown operator */
7334
7335       default:
7336       expand_string_message =
7337         string_sprintf("unknown expansion operator \"%s\"", name);
7338       goto EXPAND_FAILED;
7339       }
7340     }
7341
7342   /* Handle a plain name. If this is the first thing in the expansion, release
7343   the pre-allocated buffer. If the result data is known to be in a new buffer,
7344   newsize will be set to the size of that buffer, and we can just point at that
7345   store instead of copying. Many expansion strings contain just one reference,
7346   so this is a useful optimization, especially for humungous headers
7347   ($message_headers). */
7348                                                 /*{*/
7349   if (*s++ == '}')
7350     {
7351     int len;
7352     int newsize = 0;
7353     if (ptr == 0)
7354       {
7355       if (resetok) store_reset(yield);
7356       yield = NULL;
7357       size = 0;
7358       }
7359     value = find_variable(name, FALSE, skipping, &newsize);
7360     if (value == NULL)
7361       {
7362       expand_string_message =
7363         string_sprintf("unknown variable in \"${%s}\"", name);
7364       check_variable_error_message(name);
7365       goto EXPAND_FAILED;
7366       }
7367     len = Ustrlen(value);
7368     if (yield == NULL && newsize != 0)
7369       {
7370       yield = value;
7371       size = newsize;
7372       ptr = len;
7373       }
7374     else yield = string_catn(yield, &size, &ptr, value, len);
7375     continue;
7376     }
7377
7378   /* Else there's something wrong */
7379
7380   expand_string_message =
7381     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
7382     "in a variable reference)", name);
7383   goto EXPAND_FAILED;
7384   }
7385
7386 /* If we hit the end of the string when ket_ends is set, there is a missing
7387 terminating brace. */
7388
7389 if (ket_ends && *s == 0)
7390   {
7391   expand_string_message = malformed_header?
7392     US"missing } at end of string - could be header name not terminated by colon"
7393     :
7394     US"missing } at end of string";
7395   goto EXPAND_FAILED;
7396   }
7397
7398 /* Expansion succeeded; yield may still be NULL here if nothing was actually
7399 added to the string. If so, set up an empty string. Add a terminating zero. If
7400 left != NULL, return a pointer to the terminator. */
7401
7402 if (yield == NULL) yield = store_get(1);
7403 yield[ptr] = 0;
7404 if (left != NULL) *left = s;
7405
7406 /* Any stacking store that was used above the final string is no longer needed.
7407 In many cases the final string will be the first one that was got and so there
7408 will be optimal store usage. */
7409
7410 if (resetok) store_reset(yield + ptr + 1);
7411 else if (resetok_p) *resetok_p = FALSE;
7412
7413 DEBUG(D_expand)
7414   {
7415   debug_printf("  expanding: %.*s\n     result: %s\n", (int)(s - string), string,
7416     yield);
7417   if (skipping) debug_printf("   skipping: result is not used\n");
7418   }
7419 return yield;
7420
7421 /* This is the failure exit: easiest to program with a goto. We still need
7422 to update the pointer to the terminator, for cases of nested calls with "fail".
7423 */
7424
7425 EXPAND_FAILED_CURLY:
7426 if (malformed_header)
7427   expand_string_message =
7428     US"missing or misplaced { or } - could be header name not terminated by colon";
7429
7430 else if (!expand_string_message || !*expand_string_message)
7431   expand_string_message = US"missing or misplaced { or }";
7432
7433 /* At one point, Exim reset the store to yield (if yield was not NULL), but
7434 that is a bad idea, because expand_string_message is in dynamic store. */
7435
7436 EXPAND_FAILED:
7437 if (left != NULL) *left = s;
7438 DEBUG(D_expand)
7439   {
7440   debug_printf("failed to expand: %s\n", string);
7441   debug_printf("   error message: %s\n", expand_string_message);
7442   if (expand_string_forcedfail) debug_printf("failure was forced\n");
7443   }
7444 if (resetok_p) *resetok_p = resetok;
7445 return NULL;
7446 }
7447
7448
7449 /* This is the external function call. Do a quick check for any expansion
7450 metacharacters, and if there are none, just return the input string.
7451
7452 Argument: the string to be expanded
7453 Returns:  the expanded string, or NULL if expansion failed; if failure was
7454           due to a lookup deferring, search_find_defer will be TRUE
7455 */
7456
7457 uschar *
7458 expand_string(uschar *string)
7459 {
7460 search_find_defer = FALSE;
7461 malformed_header = FALSE;
7462 return (Ustrpbrk(string, "$\\") == NULL)? string :
7463   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7464 }
7465
7466
7467
7468 const uschar *
7469 expand_cstring(const uschar *string)
7470 {
7471 search_find_defer = FALSE;
7472 malformed_header = FALSE;
7473 return (Ustrpbrk(string, "$\\") == NULL)? string :
7474   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7475 }
7476
7477
7478
7479 /*************************************************
7480 *              Expand and copy                   *
7481 *************************************************/
7482
7483 /* Now and again we want to expand a string and be sure that the result is in a
7484 new bit of store. This function does that.
7485 Since we know it has been copied, the de-const cast is safe.
7486
7487 Argument: the string to be expanded
7488 Returns:  the expanded string, always in a new bit of store, or NULL
7489 */
7490
7491 uschar *
7492 expand_string_copy(const uschar *string)
7493 {
7494 const uschar *yield = expand_cstring(string);
7495 if (yield == string) yield = string_copy(string);
7496 return US yield;
7497 }
7498
7499
7500
7501 /*************************************************
7502 *        Expand and interpret as an integer      *
7503 *************************************************/
7504
7505 /* Expand a string, and convert the result into an integer.
7506
7507 Arguments:
7508   string  the string to be expanded
7509   isplus  TRUE if a non-negative number is expected
7510
7511 Returns:  the integer value, or
7512           -1 for an expansion error               ) in both cases, message in
7513           -2 for an integer interpretation error  ) expand_string_message
7514           expand_string_message is set NULL for an OK integer
7515 */
7516
7517 int_eximarith_t
7518 expand_string_integer(uschar *string, BOOL isplus)
7519 {
7520 return expanded_string_integer(expand_string(string), isplus);
7521 }
7522
7523
7524 /*************************************************
7525  *         Interpret string as an integer        *
7526  *************************************************/
7527
7528 /* Convert a string (that has already been expanded) into an integer.
7529
7530 This function is used inside the expansion code.
7531
7532 Arguments:
7533   s       the string to be expanded
7534   isplus  TRUE if a non-negative number is expected
7535
7536 Returns:  the integer value, or
7537           -1 if string is NULL (which implies an expansion error)
7538           -2 for an integer interpretation error
7539           expand_string_message is set NULL for an OK integer
7540 */
7541
7542 static int_eximarith_t
7543 expanded_string_integer(const uschar *s, BOOL isplus)
7544 {
7545 int_eximarith_t value;
7546 uschar *msg = US"invalid integer \"%s\"";
7547 uschar *endptr;
7548
7549 /* If expansion failed, expand_string_message will be set. */
7550
7551 if (s == NULL) return -1;
7552
7553 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
7554 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
7555 systems, so we set it zero ourselves. */
7556
7557 errno = 0;
7558 expand_string_message = NULL;               /* Indicates no error */
7559
7560 /* Before Exim 4.64, strings consisting entirely of whitespace compared
7561 equal to 0.  Unfortunately, people actually relied upon that, so preserve
7562 the behaviour explicitly.  Stripping leading whitespace is a harmless
7563 noop change since strtol skips it anyway (provided that there is a number
7564 to find at all). */
7565 if (isspace(*s))
7566   {
7567   while (isspace(*s)) ++s;
7568   if (*s == '\0')
7569     {
7570       DEBUG(D_expand)
7571        debug_printf("treating blank string as number 0\n");
7572       return 0;
7573     }
7574   }
7575
7576 value = strtoll(CS s, CSS &endptr, 10);
7577
7578 if (endptr == s)
7579   {
7580   msg = US"integer expected but \"%s\" found";
7581   }
7582 else if (value < 0 && isplus)
7583   {
7584   msg = US"non-negative integer expected but \"%s\" found";
7585   }
7586 else
7587   {
7588   switch (tolower(*endptr))
7589     {
7590     default:
7591       break;
7592     case 'k':
7593       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
7594       else value *= 1024;
7595       endptr++;
7596       break;
7597     case 'm':
7598       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
7599       else value *= 1024*1024;
7600       endptr++;
7601       break;
7602     case 'g':
7603       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
7604       else value *= 1024*1024*1024;
7605       endptr++;
7606       break;
7607     }
7608   if (errno == ERANGE)
7609     msg = US"absolute value of integer \"%s\" is too large (overflow)";
7610   else
7611     {
7612     while (isspace(*endptr)) endptr++;
7613     if (*endptr == 0) return value;
7614     }
7615   }
7616
7617 expand_string_message = string_sprintf(CS msg, s);
7618 return -2;
7619 }
7620
7621
7622 /* These values are usually fixed boolean values, but they are permitted to be
7623 expanded strings.
7624
7625 Arguments:
7626   addr       address being routed
7627   mtype      the module type
7628   mname      the module name
7629   dbg_opt    debug selectors
7630   oname      the option name
7631   bvalue     the router's boolean value
7632   svalue     the router's string value
7633   rvalue     where to put the returned value
7634
7635 Returns:     OK     value placed in rvalue
7636              DEFER  expansion failed
7637 */
7638
7639 int
7640 exp_bool(address_item *addr,
7641   uschar *mtype, uschar *mname, unsigned dbg_opt,
7642   uschar *oname, BOOL bvalue,
7643   uschar *svalue, BOOL *rvalue)
7644 {
7645 uschar *expanded;
7646 if (svalue == NULL) { *rvalue = bvalue; return OK; }
7647
7648 expanded = expand_string(svalue);
7649 if (expanded == NULL)
7650   {
7651   if (expand_string_forcedfail)
7652     {
7653     DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname);
7654     *rvalue = bvalue;
7655     return OK;
7656     }
7657   addr->message = string_sprintf("failed to expand \"%s\" in %s %s: %s",
7658       oname, mname, mtype, expand_string_message);
7659   DEBUG(dbg_opt) debug_printf("%s\n", addr->message);
7660   return DEFER;
7661   }
7662
7663 DEBUG(dbg_opt) debug_printf("expansion of \"%s\" yields \"%s\"\n", oname,
7664   expanded);
7665
7666 if (strcmpic(expanded, US"true") == 0 || strcmpic(expanded, US"yes") == 0)
7667   *rvalue = TRUE;
7668 else if (strcmpic(expanded, US"false") == 0 || strcmpic(expanded, US"no") == 0)
7669   *rvalue = FALSE;
7670 else
7671   {
7672   addr->message = string_sprintf("\"%s\" is not a valid value for the "
7673     "\"%s\" option in the %s %s", expanded, oname, mname, mtype);
7674   return DEFER;
7675   }
7676
7677 return OK;
7678 }
7679
7680
7681
7682 /* Avoid potentially exposing a password in a string about to be logged */
7683
7684 uschar *
7685 expand_hide_passwords(uschar * s)
7686 {
7687 return (  (  Ustrstr(s, "failed to expand") != NULL
7688           || Ustrstr(s, "expansion of ")    != NULL
7689           )
7690        && (  Ustrstr(s, "mysql")   != NULL
7691           || Ustrstr(s, "pgsql")   != NULL
7692           || Ustrstr(s, "redis")   != NULL
7693           || Ustrstr(s, "sqlite")  != NULL
7694           || Ustrstr(s, "ldap:")   != NULL
7695           || Ustrstr(s, "ldaps:")  != NULL
7696           || Ustrstr(s, "ldapi:")  != NULL
7697           || Ustrstr(s, "ldapdn:") != NULL
7698           || Ustrstr(s, "ldapm:")  != NULL
7699        )  )
7700   ? US"Temporary internal error" : s;
7701 }
7702
7703
7704
7705
7706 /*************************************************
7707 **************************************************
7708 *             Stand-alone test program           *
7709 **************************************************
7710 *************************************************/
7711
7712 #ifdef STAND_ALONE
7713
7714
7715 BOOL
7716 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
7717 {
7718 int ovector[3*(EXPAND_MAXN+1)];
7719 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
7720   ovector, nelem(ovector));
7721 BOOL yield = n >= 0;
7722 if (n == 0) n = EXPAND_MAXN + 1;
7723 if (yield)
7724   {
7725   int nn;
7726   expand_nmax = (setup < 0)? 0 : setup + 1;
7727   for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2)
7728     {
7729     expand_nstring[expand_nmax] = subject + ovector[nn];
7730     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
7731     }
7732   expand_nmax--;
7733   }
7734 return yield;
7735 }
7736
7737
7738 int main(int argc, uschar **argv)
7739 {
7740 int i;
7741 uschar buffer[1024];
7742
7743 debug_selector = D_v;
7744 debug_file = stderr;
7745 debug_fd = fileno(debug_file);
7746 big_buffer = malloc(big_buffer_size);
7747
7748 for (i = 1; i < argc; i++)
7749   {
7750   if (argv[i][0] == '+')
7751     {
7752     debug_trace_memory = 2;
7753     argv[i]++;
7754     }
7755   if (isdigit(argv[i][0]))
7756     debug_selector = Ustrtol(argv[i], NULL, 0);
7757   else
7758     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
7759         Ustrlen(argv[i]))
7760       {
7761 #ifdef LOOKUP_LDAP
7762       eldap_default_servers = argv[i];
7763 #endif
7764 #ifdef LOOKUP_MYSQL
7765       mysql_servers = argv[i];
7766 #endif
7767 #ifdef LOOKUP_PGSQL
7768       pgsql_servers = argv[i];
7769 #endif
7770 #ifdef LOOKUP_REDIS
7771       redis_servers = argv[i];
7772 #endif
7773       }
7774 #ifdef EXIM_PERL
7775   else opt_perl_startup = argv[i];
7776 #endif
7777   }
7778
7779 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
7780
7781 expand_nstring[1] = US"string 1....";
7782 expand_nlength[1] = 8;
7783 expand_nmax = 1;
7784
7785 #ifdef EXIM_PERL
7786 if (opt_perl_startup != NULL)
7787   {
7788   uschar *errstr;
7789   printf("Starting Perl interpreter\n");
7790   errstr = init_perl(opt_perl_startup);
7791   if (errstr != NULL)
7792     {
7793     printf("** error in perl_startup code: %s\n", errstr);
7794     return EXIT_FAILURE;
7795     }
7796   }
7797 #endif /* EXIM_PERL */
7798
7799 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
7800   {
7801   void *reset_point = store_get(0);
7802   uschar *yield = expand_string(buffer);
7803   if (yield != NULL)
7804     {
7805     printf("%s\n", yield);
7806     store_reset(reset_point);
7807     }
7808   else
7809     {
7810     if (search_find_defer) printf("search_find deferred\n");
7811     printf("Failed: %s\n", expand_string_message);
7812     if (expand_string_forcedfail) printf("Forced failure\n");
7813     printf("\n");
7814     }
7815   }
7816
7817 search_tidyup();
7818
7819 return 0;
7820 }
7821
7822 #endif
7823
7824 /* vi: aw ai sw=2
7825 */
7826 /* End of expand.c */