src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2016 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOOL, BOOL, BOOL *);
  17 static int_eximarith_t expanded_string_integer(const uschar *, BOOL);
  18
  19 #ifdef STAND_ALONE
  20 #ifndef SUPPORT_CRYPTEQ
  21 #define SUPPORT_CRYPTEQ
  22 #endif
  23 #endif
  24
  25 #ifdef LOOKUP_LDAP
  26 #include "lookups/ldap.h"
  27 #endif
  28
  29 #ifdef SUPPORT_CRYPTEQ
  30 #ifdef CRYPT_H
  31 #include <crypt.h>
  32 #endif
  33 #ifndef HAVE_CRYPT16
  34 extern char* crypt16(char*, char*);
  35 #endif
  36 #endif
  37
  38 /* The handling of crypt16() is a mess. I will record below the analysis of the
  39 mess that was sent to me. We decided, however, to make changing this very low
  40 priority, because in practice people are moving away from the crypt()
  41 algorithms nowadays, so it doesn't seem worth it.
  42
  43 <quote>
  44 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  45 the first 8 characters of the password using a 20-round version of crypt
  46 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  47 or an empty block if the password is less than 9 characters, using a
  48 20-round version of crypt and the same salt as was used for the first
  49 block.  Characters after the first 16 are ignored.  It always generates
  50 a 16-byte hash, which is expressed together with the salt as a string
  51 of 24 base 64 digits.  Here are some links to peruse:
  52
  53         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  54         http://seclists.org/bugtraq/1999/Mar/0076.html
  55
  56 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  57 and OSF/1.  This is the same as the standard crypt if given a password
  58 of 8 characters or less.  If given more, it first does the same as crypt
  59 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  60 using as salt the first two base 64 digits from the first hash block.
  61 If the password is more than 16 characters then it crypts the 17th to 24th
  62 characters using as salt the first two base 64 digits from the second hash
  63 block.  And so on: I've seen references to it cutting off the password at
  64 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  65
  66         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  67         http://seclists.org/bugtraq/1999/Mar/0109.html
  68         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  69              TET1_html/sec.c222.html#no_id_208
  70
  71 Exim has something it calls "crypt16".  It will either use a native
  72 crypt16 or its own implementation.  A native crypt16 will presumably
  73 be the one that I called "crypt16" above.  The internal "crypt16"
  74 function, however, is a two-block-maximum implementation of what I called
  75 "bigcrypt".  The documentation matches the internal code.
  76
  77 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  78 that crypt16 and bigcrypt were different things.
  79
  80 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  81 to whatever it is using under that name.  This unfortunately sets a
  82 precedent for using "{crypt16}" to identify two incompatible algorithms
  83 whose output can't be distinguished.  With "{crypt16}" thus rendered
  84 ambiguous, I suggest you deprecate it and invent two new identifiers
  85 for the two algorithms.
  86
  87 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  88 of the password separately means they can be cracked separately, so
  89 the double-length hash only doubles the cracking effort instead of
  90 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  91 bcrypt ({CRYPT}$2a$).
  92 </quote>
  93 */
  94
  95
  96
  97 /*************************************************
  98 *            Local statics and tables            *
  99 *************************************************/
 100
 101 /* Table of item names, and corresponding switch numbers. The names must be in
 102 alphabetical order. */
 103
 104 static uschar *item_table[] = {
 105   US"acl",
 106   US"certextract",
 107   US"dlfunc",
 108   US"env",
 109   US"extract",
 110   US"filter",
 111   US"hash",
 112   US"hmac",
 113   US"if",
 114 #ifdef SUPPORT_I18N
 115   US"imapfolder",
 116 #endif
 117   US"length",
 118   US"listextract",
 119   US"lookup",
 120   US"map",
 121   US"nhash",
 122   US"perl",
 123   US"prvs",
 124   US"prvscheck",
 125   US"readfile",
 126   US"readsocket",
 127   US"reduce",
 128   US"run",
 129   US"sg",
 130   US"sort",
 131   US"substr",
 132   US"tr" };
 133
 134 enum {
 135   EITEM_ACL,
 136   EITEM_CERTEXTRACT,
 137   EITEM_DLFUNC,
 138   EITEM_ENV,
 139   EITEM_EXTRACT,
 140   EITEM_FILTER,
 141   EITEM_HASH,
 142   EITEM_HMAC,
 143   EITEM_IF,
 144 #ifdef SUPPORT_I18N
 145   EITEM_IMAPFOLDER,
 146 #endif
 147   EITEM_LENGTH,
 148   EITEM_LISTEXTRACT,
 149   EITEM_LOOKUP,
 150   EITEM_MAP,
 151   EITEM_NHASH,
 152   EITEM_PERL,
 153   EITEM_PRVS,
 154   EITEM_PRVSCHECK,
 155   EITEM_READFILE,
 156   EITEM_READSOCK,
 157   EITEM_REDUCE,
 158   EITEM_RUN,
 159   EITEM_SG,
 160   EITEM_SORT,
 161   EITEM_SUBSTR,
 162   EITEM_TR };
 163
 164 /* Tables of operator names, and corresponding switch numbers. The names must be
 165 in alphabetical order. There are two tables, because underscore is used in some
 166 cases to introduce arguments, whereas for other it is part of the name. This is
 167 an historical mis-design. */
 168
 169 static uschar *op_table_underscore[] = {
 170   US"from_utf8",
 171   US"local_part",
 172   US"quote_local_part",
 173   US"reverse_ip",
 174   US"time_eval",
 175   US"time_interval"
 176 #ifdef SUPPORT_I18N
 177  ,US"utf8_domain_from_alabel",
 178   US"utf8_domain_to_alabel",
 179   US"utf8_localpart_from_alabel",
 180   US"utf8_localpart_to_alabel"
 181 #endif
 182   };
 183
 184 enum {
 185   EOP_FROM_UTF8,
 186   EOP_LOCAL_PART,
 187   EOP_QUOTE_LOCAL_PART,
 188   EOP_REVERSE_IP,
 189   EOP_TIME_EVAL,
 190   EOP_TIME_INTERVAL
 191 #ifdef SUPPORT_I18N
 192  ,EOP_UTF8_DOMAIN_FROM_ALABEL,
 193   EOP_UTF8_DOMAIN_TO_ALABEL,
 194   EOP_UTF8_LOCALPART_FROM_ALABEL,
 195   EOP_UTF8_LOCALPART_TO_ALABEL
 196 #endif
 197   };
 198
 199 static uschar *op_table_main[] = {
 200   US"address",
 201   US"addresses",
 202   US"base32",
 203   US"base32d",
 204   US"base62",
 205   US"base62d",
 206   US"base64",
 207   US"base64d",
 208   US"domain",
 209   US"escape",
 210   US"escape8bit",
 211   US"eval",
 212   US"eval10",
 213   US"expand",
 214   US"h",
 215   US"hash",
 216   US"hex2b64",
 217   US"hexquote",
 218   US"ipv6denorm",
 219   US"ipv6norm",
 220   US"l",
 221   US"lc",
 222   US"length",
 223   US"listcount",
 224   US"listnamed",
 225   US"mask",
 226   US"md5",
 227   US"nh",
 228   US"nhash",
 229   US"quote",
 230   US"randint",
 231   US"rfc2047",
 232   US"rfc2047d",
 233   US"rxquote",
 234   US"s",
 235   US"sha1",
 236   US"sha256",
 237   US"sha3",
 238   US"stat",
 239   US"str2b64",
 240   US"strlen",
 241   US"substr",
 242   US"uc",
 243   US"utf8clean" };
 244
 245 enum {
 246   EOP_ADDRESS =  nelem(op_table_underscore),
 247   EOP_ADDRESSES,
 248   EOP_BASE32,
 249   EOP_BASE32D,
 250   EOP_BASE62,
 251   EOP_BASE62D,
 252   EOP_BASE64,
 253   EOP_BASE64D,
 254   EOP_DOMAIN,
 255   EOP_ESCAPE,
 256   EOP_ESCAPE8BIT,
 257   EOP_EVAL,
 258   EOP_EVAL10,
 259   EOP_EXPAND,
 260   EOP_H,
 261   EOP_HASH,
 262   EOP_HEX2B64,
 263   EOP_HEXQUOTE,
 264   EOP_IPV6DENORM,
 265   EOP_IPV6NORM,
 266   EOP_L,
 267   EOP_LC,
 268   EOP_LENGTH,
 269   EOP_LISTCOUNT,
 270   EOP_LISTNAMED,
 271   EOP_MASK,
 272   EOP_MD5,
 273   EOP_NH,
 274   EOP_NHASH,
 275   EOP_QUOTE,
 276   EOP_RANDINT,
 277   EOP_RFC2047,
 278   EOP_RFC2047D,
 279   EOP_RXQUOTE,
 280   EOP_S,
 281   EOP_SHA1,
 282   EOP_SHA256,
 283   EOP_SHA3,
 284   EOP_STAT,
 285   EOP_STR2B64,
 286   EOP_STRLEN,
 287   EOP_SUBSTR,
 288   EOP_UC,
 289   EOP_UTF8CLEAN };
 290
 291
 292 /* Table of condition names, and corresponding switch numbers. The names must
 293 be in alphabetical order. */
 294
 295 static uschar *cond_table[] = {
 296   US"<",
 297   US"<=",
 298   US"=",
 299   US"==",     /* Backward compatibility */
 300   US">",
 301   US">=",
 302   US"acl",
 303   US"and",
 304   US"bool",
 305   US"bool_lax",
 306   US"crypteq",
 307   US"def",
 308   US"eq",
 309   US"eqi",
 310   US"exists",
 311   US"first_delivery",
 312   US"forall",
 313   US"forany",
 314   US"ge",
 315   US"gei",
 316   US"gt",
 317   US"gti",
 318   US"inlist",
 319   US"inlisti",
 320   US"isip",
 321   US"isip4",
 322   US"isip6",
 323   US"ldapauth",
 324   US"le",
 325   US"lei",
 326   US"lt",
 327   US"lti",
 328   US"match",
 329   US"match_address",
 330   US"match_domain",
 331   US"match_ip",
 332   US"match_local_part",
 333   US"or",
 334   US"pam",
 335   US"pwcheck",
 336   US"queue_running",
 337   US"radius",
 338   US"saslauthd"
 339 };
 340
 341 enum {
 342   ECOND_NUM_L,
 343   ECOND_NUM_LE,
 344   ECOND_NUM_E,
 345   ECOND_NUM_EE,
 346   ECOND_NUM_G,
 347   ECOND_NUM_GE,
 348   ECOND_ACL,
 349   ECOND_AND,
 350   ECOND_BOOL,
 351   ECOND_BOOL_LAX,
 352   ECOND_CRYPTEQ,
 353   ECOND_DEF,
 354   ECOND_STR_EQ,
 355   ECOND_STR_EQI,
 356   ECOND_EXISTS,
 357   ECOND_FIRST_DELIVERY,
 358   ECOND_FORALL,
 359   ECOND_FORANY,
 360   ECOND_STR_GE,
 361   ECOND_STR_GEI,
 362   ECOND_STR_GT,
 363   ECOND_STR_GTI,
 364   ECOND_INLIST,
 365   ECOND_INLISTI,
 366   ECOND_ISIP,
 367   ECOND_ISIP4,
 368   ECOND_ISIP6,
 369   ECOND_LDAPAUTH,
 370   ECOND_STR_LE,
 371   ECOND_STR_LEI,
 372   ECOND_STR_LT,
 373   ECOND_STR_LTI,
 374   ECOND_MATCH,
 375   ECOND_MATCH_ADDRESS,
 376   ECOND_MATCH_DOMAIN,
 377   ECOND_MATCH_IP,
 378   ECOND_MATCH_LOCAL_PART,
 379   ECOND_OR,
 380   ECOND_PAM,
 381   ECOND_PWCHECK,
 382   ECOND_QUEUE_RUNNING,
 383   ECOND_RADIUS,
 384   ECOND_SASLAUTHD
 385 };
 386
 387
 388 /* Types of table entry */
 389
 390 enum vtypes {
 391   vtype_int,            /* value is address of int */
 392   vtype_filter_int,     /* ditto, but recognized only when filtering */
 393   vtype_ino,            /* value is address of ino_t (not always an int) */
 394   vtype_uid,            /* value is address of uid_t (not always an int) */
 395   vtype_gid,            /* value is address of gid_t (not always an int) */
 396   vtype_bool,           /* value is address of bool */
 397   vtype_stringptr,      /* value is address of pointer to string */
 398   vtype_msgbody,        /* as stringptr, but read when first required */
 399   vtype_msgbody_end,    /* ditto, the end of the message */
 400   vtype_msgheaders,     /* the message's headers, processed */
 401   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 402   vtype_localpart,      /* extract local part from string */
 403   vtype_domain,         /* extract domain from string */
 404   vtype_string_func,    /* value is string returned by given function */
 405   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 406   vtype_tode,           /* value not used; generate tod in epoch format */
 407   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 408   vtype_todf,           /* value not used; generate full tod */
 409   vtype_todl,           /* value not used; generate log tod */
 410   vtype_todlf,          /* value not used; generate log file datestamp tod */
 411   vtype_todzone,        /* value not used; generate time zone only */
 412   vtype_todzulu,        /* value not used; generate zulu tod */
 413   vtype_reply,          /* value not used; get reply from headers */
 414   vtype_pid,            /* value not used; result is pid */
 415   vtype_host_lookup,    /* value not used; get host name */
 416   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 417   vtype_pspace,         /* partition space; value is T/F for spool/log */
 418   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 419   vtype_cert            /* SSL certificate */
 420   #ifndef DISABLE_DKIM
 421   ,vtype_dkim           /* Lookup of value in DKIM signature */
 422   #endif
 423 };
 424
 425 /* Type for main variable table */
 426
 427 typedef struct {
 428   const char *name;
 429   enum vtypes type;
 430   void       *value;
 431 } var_entry;
 432
 433 /* Type for entries pointing to address/length pairs. Not currently
 434 in use. */
 435
 436 typedef struct {
 437   uschar **address;
 438   int  *length;
 439 } alblock;
 440
 441 static uschar * fn_recipients(void);
 442
 443 /* This table must be kept in alphabetical order. */
 444
 445 static var_entry var_table[] = {
 446   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 447      they will be confused with user-creatable ACL variables. */
 448   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 449   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 450   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 451   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 452   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 453   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 454   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 455   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 456   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 457   { "acl_narg",            vtype_int,         &acl_narg },
 458   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 459   { "address_data",        vtype_stringptr,   &deliver_address_data },
 460   { "address_file",        vtype_stringptr,   &address_file },
 461   { "address_pipe",        vtype_stringptr,   &address_pipe },
 462   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 463   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 464   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 465   { "authentication_failed",vtype_int,        &authentication_failed },
 466 #ifdef WITH_CONTENT_SCAN
 467   { "av_failed",           vtype_int,         &av_failed },
 468 #endif
 469 #ifdef EXPERIMENTAL_BRIGHTMAIL
 470   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 471   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 472   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 473   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 474 #endif
 475   { "body_linecount",      vtype_int,         &body_linecount },
 476   { "body_zerocount",      vtype_int,         &body_zerocount },
 477   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 478   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 479   { "caller_gid",          vtype_gid,         &real_gid },
 480   { "caller_uid",          vtype_uid,         &real_uid },
 481   { "callout_address",     vtype_stringptr,   &callout_address },
 482   { "compile_date",        vtype_stringptr,   &version_date },
 483   { "compile_number",      vtype_stringptr,   &version_cnumber },
 484   { "config_dir",          vtype_stringptr,   &config_main_directory },
 485   { "config_file",         vtype_stringptr,   &config_main_filename },
 486   { "csa_status",          vtype_stringptr,   &csa_status },
 487 #ifdef EXPERIMENTAL_DCC
 488   { "dcc_header",          vtype_stringptr,   &dcc_header },
 489   { "dcc_result",          vtype_stringptr,   &dcc_result },
 490 #endif
 491 #ifndef DISABLE_DKIM
 492   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 493   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 494   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 495   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 496   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 497   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 498   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 499   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 500   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 501   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 502   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 503   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 504   { "dkim_key_length",     vtype_int,         &dkim_key_length },
 505   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 506   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 507   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 508   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 509   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 510   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 511   { "dkim_verify_reason",  vtype_dkim,        (void *)DKIM_VERIFY_REASON },
 512   { "dkim_verify_status",  vtype_dkim,        (void *)DKIM_VERIFY_STATUS},
 513 #endif
 514 #ifdef EXPERIMENTAL_DMARC
 515   { "dmarc_ar_header",     vtype_stringptr,   &dmarc_ar_header },
 516   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 517   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 518   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 519   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 520 #endif
 521   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 522   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 523   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 524   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 525   { "domain",              vtype_stringptr,   &deliver_domain },
 526   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 527 #ifndef DISABLE_EVENT
 528   { "event_data",          vtype_stringptr,   &event_data },
 529
 530   /*XXX want to use generic vars for as many of these as possible*/
 531   { "event_defer_errno",   vtype_int,         &event_defer_errno },
 532
 533   { "event_name",          vtype_stringptr,   &event_name },
 534 #endif
 535   { "exim_gid",            vtype_gid,         &exim_gid },
 536   { "exim_path",           vtype_stringptr,   &exim_path },
 537   { "exim_uid",            vtype_uid,         &exim_uid },
 538   { "exim_version",        vtype_stringptr,   &version_string },
 539   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 540   { "home",                vtype_stringptr,   &deliver_home },
 541   { "host",                vtype_stringptr,   &deliver_host },
 542   { "host_address",        vtype_stringptr,   &deliver_host_address },
 543   { "host_data",           vtype_stringptr,   &host_data },
 544   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 545   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 546   { "host_port",           vtype_int,         &deliver_host_port },
 547   { "initial_cwd",         vtype_stringptr,   &initial_cwd },
 548   { "inode",               vtype_ino,         &deliver_inode },
 549   { "interface_address",   vtype_stringptr,   &interface_address },
 550   { "interface_port",      vtype_int,         &interface_port },
 551   { "item",                vtype_stringptr,   &iterate_item },
 552   #ifdef LOOKUP_LDAP
 553   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 554   #endif
 555   { "load_average",        vtype_load_avg,    NULL },
 556   { "local_part",          vtype_stringptr,   &deliver_localpart },
 557   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 558   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 559   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 560   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 561   { "local_user_gid",      vtype_gid,         &local_user_gid },
 562   { "local_user_uid",      vtype_uid,         &local_user_uid },
 563   { "localhost_number",    vtype_int,         &host_number },
 564   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 565   { "log_space",           vtype_pspace,      (void *)FALSE },
 566   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 567   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 568 #ifdef WITH_CONTENT_SCAN
 569   { "malware_name",        vtype_stringptr,   &malware_name },
 570 #endif
 571   { "max_received_linelength", vtype_int,     &max_received_linelength },
 572   { "message_age",         vtype_int,         &message_age },
 573   { "message_body",        vtype_msgbody,     &message_body },
 574   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 575   { "message_body_size",   vtype_int,         &message_body_size },
 576   { "message_exim_id",     vtype_stringptr,   &message_id },
 577   { "message_headers",     vtype_msgheaders,  NULL },
 578   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 579   { "message_id",          vtype_stringptr,   &message_id },
 580   { "message_linecount",   vtype_int,         &message_linecount },
 581   { "message_size",        vtype_int,         &message_size },
 582 #ifdef SUPPORT_I18N
 583   { "message_smtputf8",    vtype_bool,        &message_smtputf8 },
 584 #endif
 585 #ifdef WITH_CONTENT_SCAN
 586   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 587   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 588   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 589   { "mime_charset",        vtype_stringptr,   &mime_charset },
 590   { "mime_content_description", vtype_stringptr, &mime_content_description },
 591   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 592   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 593   { "mime_content_size",   vtype_int,         &mime_content_size },
 594   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 595   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 596   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 597   { "mime_filename",       vtype_stringptr,   &mime_filename },
 598   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 599   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 600   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 601   { "mime_part_count",     vtype_int,         &mime_part_count },
 602 #endif
 603   { "n0",                  vtype_filter_int,  &filter_n[0] },
 604   { "n1",                  vtype_filter_int,  &filter_n[1] },
 605   { "n2",                  vtype_filter_int,  &filter_n[2] },
 606   { "n3",                  vtype_filter_int,  &filter_n[3] },
 607   { "n4",                  vtype_filter_int,  &filter_n[4] },
 608   { "n5",                  vtype_filter_int,  &filter_n[5] },
 609   { "n6",                  vtype_filter_int,  &filter_n[6] },
 610   { "n7",                  vtype_filter_int,  &filter_n[7] },
 611   { "n8",                  vtype_filter_int,  &filter_n[8] },
 612   { "n9",                  vtype_filter_int,  &filter_n[9] },
 613   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 614   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 615   { "originator_gid",      vtype_gid,         &originator_gid },
 616   { "originator_uid",      vtype_uid,         &originator_uid },
 617   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 618   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 619   { "pid",                 vtype_pid,         NULL },
 620 #ifndef DISABLE_PRDR
 621   { "prdr_requested",      vtype_bool,        &prdr_requested },
 622 #endif
 623   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 624 #if defined(SUPPORT_PROXY) || defined(SUPPORT_SOCKS)
 625   { "proxy_external_address",vtype_stringptr, &proxy_external_address },
 626   { "proxy_external_port", vtype_int,         &proxy_external_port },
 627   { "proxy_local_address", vtype_stringptr,   &proxy_local_address },
 628   { "proxy_local_port",    vtype_int,         &proxy_local_port },
 629   { "proxy_session",       vtype_bool,        &proxy_session },
 630 #endif
 631   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 632   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 633   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 634   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 635   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 636   { "queue_name",          vtype_stringptr,   &queue_name },
 637   { "rcpt_count",          vtype_int,         &rcpt_count },
 638   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 639   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 640   { "received_count",      vtype_int,         &received_count },
 641   { "received_for",        vtype_stringptr,   &received_for },
 642   { "received_ip_address", vtype_stringptr,   &interface_address },
 643   { "received_port",       vtype_int,         &interface_port },
 644   { "received_protocol",   vtype_stringptr,   &received_protocol },
 645   { "received_time",       vtype_int,         &received_time.tv_sec },
 646   { "recipient_data",      vtype_stringptr,   &recipient_data },
 647   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 648   { "recipients",          vtype_string_func, &fn_recipients },
 649   { "recipients_count",    vtype_int,         &recipients_count },
 650 #ifdef WITH_CONTENT_SCAN
 651   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 652 #endif
 653   { "reply_address",       vtype_reply,       NULL },
 654   { "return_path",         vtype_stringptr,   &return_path },
 655   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 656   { "router_name",         vtype_stringptr,   &router_name },
 657   { "runrc",               vtype_int,         &runrc },
 658   { "self_hostname",       vtype_stringptr,   &self_hostname },
 659   { "sender_address",      vtype_stringptr,   &sender_address },
 660   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 661   { "sender_address_domain", vtype_domain,    &sender_address },
 662   { "sender_address_local_part", vtype_localpart, &sender_address },
 663   { "sender_data",         vtype_stringptr,   &sender_data },
 664   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 665   { "sender_helo_dnssec",  vtype_bool,        &sender_helo_dnssec },
 666   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 667   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 668   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 669   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 670   { "sender_host_name",    vtype_host_lookup, NULL },
 671   { "sender_host_port",    vtype_int,         &sender_host_port },
 672   { "sender_ident",        vtype_stringptr,   &sender_ident },
 673   { "sender_rate",         vtype_stringptr,   &sender_rate },
 674   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 675   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 676   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 677   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 678   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 679   { "sending_port",        vtype_int,         &sending_port },
 680   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 681   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 682   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 683   { "smtp_command_history", vtype_string_func, &smtp_cmd_hist },
 684   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 685   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 686   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 687   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 688   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 689   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 690   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 691   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 692   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 693   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 694   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 695   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 696 #ifdef WITH_CONTENT_SCAN
 697   { "spam_action",         vtype_stringptr,   &spam_action },
 698   { "spam_bar",            vtype_stringptr,   &spam_bar },
 699   { "spam_report",         vtype_stringptr,   &spam_report },
 700   { "spam_score",          vtype_stringptr,   &spam_score },
 701   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 702 #endif
 703 #ifdef EXPERIMENTAL_SPF
 704   { "spf_guess",           vtype_stringptr,   &spf_guess },
 705   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 706   { "spf_received",        vtype_stringptr,   &spf_received },
 707   { "spf_result",          vtype_stringptr,   &spf_result },
 708   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 709 #endif
 710   { "spool_directory",     vtype_stringptr,   &spool_directory },
 711   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 712   { "spool_space",         vtype_pspace,      (void *)TRUE },
 713 #ifdef EXPERIMENTAL_SRS
 714   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 715   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 716   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 717   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 718   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 719   { "srs_status",          vtype_stringptr,   &srs_status },
 720 #endif
 721   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 722
 723   /* The non-(in,out) variables are now deprecated */
 724   { "tls_bits",            vtype_int,         &tls_in.bits },
 725   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 726   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 727
 728   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 729   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 730   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 731   { "tls_in_ocsp",         vtype_int,         &tls_in.ocsp },
 732   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 733   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 734   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 735 #if defined(SUPPORT_TLS)
 736   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 737 #endif
 738   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 739   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 740   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 741 #ifdef EXPERIMENTAL_DANE
 742   { "tls_out_dane",        vtype_bool,        &tls_out.dane_verified },
 743 #endif
 744   { "tls_out_ocsp",        vtype_int,         &tls_out.ocsp },
 745   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 746   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 747   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 748 #if defined(SUPPORT_TLS)
 749   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 750 #endif
 751 #ifdef EXPERIMENTAL_DANE
 752   { "tls_out_tlsa_usage",  vtype_int,         &tls_out.tlsa_usage },
 753 #endif
 754
 755   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 756 #if defined(SUPPORT_TLS)
 757   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 758 #endif
 759
 760   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 761   { "tod_epoch",           vtype_tode,        NULL },
 762   { "tod_epoch_l",         vtype_todel,       NULL },
 763   { "tod_full",            vtype_todf,        NULL },
 764   { "tod_log",             vtype_todl,        NULL },
 765   { "tod_logfile",         vtype_todlf,       NULL },
 766   { "tod_zone",            vtype_todzone,     NULL },
 767   { "tod_zulu",            vtype_todzulu,     NULL },
 768   { "transport_name",      vtype_stringptr,   &transport_name },
 769   { "value",               vtype_stringptr,   &lookup_value },
 770   { "verify_mode",         vtype_stringptr,   &verify_mode },
 771   { "version_number",      vtype_stringptr,   &version_string },
 772   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 773   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 774   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 775   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 776   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 777   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 778 };
 779
 780 static int var_table_size = nelem(var_table);
 781 static uschar var_buffer[256];
 782 static BOOL malformed_header;
 783
 784 /* For textual hashes */
 785
 786 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 787                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 788                                "0123456789";
 789
 790 enum { HMAC_MD5, HMAC_SHA1 };
 791
 792 /* For numeric hashes */
 793
 794 static unsigned int prime[] = {
 795   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 796  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 797  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 798
 799 /* For printing modes in symbolic form */
 800
 801 static uschar *mtable_normal[] =
 802   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 803
 804 static uschar *mtable_setid[] =
 805   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 806
 807 static uschar *mtable_sticky[] =
 808   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 809
 810
 811
 812 /*************************************************
 813 *           Tables for UTF-8 support             *
 814 *************************************************/
 815
 816 /* Table of the number of extra characters, indexed by the first character
 817 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 818 0x3d. */
 819
 820 static uschar utf8_table1[] = {
 821   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 822   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 823   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 824   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 825
 826 /* These are the masks for the data bits in the first byte of a character,
 827 indexed by the number of additional bytes. */
 828
 829 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 830
 831 /* Get the next UTF-8 character, advancing the pointer. */
 832
 833 #define GETUTF8INC(c, ptr) \
 834   c = *ptr++; \
 835   if ((c & 0xc0) == 0xc0) \
 836     { \
 837     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 838     int s = 6*a; \
 839     c = (c & utf8_table2[a]) << s; \
 840     while (a-- > 0) \
 841       { \
 842       s -= 6; \
 843       c |= (*ptr++ & 0x3f) << s; \
 844       } \
 845     }
 846
 847
 848
 849 static uschar * base32_chars = US"abcdefghijklmnopqrstuvwxyz234567";
 850
 851 /*************************************************
 852 *           Binary chop search on a table        *
 853 *************************************************/
 854
 855 /* This is used for matching expansion items and operators.
 856
 857 Arguments:
 858   name        the name that is being sought
 859   table       the table to search
 860   table_size  the number of items in the table
 861
 862 Returns:      the offset in the table, or -1
 863 */
 864
 865 static int
 866 chop_match(uschar *name, uschar **table, int table_size)
 867 {
 868 uschar **bot = table;
 869 uschar **top = table + table_size;
 870
 871 while (top > bot)
 872   {
 873   uschar **mid = bot + (top - bot)/2;
 874   int c = Ustrcmp(name, *mid);
 875   if (c == 0) return mid - table;
 876   if (c > 0) bot = mid + 1; else top = mid;
 877   }
 878
 879 return -1;
 880 }
 881
 882
 883
 884 /*************************************************
 885 *          Check a condition string              *
 886 *************************************************/
 887
 888 /* This function is called to expand a string, and test the result for a "true"
 889 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 890 forced fail or lookup defer.
 891
 892 We used to release all store used, but this is not not safe due
 893 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 894 is reasonably careful to release what it can.
 895
 896 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 897
 898 Arguments:
 899   condition     the condition string
 900   m1            text to be incorporated in panic error
 901   m2            ditto
 902
 903 Returns:        TRUE if condition is met, FALSE if not
 904 */
 905
 906 BOOL
 907 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 908 {
 909 int rc;
 910 uschar *ss = expand_string(condition);
 911 if (ss == NULL)
 912   {
 913   if (!expand_string_forcedfail && !search_find_defer)
 914     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 915       "for %s %s: %s", condition, m1, m2, expand_string_message);
 916   return FALSE;
 917   }
 918 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 919   strcmpic(ss, US"false") != 0;
 920 return rc;
 921 }
 922
 923
 924
 925
 926 /*************************************************
 927 *        Pseudo-random number generation         *
 928 *************************************************/
 929
 930 /* Pseudo-random number generation.  The result is not "expected" to be
 931 cryptographically strong but not so weak that someone will shoot themselves
 932 in the foot using it as a nonce in some email header scheme or whatever
 933 weirdness they'll twist this into.  The result should ideally handle fork().
 934
 935 However, if we're stuck unable to provide this, then we'll fall back to
 936 appallingly bad randomness.
 937
 938 If SUPPORT_TLS is defined then this will not be used except as an emergency
 939 fallback.
 940
 941 Arguments:
 942   max       range maximum
 943 Returns     a random number in range [0, max-1]
 944 */
 945
 946 #ifdef SUPPORT_TLS
 947 # define vaguely_random_number vaguely_random_number_fallback
 948 #endif
 949 int
 950 vaguely_random_number(int max)
 951 {
 952 #ifdef SUPPORT_TLS
 953 # undef vaguely_random_number
 954 #endif
 955   static pid_t pid = 0;
 956   pid_t p2;
 957 #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
 958   struct timeval tv;
 959 #endif
 960
 961   p2 = getpid();
 962   if (p2 != pid)
 963     {
 964     if (pid != 0)
 965       {
 966
 967 #ifdef HAVE_ARC4RANDOM
 968       /* cryptographically strong randomness, common on *BSD platforms, not
 969       so much elsewhere.  Alas. */
 970 #ifndef NOT_HAVE_ARC4RANDOM_STIR
 971       arc4random_stir();
 972 #endif
 973 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 974 #ifdef HAVE_SRANDOMDEV
 975       /* uses random(4) for seeding */
 976       srandomdev();
 977 #else
 978       gettimeofday(&tv, NULL);
 979       srandom(tv.tv_sec | tv.tv_usec | getpid());
 980 #endif
 981 #else
 982       /* Poor randomness and no seeding here */
 983 #endif
 984
 985       }
 986     pid = p2;
 987     }
 988
 989 #ifdef HAVE_ARC4RANDOM
 990   return arc4random() % max;
 991 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 992   return random() % max;
 993 #else
 994   /* This one returns a 16-bit number, definitely not crypto-strong */
 995   return random_number(max);
 996 #endif
 997 }
 998
 999
1000
1001
1002 /*************************************************
1003 *             Pick out a name from a string      *
1004 *************************************************/
1005
1006 /* If the name is too long, it is silently truncated.
1007
1008 Arguments:
1009   name      points to a buffer into which to put the name
1010   max       is the length of the buffer
1011   s         points to the first alphabetic character of the name
1012   extras    chars other than alphanumerics to permit
1013
1014 Returns:    pointer to the first character after the name
1015
1016 Note: The test for *s != 0 in the while loop is necessary because
1017 Ustrchr() yields non-NULL if the character is zero (which is not something
1018 I expected). */
1019
1020 static const uschar *
1021 read_name(uschar *name, int max, const uschar *s, uschar *extras)
1022 {
1023 int ptr = 0;
1024 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
1025   {
1026   if (ptr < max-1) name[ptr++] = *s;
1027   s++;
1028   }
1029 name[ptr] = 0;
1030 return s;
1031 }
1032
1033
1034
1035 /*************************************************
1036 *     Pick out the rest of a header name         *
1037 *************************************************/
1038
1039 /* A variable name starting $header_ (or just $h_ for those who like
1040 abbreviations) might not be the complete header name because headers can
1041 contain any printing characters in their names, except ':'. This function is
1042 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
1043 on the end, if the name was terminated by white space.
1044
1045 Arguments:
1046   name      points to a buffer in which the name read so far exists
1047   max       is the length of the buffer
1048   s         points to the first character after the name so far, i.e. the
1049             first non-alphameric character after $header_xxxxx
1050
1051 Returns:    a pointer to the first character after the header name
1052 */
1053
1054 static const uschar *
1055 read_header_name(uschar *name, int max, const uschar *s)
1056 {
1057 int prelen = Ustrchr(name, '_') - name + 1;
1058 int ptr = Ustrlen(name) - prelen;
1059 if (ptr > 0) memmove(name, name+prelen, ptr);
1060 while (mac_isgraph(*s) && *s != ':')
1061   {
1062   if (ptr < max-1) name[ptr++] = *s;
1063   s++;
1064   }
1065 if (*s == ':') s++;
1066 name[ptr++] = ':';
1067 name[ptr] = 0;
1068 return s;
1069 }
1070
1071
1072
1073 /*************************************************
1074 *           Pick out a number from a string      *
1075 *************************************************/
1076
1077 /* Arguments:
1078   n     points to an integer into which to put the number
1079   s     points to the first digit of the number
1080
1081 Returns:  a pointer to the character after the last digit
1082 */
1083 /*XXX consider expanding to int_eximarith_t.  But the test for
1084 "overbig numbers" in 0002 still needs to overflow it. */
1085
1086 static uschar *
1087 read_number(int *n, uschar *s)
1088 {
1089 *n = 0;
1090 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1091 return s;
1092 }
1093
1094 static const uschar *
1095 read_cnumber(int *n, const uschar *s)
1096 {
1097 *n = 0;
1098 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1099 return s;
1100 }
1101
1102
1103
1104 /*************************************************
1105 *        Extract keyed subfield from a string    *
1106 *************************************************/
1107
1108 /* The yield is in dynamic store; NULL means that the key was not found.
1109
1110 Arguments:
1111   key       points to the name of the key
1112   s         points to the string from which to extract the subfield
1113
1114 Returns:    NULL if the subfield was not found, or
1115             a pointer to the subfield's data
1116 */
1117
1118 static uschar *
1119 expand_getkeyed(uschar *key, const uschar *s)
1120 {
1121 int length = Ustrlen(key);
1122 while (isspace(*s)) s++;
1123
1124 /* Loop to search for the key */
1125
1126 while (*s != 0)
1127   {
1128   int dkeylength;
1129   uschar *data;
1130   const uschar *dkey = s;
1131
1132   while (*s != 0 && *s != '=' && !isspace(*s)) s++;
1133   dkeylength = s - dkey;
1134   while (isspace(*s)) s++;
1135   if (*s == '=') while (isspace((*(++s))));
1136
1137   data = string_dequote(&s);
1138   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1139     return data;
1140
1141   while (isspace(*s)) s++;
1142   }
1143
1144 return NULL;
1145 }
1146
1147
1148
1149 static var_entry *
1150 find_var_ent(uschar * name)
1151 {
1152 int first = 0;
1153 int last = var_table_size;
1154
1155 while (last > first)
1156   {
1157   int middle = (first + last)/2;
1158   int c = Ustrcmp(name, var_table[middle].name);
1159
1160   if (c > 0) { first = middle + 1; continue; }
1161   if (c < 0) { last = middle; continue; }
1162   return &var_table[middle];
1163   }
1164 return NULL;
1165 }
1166
1167 /*************************************************
1168 *   Extract numbered subfield from string        *
1169 *************************************************/
1170
1171 /* Extracts a numbered field from a string that is divided by tokens - for
1172 example a line from /etc/passwd is divided by colon characters.  First field is
1173 numbered one.  Negative arguments count from the right. Zero returns the whole
1174 string. Returns NULL if there are insufficient tokens in the string
1175
1176 ***WARNING***
1177 Modifies final argument - this is a dynamically generated string, so that's OK.
1178
1179 Arguments:
1180   field       number of field to be extracted,
1181                 first field = 1, whole string = 0, last field = -1
1182   separators  characters that are used to break string into tokens
1183   s           points to the string from which to extract the subfield
1184
1185 Returns:      NULL if the field was not found,
1186               a pointer to the field's data inside s (modified to add 0)
1187 */
1188
1189 static uschar *
1190 expand_gettokened (int field, uschar *separators, uschar *s)
1191 {
1192 int sep = 1;
1193 int count;
1194 uschar *ss = s;
1195 uschar *fieldtext = NULL;
1196
1197 if (field == 0) return s;
1198
1199 /* Break the line up into fields in place; for field > 0 we stop when we have
1200 done the number of fields we want. For field < 0 we continue till the end of
1201 the string, counting the number of fields. */
1202
1203 count = (field > 0)? field : INT_MAX;
1204
1205 while (count-- > 0)
1206   {
1207   size_t len;
1208
1209   /* Previous field was the last one in the string. For a positive field
1210   number, this means there are not enough fields. For a negative field number,
1211   check that there are enough, and scan back to find the one that is wanted. */
1212
1213   if (sep == 0)
1214     {
1215     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1216     if ((-field) == (INT_MAX - count - 1)) return s;
1217     while (field++ < 0)
1218       {
1219       ss--;
1220       while (ss[-1] != 0) ss--;
1221       }
1222     fieldtext = ss;
1223     break;
1224     }
1225
1226   /* Previous field was not last in the string; save its start and put a
1227   zero at its end. */
1228
1229   fieldtext = ss;
1230   len = Ustrcspn(ss, separators);
1231   sep = ss[len];
1232   ss[len] = 0;
1233   ss += len + 1;
1234   }
1235
1236 return fieldtext;
1237 }
1238
1239
1240 static uschar *
1241 expand_getlistele(int field, const uschar * list)
1242 {
1243 const uschar * tlist= list;
1244 int sep= 0;
1245 uschar dummy;
1246
1247 if(field<0)
1248   {
1249   for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1250   sep= 0;
1251   }
1252 if(field==0) return NULL;
1253 while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1254 return string_nextinlist(&list, &sep, NULL, 0);
1255 }
1256
1257
1258 /* Certificate fields, by name.  Worry about by-OID later */
1259 /* Names are chosen to not have common prefixes */
1260
1261 #ifdef SUPPORT_TLS
1262 typedef struct
1263 {
1264 uschar * name;
1265 int      namelen;
1266 uschar * (*getfn)(void * cert, uschar * mod);
1267 } certfield;
1268 static certfield certfields[] =
1269 {                       /* linear search; no special order */
1270   { US"version",         7,  &tls_cert_version },
1271   { US"serial_number",   13, &tls_cert_serial_number },
1272   { US"subject",         7,  &tls_cert_subject },
1273   { US"notbefore",       9,  &tls_cert_not_before },
1274   { US"notafter",        8,  &tls_cert_not_after },
1275   { US"issuer",          6,  &tls_cert_issuer },
1276   { US"signature",       9,  &tls_cert_signature },
1277   { US"sig_algorithm",   13, &tls_cert_signature_algorithm },
1278   { US"subj_altname",    12, &tls_cert_subject_altname },
1279   { US"ocsp_uri",        8,  &tls_cert_ocsp_uri },
1280   { US"crl_uri",         7,  &tls_cert_crl_uri },
1281 };
1282
1283 static uschar *
1284 expand_getcertele(uschar * field, uschar * certvar)
1285 {
1286 var_entry * vp;
1287 certfield * cp;
1288
1289 if (!(vp = find_var_ent(certvar)))
1290   {
1291   expand_string_message =
1292     string_sprintf("no variable named \"%s\"", certvar);
1293   return NULL;          /* Unknown variable name */
1294   }
1295 /* NB this stops us passing certs around in variable.  Might
1296 want to do that in future */
1297 if (vp->type != vtype_cert)
1298   {
1299   expand_string_message =
1300     string_sprintf("\"%s\" is not a certificate", certvar);
1301   return NULL;          /* Unknown variable name */
1302   }
1303 if (!*(void **)vp->value)
1304   return NULL;
1305
1306 if (*field >= '0' && *field <= '9')
1307   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1308
1309 for(cp = certfields;
1310     cp < certfields + nelem(certfields);
1311     cp++)
1312   if (Ustrncmp(cp->name, field, cp->namelen) == 0)
1313     {
1314     uschar * modifier = *(field += cp->namelen) == ','
1315       ? ++field : NULL;
1316     return (*cp->getfn)( *(void **)vp->value, modifier );
1317     }
1318
1319 expand_string_message =
1320   string_sprintf("bad field selector \"%s\" for certextract", field);
1321 return NULL;
1322 }
1323 #endif  /*SUPPORT_TLS*/
1324
1325 /*************************************************
1326 *        Extract a substring from a string       *
1327 *************************************************/
1328
1329 /* Perform the ${substr or ${length expansion operations.
1330
1331 Arguments:
1332   subject     the input string
1333   value1      the offset from the start of the input string to the start of
1334                 the output string; if negative, count from the right.
1335   value2      the length of the output string, or negative (-1) for unset
1336                 if value1 is positive, unset means "all after"
1337                 if value1 is negative, unset means "all before"
1338   len         set to the length of the returned string
1339
1340 Returns:      pointer to the output string, or NULL if there is an error
1341 */
1342
1343 static uschar *
1344 extract_substr(uschar *subject, int value1, int value2, int *len)
1345 {
1346 int sublen = Ustrlen(subject);
1347
1348 if (value1 < 0)    /* count from right */
1349   {
1350   value1 += sublen;
1351
1352   /* If the position is before the start, skip to the start, and adjust the
1353   length. If the length ends up negative, the substring is null because nothing
1354   can precede. This falls out naturally when the length is unset, meaning "all
1355   to the left". */
1356
1357   if (value1 < 0)
1358     {
1359     value2 += value1;
1360     if (value2 < 0) value2 = 0;
1361     value1 = 0;
1362     }
1363
1364   /* Otherwise an unset length => characters before value1 */
1365
1366   else if (value2 < 0)
1367     {
1368     value2 = value1;
1369     value1 = 0;
1370     }
1371   }
1372
1373 /* For a non-negative offset, if the starting position is past the end of the
1374 string, the result will be the null string. Otherwise, an unset length means
1375 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1376
1377 else
1378   {
1379   if (value1 > sublen)
1380     {
1381     value1 = sublen;
1382     value2 = 0;
1383     }
1384   else if (value2 < 0) value2 = sublen;
1385   }
1386
1387 /* Cut the length down to the maximum possible for the offset value, and get
1388 the required characters. */
1389
1390 if (value1 + value2 > sublen) value2 = sublen - value1;
1391 *len = value2;
1392 return subject + value1;
1393 }
1394
1395
1396
1397
1398 /*************************************************
1399 *            Old-style hash of a string          *
1400 *************************************************/
1401
1402 /* Perform the ${hash expansion operation.
1403
1404 Arguments:
1405   subject     the input string (an expanded substring)
1406   value1      the length of the output string; if greater or equal to the
1407                 length of the input string, the input string is returned
1408   value2      the number of hash characters to use, or 26 if negative
1409   len         set to the length of the returned string
1410
1411 Returns:      pointer to the output string, or NULL if there is an error
1412 */
1413
1414 static uschar *
1415 compute_hash(uschar *subject, int value1, int value2, int *len)
1416 {
1417 int sublen = Ustrlen(subject);
1418
1419 if (value2 < 0) value2 = 26;
1420 else if (value2 > Ustrlen(hashcodes))
1421   {
1422   expand_string_message =
1423     string_sprintf("hash count \"%d\" too big", value2);
1424   return NULL;
1425   }
1426
1427 /* Calculate the hash text. We know it is shorter than the original string, so
1428 can safely place it in subject[] (we know that subject is always itself an
1429 expanded substring). */
1430
1431 if (value1 < sublen)
1432   {
1433   int c;
1434   int i = 0;
1435   int j = value1;
1436   while ((c = (subject[j])) != 0)
1437     {
1438     int shift = (c + j++) & 7;
1439     subject[i] ^= (c << shift) | (c >> (8-shift));
1440     if (++i >= value1) i = 0;
1441     }
1442   for (i = 0; i < value1; i++)
1443     subject[i] = hashcodes[(subject[i]) % value2];
1444   }
1445 else value1 = sublen;
1446
1447 *len = value1;
1448 return subject;
1449 }
1450
1451
1452
1453
1454 /*************************************************
1455 *             Numeric hash of a string           *
1456 *************************************************/
1457
1458 /* Perform the ${nhash expansion operation. The first characters of the
1459 string are treated as most important, and get the highest prime numbers.
1460
1461 Arguments:
1462   subject     the input string
1463   value1      the maximum value of the first part of the result
1464   value2      the maximum value of the second part of the result,
1465                 or negative to produce only a one-part result
1466   len         set to the length of the returned string
1467
1468 Returns:  pointer to the output string, or NULL if there is an error.
1469 */
1470
1471 static uschar *
1472 compute_nhash (uschar *subject, int value1, int value2, int *len)
1473 {
1474 uschar *s = subject;
1475 int i = 0;
1476 unsigned long int total = 0; /* no overflow */
1477
1478 while (*s != 0)
1479   {
1480   if (i == 0) i = nelem(prime) - 1;
1481   total += prime[i--] * (unsigned int)(*s++);
1482   }
1483
1484 /* If value2 is unset, just compute one number */
1485
1486 if (value2 < 0)
1487   s = string_sprintf("%d", total % value1);
1488
1489 /* Otherwise do a div/mod hash */
1490
1491 else
1492   {
1493   total = total % (value1 * value2);
1494   s = string_sprintf("%d/%d", total/value2, total % value2);
1495   }
1496
1497 *len = Ustrlen(s);
1498 return s;
1499 }
1500
1501
1502
1503
1504
1505 /*************************************************
1506 *     Find the value of a header or headers      *
1507 *************************************************/
1508
1509 /* Multiple instances of the same header get concatenated, and this function
1510 can also return a concatenation of all the header lines. When concatenating
1511 specific headers that contain lists of addresses, a comma is inserted between
1512 them. Otherwise we use a straight concatenation. Because some messages can have
1513 pathologically large number of lines, there is a limit on the length that is
1514 returned. Also, to avoid massive store use which would result from using
1515 string_cat() as it copies and extends strings, we do a preliminary pass to find
1516 out exactly how much store will be needed. On "normal" messages this will be
1517 pretty trivial.
1518
1519 Arguments:
1520   name          the name of the header, without the leading $header_ or $h_,
1521                 or NULL if a concatenation of all headers is required
1522   exists_only   TRUE if called from a def: test; don't need to build a string;
1523                 just return a string that is not "" and not "0" if the header
1524                 exists
1525   newsize       return the size of memory block that was obtained; may be NULL
1526                 if exists_only is TRUE
1527   want_raw      TRUE if called for $rh_ or $rheader_ variables; no processing,
1528                 other than concatenating, will be done on the header. Also used
1529                 for $message_headers_raw.
1530   charset       name of charset to translate MIME words to; used only if
1531                 want_raw is false; if NULL, no translation is done (this is
1532                 used for $bh_ and $bheader_)
1533
1534 Returns:        NULL if the header does not exist, else a pointer to a new
1535                 store block
1536 */
1537
1538 static uschar *
1539 find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw,
1540   uschar *charset)
1541 {
1542 BOOL found = name == NULL;
1543 int comma = 0;
1544 int len = found? 0 : Ustrlen(name);
1545 int i;
1546 uschar *yield = NULL;
1547 uschar *ptr = NULL;
1548
1549 /* Loop for two passes - saves code repetition */
1550
1551 for (i = 0; i < 2; i++)
1552   {
1553   int size = 0;
1554   header_line *h;
1555
1556   for (h = header_list; size < header_insert_maxlen && h; h = h->next)
1557     if (h->type != htype_old && h->text)  /* NULL => Received: placeholder */
1558       if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1559         {
1560         int ilen;
1561         uschar *t;
1562
1563         if (exists_only) return US"1";      /* don't need actual string */
1564         found = TRUE;
1565         t = h->text + len;                  /* text to insert */
1566         if (!want_raw)                      /* unless wanted raw, */
1567           while (isspace(*t)) t++;          /* remove leading white space */
1568         ilen = h->slen - (t - h->text);     /* length to insert */
1569
1570         /* Unless wanted raw, remove trailing whitespace, including the
1571         newline. */
1572
1573         if (!want_raw)
1574           while (ilen > 0 && isspace(t[ilen-1])) ilen--;
1575
1576         /* Set comma = 1 if handling a single header and it's one of those
1577         that contains an address list, except when asked for raw headers. Only
1578         need to do this once. */
1579
1580         if (!want_raw && name && comma == 0 &&
1581             Ustrchr("BCFRST", h->type) != NULL)
1582           comma = 1;
1583
1584         /* First pass - compute total store needed; second pass - compute
1585         total store used, including this header. */
1586
1587         size += ilen + comma + 1;  /* +1 for the newline */
1588
1589         /* Second pass - concatenate the data, up to a maximum. Note that
1590         the loop stops when size hits the limit. */
1591
1592         if (i != 0)
1593           {
1594           if (size > header_insert_maxlen)
1595             {
1596             ilen -= size - header_insert_maxlen - 1;
1597             comma = 0;
1598             }
1599           Ustrncpy(ptr, t, ilen);
1600           ptr += ilen;
1601
1602           /* For a non-raw header, put in the comma if needed, then add
1603           back the newline we removed above, provided there was some text in
1604           the header. */
1605
1606           if (!want_raw && ilen > 0)
1607             {
1608             if (comma != 0) *ptr++ = ',';
1609             *ptr++ = '\n';
1610             }
1611           }
1612         }
1613
1614   /* At end of first pass, return NULL if no header found. Then truncate size
1615   if necessary, and get the buffer to hold the data, returning the buffer size.
1616   */
1617
1618   if (i == 0)
1619     {
1620     if (!found) return NULL;
1621     if (size > header_insert_maxlen) size = header_insert_maxlen;
1622     *newsize = size + 1;
1623     ptr = yield = store_get(*newsize);
1624     }
1625   }
1626
1627 /* That's all we do for raw header expansion. */
1628
1629 if (want_raw)
1630   *ptr = 0;
1631
1632 /* Otherwise, remove a final newline and a redundant added comma. Then we do
1633 RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2()
1634 function can return an error with decoded data if the charset translation
1635 fails. If decoding fails, it returns NULL. */
1636
1637 else
1638   {
1639   uschar *decoded, *error;
1640   if (ptr > yield && ptr[-1] == '\n') ptr--;
1641   if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--;
1642   *ptr = 0;
1643   decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL,
1644     newsize, &error);
1645   if (error != NULL)
1646     {
1647     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1648       "    input was: %s\n", error, yield);
1649     }
1650   if (decoded != NULL) yield = decoded;
1651   }
1652
1653 return yield;
1654 }
1655
1656
1657
1658
1659 /*************************************************
1660 *               Return list of recipients        *
1661 *************************************************/
1662 /* A recipients list is available only during system message filtering,
1663 during ACL processing after DATA, and while expanding pipe commands
1664 generated from a system filter, but not elsewhere. */
1665
1666 static uschar *
1667 fn_recipients(void)
1668 {
1669 if (!enable_dollar_recipients) return NULL; else
1670   {
1671   int size = 128;
1672   int ptr = 0;
1673   int i;
1674   uschar * s = store_get(size);
1675   for (i = 0; i < recipients_count; i++)
1676     {
1677     if (i != 0) s = string_catn(s, &size, &ptr, US", ", 2);
1678     s = string_cat(s, &size, &ptr, recipients_list[i].address);
1679     }
1680   s[ptr] = 0;     /* string_cat() leaves room */
1681   return s;
1682   }
1683 }
1684
1685
1686 /*************************************************
1687 *               Find value of a variable         *
1688 *************************************************/
1689
1690 /* The table of variables is kept in alphabetic order, so we can search it
1691 using a binary chop. The "choplen" variable is nothing to do with the binary
1692 chop.
1693
1694 Arguments:
1695   name          the name of the variable being sought
1696   exists_only   TRUE if this is a def: test; passed on to find_header()
1697   skipping      TRUE => skip any processing evaluation; this is not the same as
1698                   exists_only because def: may test for values that are first
1699                   evaluated here
1700   newsize       pointer to an int which is initially zero; if the answer is in
1701                 a new memory buffer, *newsize is set to its size
1702
1703 Returns:        NULL if the variable does not exist, or
1704                 a pointer to the variable's contents, or
1705                 something non-NULL if exists_only is TRUE
1706 */
1707
1708 static uschar *
1709 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1710 {
1711 var_entry * vp;
1712 uschar *s, *domain;
1713 uschar **ss;
1714 void * val;
1715
1716 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1717 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1718 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1719 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1720 (this gave backwards compatibility at the changeover). There may be built-in
1721 variables whose names start acl_ but they should never start in this way. This
1722 slightly messy specification is a consequence of the history, needless to say.
1723
1724 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1725 set, in which case give an error. */
1726
1727 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1728      !isalpha(name[5]))
1729   {
1730   tree_node *node =
1731     tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4);
1732   return node ? node->data.ptr : strict_acl_vars ? NULL : US"";
1733   }
1734
1735 /* Handle $auth<n> variables. */
1736
1737 if (Ustrncmp(name, "auth", 4) == 0)
1738   {
1739   uschar *endptr;
1740   int n = Ustrtoul(name + 4, &endptr, 10);
1741   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1742     return !auth_vars[n-1] ? US"" : auth_vars[n-1];
1743   }
1744 else if (Ustrncmp(name, "regex", 5) == 0)
1745   {
1746   uschar *endptr;
1747   int n = Ustrtoul(name + 5, &endptr, 10);
1748   if (*endptr == 0 && n != 0 && n <= REGEX_VARS)
1749     return !regex_vars[n-1] ? US"" : regex_vars[n-1];
1750   }
1751
1752 /* For all other variables, search the table */
1753
1754 if (!(vp = find_var_ent(name)))
1755   return NULL;          /* Unknown variable name */
1756
1757 /* Found an existing variable. If in skipping state, the value isn't needed,
1758 and we want to avoid processing (such as looking up the host name). */
1759
1760 if (skipping)
1761   return US"";
1762
1763 val = vp->value;
1764 switch (vp->type)
1765   {
1766   case vtype_filter_int:
1767     if (!filter_running) return NULL;
1768     /* Fall through */
1769     /* VVVVVVVVVVVV */
1770   case vtype_int:
1771     sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1772     return var_buffer;
1773
1774   case vtype_ino:
1775     sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1776     return var_buffer;
1777
1778   case vtype_gid:
1779     sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1780     return var_buffer;
1781
1782   case vtype_uid:
1783     sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1784     return var_buffer;
1785
1786   case vtype_bool:
1787     sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1788     return var_buffer;
1789
1790   case vtype_stringptr:                      /* Pointer to string */
1791     return (s = *((uschar **)(val))) ? s : US"";
1792
1793   case vtype_pid:
1794     sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1795     return var_buffer;
1796
1797   case vtype_load_avg:
1798     sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1799     return var_buffer;
1800
1801   case vtype_host_lookup:                    /* Lookup if not done so */
1802     if (sender_host_name == NULL && sender_host_address != NULL &&
1803         !host_lookup_failed && host_name_lookup() == OK)
1804       host_build_sender_fullhost();
1805     return (sender_host_name == NULL)? US"" : sender_host_name;
1806
1807   case vtype_localpart:                      /* Get local part from address */
1808     s = *((uschar **)(val));
1809     if (s == NULL) return US"";
1810     domain = Ustrrchr(s, '@');
1811     if (domain == NULL) return s;
1812     if (domain - s > sizeof(var_buffer) - 1)
1813       log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1814           " in string expansion", sizeof(var_buffer));
1815     Ustrncpy(var_buffer, s, domain - s);
1816     var_buffer[domain - s] = 0;
1817     return var_buffer;
1818
1819   case vtype_domain:                         /* Get domain from address */
1820     s = *((uschar **)(val));
1821     if (s == NULL) return US"";
1822     domain = Ustrrchr(s, '@');
1823     return (domain == NULL)? US"" : domain + 1;
1824
1825   case vtype_msgheaders:
1826     return find_header(NULL, exists_only, newsize, FALSE, NULL);
1827
1828   case vtype_msgheaders_raw:
1829     return find_header(NULL, exists_only, newsize, TRUE, NULL);
1830
1831   case vtype_msgbody:                        /* Pointer to msgbody string */
1832   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1833     ss = (uschar **)(val);
1834     if (!*ss && deliver_datafile >= 0)  /* Read body when needed */
1835       {
1836       uschar *body;
1837       off_t start_offset = SPOOL_DATA_START_OFFSET;
1838       int len = message_body_visible;
1839       if (len > message_size) len = message_size;
1840       *ss = body = store_malloc(len+1);
1841       body[0] = 0;
1842       if (vp->type == vtype_msgbody_end)
1843         {
1844         struct stat statbuf;
1845         if (fstat(deliver_datafile, &statbuf) == 0)
1846           {
1847           start_offset = statbuf.st_size - len;
1848           if (start_offset < SPOOL_DATA_START_OFFSET)
1849             start_offset = SPOOL_DATA_START_OFFSET;
1850           }
1851         }
1852       if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0)
1853         log_write(0, LOG_MAIN|LOG_PANIC_DIE, "deliver_datafile lseek: %s",
1854           strerror(errno));
1855       len = read(deliver_datafile, body, len);
1856       if (len > 0)
1857         {
1858         body[len] = 0;
1859         if (message_body_newlines)   /* Separate loops for efficiency */
1860           while (len > 0)
1861             { if (body[--len] == 0) body[len] = ' '; }
1862         else
1863           while (len > 0)
1864             { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1865         }
1866       }
1867     return *ss ? *ss : US"";
1868
1869   case vtype_todbsdin:                       /* BSD inbox time of day */
1870     return tod_stamp(tod_bsdin);
1871
1872   case vtype_tode:                           /* Unix epoch time of day */
1873     return tod_stamp(tod_epoch);
1874
1875   case vtype_todel:                          /* Unix epoch/usec time of day */
1876     return tod_stamp(tod_epoch_l);
1877
1878   case vtype_todf:                           /* Full time of day */
1879     return tod_stamp(tod_full);
1880
1881   case vtype_todl:                           /* Log format time of day */
1882     return tod_stamp(tod_log_bare);            /* (without timezone) */
1883
1884   case vtype_todzone:                        /* Time zone offset only */
1885     return tod_stamp(tod_zone);
1886
1887   case vtype_todzulu:                        /* Zulu time */
1888     return tod_stamp(tod_zulu);
1889
1890   case vtype_todlf:                          /* Log file datestamp tod */
1891     return tod_stamp(tod_log_datestamp_daily);
1892
1893   case vtype_reply:                          /* Get reply address */
1894     s = find_header(US"reply-to:", exists_only, newsize, TRUE,
1895       headers_charset);
1896     if (s != NULL) while (isspace(*s)) s++;
1897     if (s == NULL || *s == 0)
1898       {
1899       *newsize = 0;                            /* For the *s==0 case */
1900       s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset);
1901       }
1902     if (s != NULL)
1903       {
1904       uschar *t;
1905       while (isspace(*s)) s++;
1906       for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1907       while (t > s && isspace(t[-1])) t--;
1908       *t = 0;
1909       }
1910     return (s == NULL)? US"" : s;
1911
1912   case vtype_string_func:
1913     {
1914     uschar * (*fn)() = val;
1915     return fn();
1916     }
1917
1918   case vtype_pspace:
1919     {
1920     int inodes;
1921     sprintf(CS var_buffer, "%d",
1922       receive_statvfs(val == (void *)TRUE, &inodes));
1923     }
1924   return var_buffer;
1925
1926   case vtype_pinodes:
1927     {
1928     int inodes;
1929     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1930     sprintf(CS var_buffer, "%d", inodes);
1931     }
1932   return var_buffer;
1933
1934   case vtype_cert:
1935     return *(void **)val ? US"<cert>" : US"";
1936
1937 #ifndef DISABLE_DKIM
1938   case vtype_dkim:
1939     return dkim_exim_expand_query((int)(long)val);
1940 #endif
1941
1942   }
1943
1944 return NULL;  /* Unknown variable. Silences static checkers. */
1945 }
1946
1947
1948
1949
1950 void
1951 modify_variable(uschar *name, void * value)
1952 {
1953 var_entry * vp;
1954 if ((vp = find_var_ent(name))) vp->value = value;
1955 return;          /* Unknown variable name, fail silently */
1956 }
1957
1958
1959
1960
1961
1962
1963 /*************************************************
1964 *           Read and expand substrings           *
1965 *************************************************/
1966
1967 /* This function is called to read and expand argument substrings for various
1968 expansion items. Some have a minimum requirement that is less than the maximum;
1969 in these cases, the first non-present one is set to NULL.
1970
1971 Arguments:
1972   sub        points to vector of pointers to set
1973   n          maximum number of substrings
1974   m          minimum required
1975   sptr       points to current string pointer
1976   skipping   the skipping flag
1977   check_end  if TRUE, check for final '}'
1978   name       name of item, for error message
1979   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
1980              the store.
1981
1982 Returns:     0 OK; string pointer updated
1983              1 curly bracketing error (too few arguments)
1984              2 too many arguments (only if check_end is set); message set
1985              3 other error (expansion failure)
1986 */
1987
1988 static int
1989 read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping,
1990   BOOL check_end, uschar *name, BOOL *resetok)
1991 {
1992 int i;
1993 const uschar *s = *sptr;
1994
1995 while (isspace(*s)) s++;
1996 for (i = 0; i < n; i++)
1997   {
1998   if (*s != '{')
1999     {
2000     if (i < m)
2001       {
2002       expand_string_message = string_sprintf("Not enough arguments for '%s' "
2003         "(min is %d)", name, m);
2004       return 1;
2005       }
2006     sub[i] = NULL;
2007     break;
2008     }
2009   if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok)))
2010     return 3;
2011   if (*s++ != '}') return 1;
2012   while (isspace(*s)) s++;
2013   }
2014 if (check_end && *s++ != '}')
2015   {
2016   if (s[-1] == '{')
2017     {
2018     expand_string_message = string_sprintf("Too many arguments for '%s' "
2019       "(max is %d)", name, n);
2020     return 2;
2021     }
2022   expand_string_message = string_sprintf("missing '}' after '%s'", name);
2023   return 1;
2024   }
2025
2026 *sptr = s;
2027 return 0;
2028 }
2029
2030
2031
2032
2033 /*************************************************
2034 *     Elaborate message for bad variable         *
2035 *************************************************/
2036
2037 /* For the "unknown variable" message, take a look at the variable's name, and
2038 give additional information about possible ACL variables. The extra information
2039 is added on to expand_string_message.
2040
2041 Argument:   the name of the variable
2042 Returns:    nothing
2043 */
2044
2045 static void
2046 check_variable_error_message(uschar *name)
2047 {
2048 if (Ustrncmp(name, "acl_", 4) == 0)
2049   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
2050     (name[4] == 'c' || name[4] == 'm')?
2051       (isalpha(name[5])?
2052         US"6th character of a user-defined ACL variable must be a digit or underscore" :
2053         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
2054       ) :
2055       US"user-defined ACL variables must start acl_c or acl_m");
2056 }
2057
2058
2059
2060 /*
2061 Load args from sub array to globals, and call acl_check().
2062 Sub array will be corrupted on return.
2063
2064 Returns:       OK         access is granted by an ACCEPT verb
2065                DISCARD    access is (apparently) granted by a DISCARD verb
2066                FAIL       access is denied
2067                FAIL_DROP  access is denied; drop the connection
2068                DEFER      can't tell at the moment
2069                ERROR      disaster
2070 */
2071 static int
2072 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
2073 {
2074 int i;
2075 int sav_narg = acl_narg;
2076 int ret;
2077 uschar * dummy_logmsg;
2078 extern int acl_where;
2079
2080 if(--nsub > nelem(acl_arg)) nsub = nelem(acl_arg);
2081 for (i = 0; i < nsub && sub[i+1]; i++)
2082   {
2083   uschar * tmp = acl_arg[i];
2084   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2085   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2086   }
2087 acl_narg = i;
2088 while (i < nsub)
2089   {
2090   sub[i+1] = acl_arg[i];
2091   acl_arg[i++] = NULL;
2092   }
2093
2094 DEBUG(D_expand)
2095   debug_printf_indent("expanding: acl: %s  arg: %s%s\n",
2096     sub[0],
2097     acl_narg>0 ? acl_arg[0] : US"<none>",
2098     acl_narg>1 ? " +more"   : "");
2099
2100 ret = acl_eval(acl_where, sub[0], user_msgp, &dummy_logmsg);
2101
2102 for (i = 0; i < nsub; i++)
2103   acl_arg[i] = sub[i+1];        /* restore old args */
2104 acl_narg = sav_narg;
2105
2106 return ret;
2107 }
2108
2109
2110
2111
2112 /*************************************************
2113 *        Read and evaluate a condition           *
2114 *************************************************/
2115
2116 /*
2117 Arguments:
2118   s        points to the start of the condition text
2119   resetok  points to a BOOL which is written false if it is unsafe to
2120            free memory. Certain condition types (acl) may have side-effect
2121            allocation which must be preserved.
2122   yield    points to a BOOL to hold the result of the condition test;
2123            if NULL, we are just reading through a condition that is
2124            part of an "or" combination to check syntax, or in a state
2125            where the answer isn't required
2126
2127 Returns:   a pointer to the first character after the condition, or
2128            NULL after an error
2129 */
2130
2131 static const uschar *
2132 eval_condition(const uschar *s, BOOL *resetok, BOOL *yield)
2133 {
2134 BOOL testfor = TRUE;
2135 BOOL tempcond, combined_cond;
2136 BOOL *subcondptr;
2137 BOOL sub2_honour_dollar = TRUE;
2138 int i, rc, cond_type, roffset;
2139 int_eximarith_t num[2];
2140 struct stat statbuf;
2141 uschar name[256];
2142 const uschar *sub[10];
2143
2144 const pcre *re;
2145 const uschar *rerror;
2146
2147 for (;;)
2148   {
2149   while (isspace(*s)) s++;
2150   if (*s == '!') { testfor = !testfor; s++; } else break;
2151   }
2152
2153 /* Numeric comparisons are symbolic */
2154
2155 if (*s == '=' || *s == '>' || *s == '<')
2156   {
2157   int p = 0;
2158   name[p++] = *s++;
2159   if (*s == '=')
2160     {
2161     name[p++] = '=';
2162     s++;
2163     }
2164   name[p] = 0;
2165   }
2166
2167 /* All other conditions are named */
2168
2169 else s = read_name(name, 256, s, US"_");
2170
2171 /* If we haven't read a name, it means some non-alpha character is first. */
2172
2173 if (name[0] == 0)
2174   {
2175   expand_string_message = string_sprintf("condition name expected, "
2176     "but found \"%.16s\"", s);
2177   return NULL;
2178   }
2179
2180 /* Find which condition we are dealing with, and switch on it */
2181
2182 cond_type = chop_match(name, cond_table, nelem(cond_table));
2183 switch(cond_type)
2184   {
2185   /* def: tests for a non-empty variable, or for the existence of a header. If
2186   yield == NULL we are in a skipping state, and don't care about the answer. */
2187
2188   case ECOND_DEF:
2189   if (*s != ':')
2190     {
2191     expand_string_message = US"\":\" expected after \"def\"";
2192     return NULL;
2193     }
2194
2195   s = read_name(name, 256, s+1, US"_");
2196
2197   /* Test for a header's existence. If the name contains a closing brace
2198   character, this may be a user error where the terminating colon has been
2199   omitted. Set a flag to adjust a subsequent error message in this case. */
2200
2201   if (Ustrncmp(name, "h_", 2) == 0 ||
2202       Ustrncmp(name, "rh_", 3) == 0 ||
2203       Ustrncmp(name, "bh_", 3) == 0 ||
2204       Ustrncmp(name, "header_", 7) == 0 ||
2205       Ustrncmp(name, "rheader_", 8) == 0 ||
2206       Ustrncmp(name, "bheader_", 8) == 0)
2207     {
2208     s = read_header_name(name, 256, s);
2209     /* {-for-text-editors */
2210     if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2211     if (yield != NULL) *yield =
2212       (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor;
2213     }
2214
2215   /* Test for a variable's having a non-empty value. A non-existent variable
2216   causes an expansion failure. */
2217
2218   else
2219     {
2220     uschar *value = find_variable(name, TRUE, yield == NULL, NULL);
2221     if (value == NULL)
2222       {
2223       expand_string_message = (name[0] == 0)?
2224         string_sprintf("variable name omitted after \"def:\"") :
2225         string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2226       check_variable_error_message(name);
2227       return NULL;
2228       }
2229     if (yield != NULL) *yield = (value[0] != 0) == testfor;
2230     }
2231
2232   return s;
2233
2234
2235   /* first_delivery tests for first delivery attempt */
2236
2237   case ECOND_FIRST_DELIVERY:
2238   if (yield != NULL) *yield = deliver_firsttime == testfor;
2239   return s;
2240
2241
2242   /* queue_running tests for any process started by a queue runner */
2243
2244   case ECOND_QUEUE_RUNNING:
2245   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2246   return s;
2247
2248
2249   /* exists:  tests for file existence
2250        isip:  tests for any IP address
2251       isip4:  tests for an IPv4 address
2252       isip6:  tests for an IPv6 address
2253         pam:  does PAM authentication
2254      radius:  does RADIUS authentication
2255    ldapauth:  does LDAP authentication
2256     pwcheck:  does Cyrus SASL pwcheck authentication
2257   */
2258
2259   case ECOND_EXISTS:
2260   case ECOND_ISIP:
2261   case ECOND_ISIP4:
2262   case ECOND_ISIP6:
2263   case ECOND_PAM:
2264   case ECOND_RADIUS:
2265   case ECOND_LDAPAUTH:
2266   case ECOND_PWCHECK:
2267
2268   while (isspace(*s)) s++;
2269   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2270
2271   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2272   if (sub[0] == NULL) return NULL;
2273   /* {-for-text-editors */
2274   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2275
2276   if (yield == NULL) return s;   /* No need to run the test if skipping */
2277
2278   switch(cond_type)
2279     {
2280     case ECOND_EXISTS:
2281     if ((expand_forbid & RDO_EXISTS) != 0)
2282       {
2283       expand_string_message = US"File existence tests are not permitted";
2284       return NULL;
2285       }
2286     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2287     break;
2288
2289     case ECOND_ISIP:
2290     case ECOND_ISIP4:
2291     case ECOND_ISIP6:
2292     rc = string_is_ip_address(sub[0], NULL);
2293     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2294              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2295     break;
2296
2297     /* Various authentication tests - all optionally compiled */
2298
2299     case ECOND_PAM:
2300     #ifdef SUPPORT_PAM
2301     rc = auth_call_pam(sub[0], &expand_string_message);
2302     goto END_AUTH;
2303     #else
2304     goto COND_FAILED_NOT_COMPILED;
2305     #endif  /* SUPPORT_PAM */
2306
2307     case ECOND_RADIUS:
2308     #ifdef RADIUS_CONFIG_FILE
2309     rc = auth_call_radius(sub[0], &expand_string_message);
2310     goto END_AUTH;
2311     #else
2312     goto COND_FAILED_NOT_COMPILED;
2313     #endif  /* RADIUS_CONFIG_FILE */
2314
2315     case ECOND_LDAPAUTH:
2316     #ifdef LOOKUP_LDAP
2317       {
2318       /* Just to keep the interface the same */
2319       BOOL do_cache;
2320       int old_pool = store_pool;
2321       store_pool = POOL_SEARCH;
2322       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2323         &expand_string_message, &do_cache);
2324       store_pool = old_pool;
2325       }
2326     goto END_AUTH;
2327     #else
2328     goto COND_FAILED_NOT_COMPILED;
2329     #endif  /* LOOKUP_LDAP */
2330
2331     case ECOND_PWCHECK:
2332     #ifdef CYRUS_PWCHECK_SOCKET
2333     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2334     goto END_AUTH;
2335     #else
2336     goto COND_FAILED_NOT_COMPILED;
2337     #endif  /* CYRUS_PWCHECK_SOCKET */
2338
2339     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2340         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2341     END_AUTH:
2342     if (rc == ERROR || rc == DEFER) return NULL;
2343     *yield = (rc == OK) == testfor;
2344     #endif
2345     }
2346   return s;
2347
2348
2349   /* call ACL (in a conditional context).  Accept true, deny false.
2350   Defer is a forced-fail.  Anything set by message= goes to $value.
2351   Up to ten parameters are used; we use the braces round the name+args
2352   like the saslauthd condition does, to permit a variable number of args.
2353   See also the expansion-item version EITEM_ACL and the traditional
2354   acl modifier ACLC_ACL.
2355   Since the ACL may allocate new global variables, tell our caller to not
2356   reclaim memory.
2357   */
2358
2359   case ECOND_ACL:
2360     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2361     {
2362     uschar *sub[10];
2363     uschar *user_msg;
2364     BOOL cond = FALSE;
2365     int size = 0;
2366     int ptr = 0;
2367
2368     while (isspace(*s)) s++;
2369     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2370
2371     switch(read_subs(sub, nelem(sub), 1,
2372       &s, yield == NULL, TRUE, US"acl", resetok))
2373       {
2374       case 1: expand_string_message = US"too few arguments or bracketing "
2375         "error for acl";
2376       case 2:
2377       case 3: return NULL;
2378       }
2379
2380     *resetok = FALSE;   /* eval_acl() might allocate; do not reclaim */
2381     if (yield != NULL) switch(eval_acl(sub, nelem(sub), &user_msg))
2382         {
2383         case OK:
2384           cond = TRUE;
2385         case FAIL:
2386           lookup_value = NULL;
2387           if (user_msg)
2388             {
2389             lookup_value = string_cat(NULL, &size, &ptr, user_msg);
2390             lookup_value[ptr] = '\0';
2391             }
2392           *yield = cond == testfor;
2393           break;
2394
2395         case DEFER:
2396           expand_string_forcedfail = TRUE;
2397           /*FALLTHROUGH*/
2398         default:
2399           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
2400           return NULL;
2401         }
2402     return s;
2403     }
2404
2405
2406   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2407
2408      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2409
2410   However, the last two are optional. That is why the whole set is enclosed
2411   in their own set of braces. */
2412
2413   case ECOND_SASLAUTHD:
2414 #ifndef CYRUS_SASLAUTHD_SOCKET
2415     goto COND_FAILED_NOT_COMPILED;
2416 #else
2417     {
2418     uschar *sub[4];
2419     while (isspace(*s)) s++;
2420     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2421     switch(read_subs(sub, nelem(sub), 2, &s, yield == NULL, TRUE, US"saslauthd",
2422                     resetok))
2423       {
2424       case 1: expand_string_message = US"too few arguments or bracketing "
2425         "error for saslauthd";
2426       case 2:
2427       case 3: return NULL;
2428       }
2429     if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2430     if (yield != NULL)
2431       {
2432       int rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2433         &expand_string_message);
2434       if (rc == ERROR || rc == DEFER) return NULL;
2435       *yield = (rc == OK) == testfor;
2436       }
2437     return s;
2438     }
2439 #endif /* CYRUS_SASLAUTHD_SOCKET */
2440
2441
2442   /* symbolic operators for numeric and string comparison, and a number of
2443   other operators, all requiring two arguments.
2444
2445   crypteq:           encrypts plaintext and compares against an encrypted text,
2446                        using crypt(), crypt16(), MD5 or SHA-1
2447   inlist/inlisti:    checks if first argument is in the list of the second
2448   match:             does a regular expression match and sets up the numerical
2449                        variables if it succeeds
2450   match_address:     matches in an address list
2451   match_domain:      matches in a domain list
2452   match_ip:          matches a host list that is restricted to IP addresses
2453   match_local_part:  matches in a local part list
2454   */
2455
2456   case ECOND_MATCH_ADDRESS:
2457   case ECOND_MATCH_DOMAIN:
2458   case ECOND_MATCH_IP:
2459   case ECOND_MATCH_LOCAL_PART:
2460 #ifndef EXPAND_LISTMATCH_RHS
2461     sub2_honour_dollar = FALSE;
2462 #endif
2463     /* FALLTHROUGH */
2464
2465   case ECOND_CRYPTEQ:
2466   case ECOND_INLIST:
2467   case ECOND_INLISTI:
2468   case ECOND_MATCH:
2469
2470   case ECOND_NUM_L:     /* Numerical comparisons */
2471   case ECOND_NUM_LE:
2472   case ECOND_NUM_E:
2473   case ECOND_NUM_EE:
2474   case ECOND_NUM_G:
2475   case ECOND_NUM_GE:
2476
2477   case ECOND_STR_LT:    /* String comparisons */
2478   case ECOND_STR_LTI:
2479   case ECOND_STR_LE:
2480   case ECOND_STR_LEI:
2481   case ECOND_STR_EQ:
2482   case ECOND_STR_EQI:
2483   case ECOND_STR_GT:
2484   case ECOND_STR_GTI:
2485   case ECOND_STR_GE:
2486   case ECOND_STR_GEI:
2487
2488   for (i = 0; i < 2; i++)
2489     {
2490     /* Sometimes, we don't expand substrings; too many insecure configurations
2491     created using match_address{}{} and friends, where the second param
2492     includes information from untrustworthy sources. */
2493     BOOL honour_dollar = TRUE;
2494     if ((i > 0) && !sub2_honour_dollar)
2495       honour_dollar = FALSE;
2496
2497     while (isspace(*s)) s++;
2498     if (*s != '{')
2499       {
2500       if (i == 0) goto COND_FAILED_CURLY_START;
2501       expand_string_message = string_sprintf("missing 2nd string in {} "
2502         "after \"%s\"", name);
2503       return NULL;
2504       }
2505     sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2506         honour_dollar, resetok);
2507     if (sub[i] == NULL) return NULL;
2508     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2509
2510     /* Convert to numerical if required; we know that the names of all the
2511     conditions that compare numbers do not start with a letter. This just saves
2512     checking for them individually. */
2513
2514     if (!isalpha(name[0]) && yield != NULL)
2515       if (sub[i][0] == 0)
2516         {
2517         num[i] = 0;
2518         DEBUG(D_expand)
2519           debug_printf_indent("empty string cast to zero for numerical comparison\n");
2520         }
2521       else
2522         {
2523         num[i] = expanded_string_integer(sub[i], FALSE);
2524         if (expand_string_message != NULL) return NULL;
2525         }
2526     }
2527
2528   /* Result not required */
2529
2530   if (yield == NULL) return s;
2531
2532   /* Do an appropriate comparison */
2533
2534   switch(cond_type)
2535     {
2536     case ECOND_NUM_E:
2537     case ECOND_NUM_EE:
2538     tempcond = (num[0] == num[1]);
2539     break;
2540
2541     case ECOND_NUM_G:
2542     tempcond = (num[0] > num[1]);
2543     break;
2544
2545     case ECOND_NUM_GE:
2546     tempcond = (num[0] >= num[1]);
2547     break;
2548
2549     case ECOND_NUM_L:
2550     tempcond = (num[0] < num[1]);
2551     break;
2552
2553     case ECOND_NUM_LE:
2554     tempcond = (num[0] <= num[1]);
2555     break;
2556
2557     case ECOND_STR_LT:
2558     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2559     break;
2560
2561     case ECOND_STR_LTI:
2562     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2563     break;
2564
2565     case ECOND_STR_LE:
2566     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2567     break;
2568
2569     case ECOND_STR_LEI:
2570     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2571     break;
2572
2573     case ECOND_STR_EQ:
2574     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2575     break;
2576
2577     case ECOND_STR_EQI:
2578     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2579     break;
2580
2581     case ECOND_STR_GT:
2582     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2583     break;
2584
2585     case ECOND_STR_GTI:
2586     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2587     break;
2588
2589     case ECOND_STR_GE:
2590     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2591     break;
2592
2593     case ECOND_STR_GEI:
2594     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2595     break;
2596
2597     case ECOND_MATCH:   /* Regular expression match */
2598     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2599       NULL);
2600     if (re == NULL)
2601       {
2602       expand_string_message = string_sprintf("regular expression error in "
2603         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2604       return NULL;
2605       }
2606     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2607     break;
2608
2609     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2610     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2611     goto MATCHED_SOMETHING;
2612
2613     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2614     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2615       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2616     goto MATCHED_SOMETHING;
2617
2618     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2619     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2620       {
2621       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2622         sub[0]);
2623       return NULL;
2624       }
2625     else
2626       {
2627       unsigned int *nullcache = NULL;
2628       check_host_block cb;
2629
2630       cb.host_name = US"";
2631       cb.host_address = sub[0];
2632
2633       /* If the host address starts off ::ffff: it is an IPv6 address in
2634       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2635       addresses. */
2636
2637       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2638         cb.host_address + 7 : cb.host_address;
2639
2640       rc = match_check_list(
2641              &sub[1],                   /* the list */
2642              0,                         /* separator character */
2643              &hostlist_anchor,          /* anchor pointer */
2644              &nullcache,                /* cache pointer */
2645              check_host,                /* function for testing */
2646              &cb,                       /* argument for function */
2647              MCL_HOST,                  /* type of check */
2648              sub[0],                    /* text for debugging */
2649              NULL);                     /* where to pass back data */
2650       }
2651     goto MATCHED_SOMETHING;
2652
2653     case ECOND_MATCH_LOCAL_PART:
2654     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2655       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2656     /* Fall through */
2657     /* VVVVVVVVVVVV */
2658     MATCHED_SOMETHING:
2659     switch(rc)
2660       {
2661       case OK:
2662       tempcond = TRUE;
2663       break;
2664
2665       case FAIL:
2666       tempcond = FALSE;
2667       break;
2668
2669       case DEFER:
2670       expand_string_message = string_sprintf("unable to complete match "
2671         "against \"%s\": %s", sub[1], search_error_message);
2672       return NULL;
2673       }
2674
2675     break;
2676
2677     /* Various "encrypted" comparisons. If the second string starts with
2678     "{" then an encryption type is given. Default to crypt() or crypt16()
2679     (build-time choice). */
2680     /* }-for-text-editors */
2681
2682     case ECOND_CRYPTEQ:
2683     #ifndef SUPPORT_CRYPTEQ
2684     goto COND_FAILED_NOT_COMPILED;
2685     #else
2686     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2687       {
2688       int sublen = Ustrlen(sub[1]+5);
2689       md5 base;
2690       uschar digest[16];
2691
2692       md5_start(&base);
2693       md5_end(&base, sub[0], Ustrlen(sub[0]), digest);
2694
2695       /* If the length that we are comparing against is 24, the MD5 digest
2696       is expressed as a base64 string. This is the way LDAP does it. However,
2697       some other software uses a straightforward hex representation. We assume
2698       this if the length is 32. Other lengths fail. */
2699
2700       if (sublen == 24)
2701         {
2702         uschar *coded = b64encode(digest, 16);
2703         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2704           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2705         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2706         }
2707       else if (sublen == 32)
2708         {
2709         int i;
2710         uschar coded[36];
2711         for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2712         coded[32] = 0;
2713         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2714           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2715         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2716         }
2717       else
2718         {
2719         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2720           "fail\n  crypted=%s\n", sub[1]+5);
2721         tempcond = FALSE;
2722         }
2723       }
2724
2725     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2726       {
2727       int sublen = Ustrlen(sub[1]+6);
2728       hctx h;
2729       uschar digest[20];
2730
2731       sha1_start(&h);
2732       sha1_end(&h, sub[0], Ustrlen(sub[0]), digest);
2733
2734       /* If the length that we are comparing against is 28, assume the SHA1
2735       digest is expressed as a base64 string. If the length is 40, assume a
2736       straightforward hex representation. Other lengths fail. */
2737
2738       if (sublen == 28)
2739         {
2740         uschar *coded = b64encode(digest, 20);
2741         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2742           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2743         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2744         }
2745       else if (sublen == 40)
2746         {
2747         int i;
2748         uschar coded[44];
2749         for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2750         coded[40] = 0;
2751         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2752           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2753         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2754         }
2755       else
2756         {
2757         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2758           "fail\n  crypted=%s\n", sub[1]+6);
2759         tempcond = FALSE;
2760         }
2761       }
2762
2763     else   /* {crypt} or {crypt16} and non-{ at start */
2764            /* }-for-text-editors */
2765       {
2766       int which = 0;
2767       uschar *coded;
2768
2769       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2770         {
2771         sub[1] += 7;
2772         which = 1;
2773         }
2774       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2775         {
2776         sub[1] += 9;
2777         which = 2;
2778         }
2779       else if (sub[1][0] == '{')                /* }-for-text-editors */
2780         {
2781         expand_string_message = string_sprintf("unknown encryption mechanism "
2782           "in \"%s\"", sub[1]);
2783         return NULL;
2784         }
2785
2786       switch(which)
2787         {
2788         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2789         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2790         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2791         }
2792
2793       #define STR(s) # s
2794       #define XSTR(s) STR(s)
2795       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2796         "  subject=%s\n  crypted=%s\n",
2797         which == 0 ? XSTR(DEFAULT_CRYPT) : which == 1 ? "crypt" : "crypt16",
2798         coded, sub[1]);
2799       #undef STR
2800       #undef XSTR
2801
2802       /* If the encrypted string contains fewer than two characters (for the
2803       salt), force failure. Otherwise we get false positives: with an empty
2804       string the yield of crypt() is an empty string! */
2805
2806       if (coded)
2807         tempcond = Ustrlen(sub[1]) < 2 ? FALSE : Ustrcmp(coded, sub[1]) == 0;
2808       else if (errno == EINVAL)
2809         tempcond = FALSE;
2810       else
2811         {
2812         expand_string_message = string_sprintf("crypt error: %s\n",
2813           US strerror(errno));
2814         return NULL;
2815         }
2816       }
2817     break;
2818     #endif  /* SUPPORT_CRYPTEQ */
2819
2820     case ECOND_INLIST:
2821     case ECOND_INLISTI:
2822       {
2823       const uschar * list = sub[1];
2824       int sep = 0;
2825       uschar *save_iterate_item = iterate_item;
2826       int (*compare)(const uschar *, const uschar *);
2827
2828       DEBUG(D_expand) debug_printf_indent("condition: %s\n", name);
2829
2830       tempcond = FALSE;
2831       compare = cond_type == ECOND_INLISTI
2832         ? strcmpic : (int (*)(const uschar *, const uschar *)) strcmp;
2833
2834       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
2835         if (compare(sub[0], iterate_item) == 0)
2836           {
2837           tempcond = TRUE;
2838           break;
2839           }
2840       iterate_item = save_iterate_item;
2841       }
2842
2843     }   /* Switch for comparison conditions */
2844
2845   *yield = tempcond == testfor;
2846   return s;    /* End of comparison conditions */
2847
2848
2849   /* and/or: computes logical and/or of several conditions */
2850
2851   case ECOND_AND:
2852   case ECOND_OR:
2853   subcondptr = (yield == NULL)? NULL : &tempcond;
2854   combined_cond = (cond_type == ECOND_AND);
2855
2856   while (isspace(*s)) s++;
2857   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2858
2859   for (;;)
2860     {
2861     while (isspace(*s)) s++;
2862     /* {-for-text-editors */
2863     if (*s == '}') break;
2864     if (*s != '{')                                      /* }-for-text-editors */
2865       {
2866       expand_string_message = string_sprintf("each subcondition "
2867         "inside an \"%s{...}\" condition must be in its own {}", name);
2868       return NULL;
2869       }
2870
2871     if (!(s = eval_condition(s+1, resetok, subcondptr)))
2872       {
2873       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
2874         expand_string_message, name);
2875       return NULL;
2876       }
2877     while (isspace(*s)) s++;
2878
2879     /* {-for-text-editors */
2880     if (*s++ != '}')
2881       {
2882       /* {-for-text-editors */
2883       expand_string_message = string_sprintf("missing } at end of condition "
2884         "inside \"%s\" group", name);
2885       return NULL;
2886       }
2887
2888     if (yield != NULL)
2889       {
2890       if (cond_type == ECOND_AND)
2891         {
2892         combined_cond &= tempcond;
2893         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
2894         }                                       /* evaluate any more */
2895       else
2896         {
2897         combined_cond |= tempcond;
2898         if (combined_cond) subcondptr = NULL;   /* once true, don't */
2899         }                                       /* evaluate any more */
2900       }
2901     }
2902
2903   if (yield != NULL) *yield = (combined_cond == testfor);
2904   return ++s;
2905
2906
2907   /* forall/forany: iterates a condition with different values */
2908
2909   case ECOND_FORALL:
2910   case ECOND_FORANY:
2911     {
2912     const uschar * list;
2913     int sep = 0;
2914     uschar *save_iterate_item = iterate_item;
2915
2916     DEBUG(D_expand) debug_printf_indent("condition: %s\n", name);
2917
2918     while (isspace(*s)) s++;
2919     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2920     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
2921     if (sub[0] == NULL) return NULL;
2922     /* {-for-text-editors */
2923     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2924
2925     while (isspace(*s)) s++;
2926     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2927
2928     sub[1] = s;
2929
2930     /* Call eval_condition once, with result discarded (as if scanning a
2931     "false" part). This allows us to find the end of the condition, because if
2932     the list it empty, we won't actually evaluate the condition for real. */
2933
2934     if (!(s = eval_condition(sub[1], resetok, NULL)))
2935       {
2936       expand_string_message = string_sprintf("%s inside \"%s\" condition",
2937         expand_string_message, name);
2938       return NULL;
2939       }
2940     while (isspace(*s)) s++;
2941
2942     /* {-for-text-editors */
2943     if (*s++ != '}')
2944       {
2945       /* {-for-text-editors */
2946       expand_string_message = string_sprintf("missing } at end of condition "
2947         "inside \"%s\"", name);
2948       return NULL;
2949       }
2950
2951     if (yield != NULL) *yield = !testfor;
2952     list = sub[0];
2953     while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
2954       {
2955       DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, iterate_item);
2956       if (!eval_condition(sub[1], resetok, &tempcond))
2957         {
2958         expand_string_message = string_sprintf("%s inside \"%s\" condition",
2959           expand_string_message, name);
2960         iterate_item = save_iterate_item;
2961         return NULL;
2962         }
2963       DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", name,
2964         tempcond? "true":"false");
2965
2966       if (yield != NULL) *yield = (tempcond == testfor);
2967       if (tempcond == (cond_type == ECOND_FORANY)) break;
2968       }
2969
2970     iterate_item = save_iterate_item;
2971     return s;
2972     }
2973
2974
2975   /* The bool{} expansion condition maps a string to boolean.
2976   The values supported should match those supported by the ACL condition
2977   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
2978   of true/false.  Note that Router "condition" rules have a different
2979   interpretation, where general data can be used and only a few values
2980   map to FALSE.
2981   Note that readconf.c boolean matching, for boolean configuration options,
2982   only matches true/yes/false/no.
2983   The bool_lax{} condition matches the Router logic, which is much more
2984   liberal. */
2985   case ECOND_BOOL:
2986   case ECOND_BOOL_LAX:
2987     {
2988     uschar *sub_arg[1];
2989     uschar *t, *t2;
2990     uschar *ourname;
2991     size_t len;
2992     BOOL boolvalue = FALSE;
2993     while (isspace(*s)) s++;
2994     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2995     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
2996     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
2997       {
2998       case 1: expand_string_message = string_sprintf(
2999                   "too few arguments or bracketing error for %s",
3000                   ourname);
3001       /*FALLTHROUGH*/
3002       case 2:
3003       case 3: return NULL;
3004       }
3005     t = sub_arg[0];
3006     while (isspace(*t)) t++;
3007     len = Ustrlen(t);
3008     if (len)
3009       {
3010       /* trailing whitespace: seems like a good idea to ignore it too */
3011       t2 = t + len - 1;
3012       while (isspace(*t2)) t2--;
3013       if (t2 != (t + len))
3014         {
3015         *++t2 = '\0';
3016         len = t2 - t;
3017         }
3018       }
3019     DEBUG(D_expand)
3020       debug_printf_indent("considering %s: %s\n", ourname, len ? t : US"<empty>");
3021     /* logic for the lax case from expand_check_condition(), which also does
3022     expands, and the logic is both short and stable enough that there should
3023     be no maintenance burden from replicating it. */
3024     if (len == 0)
3025       boolvalue = FALSE;
3026     else if (*t == '-'
3027              ? Ustrspn(t+1, "0123456789") == len-1
3028              : Ustrspn(t,   "0123456789") == len)
3029       {
3030       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
3031       /* expand_check_condition only does a literal string "0" check */
3032       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
3033         boolvalue = TRUE;
3034       }
3035     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
3036       boolvalue = TRUE;
3037     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
3038       boolvalue = FALSE;
3039     else if (cond_type == ECOND_BOOL_LAX)
3040       boolvalue = TRUE;
3041     else
3042       {
3043       expand_string_message = string_sprintf("unrecognised boolean "
3044        "value \"%s\"", t);
3045       return NULL;
3046       }
3047     DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", ourname,
3048         boolvalue? "true":"false");
3049     if (yield != NULL) *yield = (boolvalue == testfor);
3050     return s;
3051     }
3052
3053   /* Unknown condition */
3054
3055   default:
3056   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
3057   return NULL;
3058   }   /* End switch on condition type */
3059
3060 /* Missing braces at start and end of data */
3061
3062 COND_FAILED_CURLY_START:
3063 expand_string_message = string_sprintf("missing { after \"%s\"", name);
3064 return NULL;
3065
3066 COND_FAILED_CURLY_END:
3067 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
3068   name);
3069 return NULL;
3070
3071 /* A condition requires code that is not compiled */
3072
3073 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
3074     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
3075     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
3076 COND_FAILED_NOT_COMPILED:
3077 expand_string_message = string_sprintf("support for \"%s\" not compiled",
3078   name);
3079 return NULL;
3080 #endif
3081 }
3082
3083
3084
3085
3086 /*************************************************
3087 *          Save numerical variables              *
3088 *************************************************/
3089
3090 /* This function is called from items such as "if" that want to preserve and
3091 restore the numbered variables.
3092
3093 Arguments:
3094   save_expand_string    points to an array of pointers to set
3095   save_expand_nlength   points to an array of ints for the lengths
3096
3097 Returns:                the value of expand max to save
3098 */
3099
3100 static int
3101 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3102 {
3103 int i;
3104 for (i = 0; i <= expand_nmax; i++)
3105   {
3106   save_expand_nstring[i] = expand_nstring[i];
3107   save_expand_nlength[i] = expand_nlength[i];
3108   }
3109 return expand_nmax;
3110 }
3111
3112
3113
3114 /*************************************************
3115 *           Restore numerical variables          *
3116 *************************************************/
3117
3118 /* This function restored saved values of numerical strings.
3119
3120 Arguments:
3121   save_expand_nmax      the number of strings to restore
3122   save_expand_string    points to an array of pointers
3123   save_expand_nlength   points to an array of ints
3124
3125 Returns:                nothing
3126 */
3127
3128 static void
3129 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3130   int *save_expand_nlength)
3131 {
3132 int i;
3133 expand_nmax = save_expand_nmax;
3134 for (i = 0; i <= expand_nmax; i++)
3135   {
3136   expand_nstring[i] = save_expand_nstring[i];
3137   expand_nlength[i] = save_expand_nlength[i];
3138   }
3139 }
3140
3141
3142
3143
3144
3145 /*************************************************
3146 *            Handle yes/no substrings            *
3147 *************************************************/
3148
3149 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3150 alternative substrings that depend on whether or not the condition was true,
3151 or the lookup or extraction succeeded. The substrings always have to be
3152 expanded, to check their syntax, but "skipping" is set when the result is not
3153 needed - this avoids unnecessary nested lookups.
3154
3155 Arguments:
3156   skipping       TRUE if we were skipping when this item was reached
3157   yes            TRUE if the first string is to be used, else use the second
3158   save_lookup    a value to put back into lookup_value before the 2nd expansion
3159   sptr           points to the input string pointer
3160   yieldptr       points to the output string pointer
3161   sizeptr        points to the output string size
3162   ptrptr         points to the output string pointer
3163   type           "lookup", "if", "extract", "run", "env", "listextract" or
3164                  "certextract" for error message
3165   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3166                 the store.
3167
3168 Returns:         0 OK; lookup_value has been reset to save_lookup
3169                  1 expansion failed
3170                  2 expansion failed because of bracketing error
3171 */
3172
3173 static int
3174 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr,
3175   uschar **yieldptr, int *sizeptr, int *ptrptr, uschar *type, BOOL *resetok)
3176 {
3177 int rc = 0;
3178 const uschar *s = *sptr;    /* Local value */
3179 uschar *sub1, *sub2;
3180 const uschar * errwhere;
3181
3182 /* If there are no following strings, we substitute the contents of $value for
3183 lookups and for extractions in the success case. For the ${if item, the string
3184 "true" is substituted. In the fail case, nothing is substituted for all three
3185 items. */
3186
3187 while (isspace(*s)) s++;
3188 if (*s == '}')
3189   {
3190   if (type[0] == 'i')
3191     {
3192     if (yes && !skipping)
3193       *yieldptr = string_catn(*yieldptr, sizeptr, ptrptr, US"true", 4);
3194     }
3195   else
3196     {
3197     if (yes && lookup_value && !skipping)
3198       *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value);
3199     lookup_value = save_lookup;
3200     }
3201   s++;
3202   goto RETURN;
3203   }
3204
3205 /* The first following string must be braced. */
3206
3207 if (*s++ != '{')
3208   {
3209   errwhere = US"'yes' part did not start with '{'";
3210   goto FAILED_CURLY;
3211   }
3212
3213 /* Expand the first substring. Forced failures are noticed only if we actually
3214 want this string. Set skipping in the call in the fail case (this will always
3215 be the case if we were already skipping). */
3216
3217 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3218 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
3219 expand_string_forcedfail = FALSE;
3220 if (*s++ != '}')
3221   {
3222   errwhere = US"'yes' part did not end with '}'";
3223   goto FAILED_CURLY;
3224   }
3225
3226 /* If we want the first string, add it to the output */
3227
3228 if (yes)
3229   *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub1);
3230
3231 /* If this is called from a lookup/env or a (cert)extract, we want to restore
3232 $value to what it was at the start of the item, so that it has this value
3233 during the second string expansion. For the call from "if" or "run" to this
3234 function, save_lookup is set to lookup_value, so that this statement does
3235 nothing. */
3236
3237 lookup_value = save_lookup;
3238
3239 /* There now follows either another substring, or "fail", or nothing. This
3240 time, forced failures are noticed only if we want the second string. We must
3241 set skipping in the nested call if we don't want this string, or if we were
3242 already skipping. */
3243
3244 while (isspace(*s)) s++;
3245 if (*s == '{')
3246   {
3247   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3248   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
3249   expand_string_forcedfail = FALSE;
3250   if (*s++ != '}')
3251     {
3252     errwhere = US"'no' part did not start with '{'";
3253     goto FAILED_CURLY;
3254     }
3255
3256   /* If we want the second string, add it to the output */
3257
3258   if (!yes)
3259     *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub2);
3260   }
3261
3262 /* If there is no second string, but the word "fail" is present when the use of
3263 the second string is wanted, set a flag indicating it was a forced failure
3264 rather than a syntactic error. Swallow the terminating } in case this is nested
3265 inside another lookup or if or extract. */
3266
3267 else if (*s != '}')
3268   {
3269   uschar name[256];
3270   /* deconst cast ok here as source is s anyway */
3271   s = US read_name(name, sizeof(name), s, US"_");
3272   if (Ustrcmp(name, "fail") == 0)
3273     {
3274     if (!yes && !skipping)
3275       {
3276       while (isspace(*s)) s++;
3277       if (*s++ != '}')
3278         {
3279         errwhere = US"did not close with '}' after forcedfail";
3280         goto FAILED_CURLY;
3281         }
3282       expand_string_message =
3283         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3284       expand_string_forcedfail = TRUE;
3285       goto FAILED;
3286       }
3287     }
3288   else
3289     {
3290     expand_string_message =
3291       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3292     goto FAILED;
3293     }
3294   }
3295
3296 /* All we have to do now is to check on the final closing brace. */
3297
3298 while (isspace(*s)) s++;
3299 if (*s++ != '}')
3300   {
3301   errwhere = US"did not close with '}'";
3302   goto FAILED_CURLY;
3303   }
3304
3305
3306 RETURN:
3307 /* Update the input pointer value before returning */
3308 *sptr = s;
3309 return rc;
3310
3311 FAILED_CURLY:
3312   /* Get here if there is a bracketing failure */
3313   expand_string_message = string_sprintf(
3314     "curly-bracket problem in conditional yes/no parsing: %s\n"
3315     " remaining string is '%s'", errwhere, --s);
3316   rc = 2;
3317   goto RETURN;
3318
3319 FAILED:
3320   /* Get here for other failures */
3321   rc = 1;
3322   goto RETURN;
3323 }
3324
3325
3326
3327
3328 /*************************************************
3329 *    Handle MD5 or SHA-1 computation for HMAC    *
3330 *************************************************/
3331
3332 /* These are some wrapping functions that enable the HMAC code to be a bit
3333 cleaner. A good compiler will spot the tail recursion.
3334
3335 Arguments:
3336   type         HMAC_MD5 or HMAC_SHA1
3337   remaining    are as for the cryptographic hash functions
3338
3339 Returns:       nothing
3340 */
3341
3342 static void
3343 chash_start(int type, void *base)
3344 {
3345 if (type == HMAC_MD5)
3346   md5_start((md5 *)base);
3347 else
3348   sha1_start((hctx *)base);
3349 }
3350
3351 static void
3352 chash_mid(int type, void *base, uschar *string)
3353 {
3354 if (type == HMAC_MD5)
3355   md5_mid((md5 *)base, string);
3356 else
3357   sha1_mid((hctx *)base, string);
3358 }
3359
3360 static void
3361 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3362 {
3363 if (type == HMAC_MD5)
3364   md5_end((md5 *)base, string, length, digest);
3365 else
3366   sha1_end((hctx *)base, string, length, digest);
3367 }
3368
3369
3370
3371
3372
3373 /********************************************************
3374 * prvs: Get last three digits of days since Jan 1, 1970 *
3375 ********************************************************/
3376
3377 /* This is needed to implement the "prvs" BATV reverse
3378    path signing scheme
3379
3380 Argument: integer "days" offset to add or substract to
3381           or from the current number of days.
3382
3383 Returns:  pointer to string containing the last three
3384           digits of the number of days since Jan 1, 1970,
3385           modified by the offset argument, NULL if there
3386           was an error in the conversion.
3387
3388 */
3389
3390 static uschar *
3391 prvs_daystamp(int day_offset)
3392 {
3393 uschar *days = store_get(32);                /* Need at least 24 for cases */
3394 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3395   (time(NULL) + day_offset*86400)/86400);
3396 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3397 }
3398
3399
3400
3401 /********************************************************
3402 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3403 ********************************************************/
3404
3405 /* This is needed to implement the "prvs" BATV reverse
3406    path signing scheme
3407
3408 Arguments:
3409   address RFC2821 Address to use
3410       key The key to use (must be less than 64 characters
3411           in size)
3412   key_num Single-digit key number to use. Defaults to
3413           '0' when NULL.
3414
3415 Returns:  pointer to string containing the first three
3416           bytes of the final hash in hex format, NULL if
3417           there was an error in the process.
3418 */
3419
3420 static uschar *
3421 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3422 {
3423 uschar *hash_source, *p;
3424 int size = 0,offset = 0,i;
3425 hctx h;
3426 uschar innerhash[20];
3427 uschar finalhash[20];
3428 uschar innerkey[64];
3429 uschar outerkey[64];
3430 uschar *finalhash_hex = store_get(40);
3431
3432 if (key_num == NULL)
3433   key_num = US"0";
3434
3435 if (Ustrlen(key) > 64)
3436   return NULL;
3437
3438 hash_source = string_catn(NULL, &size, &offset, key_num, 1);
3439 hash_source = string_catn(hash_source, &size, &offset, daystamp, 3);
3440 hash_source = string_cat(hash_source, &size, &offset, address);
3441 hash_source[offset] = '\0';
3442
3443 DEBUG(D_expand) debug_printf_indent("prvs: hash source is '%s'\n", hash_source);
3444
3445 memset(innerkey, 0x36, 64);
3446 memset(outerkey, 0x5c, 64);
3447
3448 for (i = 0; i < Ustrlen(key); i++)
3449   {
3450   innerkey[i] ^= key[i];
3451   outerkey[i] ^= key[i];
3452   }
3453
3454 chash_start(HMAC_SHA1, &h);
3455 chash_mid(HMAC_SHA1, &h, innerkey);
3456 chash_end(HMAC_SHA1, &h, hash_source, offset, innerhash);
3457
3458 chash_start(HMAC_SHA1, &h);
3459 chash_mid(HMAC_SHA1, &h, outerkey);
3460 chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash);
3461
3462 p = finalhash_hex;
3463 for (i = 0; i < 3; i++)
3464   {
3465   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3466   *p++ = hex_digits[finalhash[i] & 0x0f];
3467   }
3468 *p = '\0';
3469
3470 return finalhash_hex;
3471 }
3472
3473
3474
3475
3476 /*************************************************
3477 *        Join a file onto the output string      *
3478 *************************************************/
3479
3480 /* This is used for readfile/readsock and after a run expansion.
3481 It joins the contents of a file onto the output string, globally replacing
3482 newlines with a given string (optionally).
3483
3484 Arguments:
3485   f            the FILE
3486   yield        pointer to the expandable string
3487   sizep        pointer to the current size
3488   ptrp         pointer to the current position
3489   eol          newline replacement string, or NULL
3490
3491 Returns:       new value of string pointer
3492 */
3493
3494 static uschar *
3495 cat_file(FILE *f, uschar *yield, int *sizep, int *ptrp, uschar *eol)
3496 {
3497 uschar buffer[1024];
3498
3499 while (Ufgets(buffer, sizeof(buffer), f))
3500   {
3501   int len = Ustrlen(buffer);
3502   if (eol && buffer[len-1] == '\n') len--;
3503   yield = string_catn(yield, sizep, ptrp, buffer, len);
3504   if (eol && buffer[len])
3505     yield = string_cat(yield, sizep, ptrp, eol);
3506   }
3507
3508 if (yield) yield[*ptrp] = 0;
3509
3510 return yield;
3511 }
3512
3513
3514
3515
3516 /*************************************************
3517 *          Evaluate numeric expression           *
3518 *************************************************/
3519
3520 /* This is a set of mutually recursive functions that evaluate an arithmetic
3521 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3522 these functions that is called from elsewhere is eval_expr, whose interface is:
3523
3524 Arguments:
3525   sptr        pointer to the pointer to the string - gets updated
3526   decimal     TRUE if numbers are to be assumed decimal
3527   error       pointer to where to put an error message - must be NULL on input
3528   endket      TRUE if ')' must terminate - FALSE for external call
3529
3530 Returns:      on success: the value of the expression, with *error still NULL
3531               on failure: an undefined value, with *error = a message
3532 */
3533
3534 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3535
3536
3537 static int_eximarith_t
3538 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3539 {
3540 uschar *s = *sptr;
3541 int_eximarith_t x = eval_op_or(&s, decimal, error);
3542 if (*error == NULL)
3543   {
3544   if (endket)
3545     {
3546     if (*s != ')')
3547       *error = US"expecting closing parenthesis";
3548     else
3549       while (isspace(*(++s)));
3550     }
3551   else if (*s != 0) *error = US"expecting operator";
3552   }
3553 *sptr = s;
3554 return x;
3555 }
3556
3557
3558 static int_eximarith_t
3559 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3560 {
3561 register int c;
3562 int_eximarith_t n;
3563 uschar *s = *sptr;
3564 while (isspace(*s)) s++;
3565 c = *s;
3566 if (isdigit(c))
3567   {
3568   int count;
3569   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3570   s += count;
3571   switch (tolower(*s))
3572     {
3573     default: break;
3574     case 'k': n *= 1024; s++; break;
3575     case 'm': n *= 1024*1024; s++; break;
3576     case 'g': n *= 1024*1024*1024; s++; break;
3577     }
3578   while (isspace (*s)) s++;
3579   }
3580 else if (c == '(')
3581   {
3582   s++;
3583   n = eval_expr(&s, decimal, error, 1);
3584   }
3585 else
3586   {
3587   *error = US"expecting number or opening parenthesis";
3588   n = 0;
3589   }
3590 *sptr = s;
3591 return n;
3592 }
3593
3594
3595 static int_eximarith_t
3596 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3597 {
3598 uschar *s = *sptr;
3599 int_eximarith_t x;
3600 while (isspace(*s)) s++;
3601 if (*s == '+' || *s == '-' || *s == '~')
3602   {
3603   int op = *s++;
3604   x = eval_op_unary(&s, decimal, error);
3605   if (op == '-') x = -x;
3606     else if (op == '~') x = ~x;
3607   }
3608 else
3609   {
3610   x = eval_number(&s, decimal, error);
3611   }
3612 *sptr = s;
3613 return x;
3614 }
3615
3616
3617 static int_eximarith_t
3618 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3619 {
3620 uschar *s = *sptr;
3621 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3622 if (*error == NULL)
3623   {
3624   while (*s == '*' || *s == '/' || *s == '%')
3625     {
3626     int op = *s++;
3627     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3628     if (*error != NULL) break;
3629     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3630      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3631      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3632      * -N*M is INT_MIN will yield INT_MIN.
3633      * Since we don't support floating point, this is somewhat simpler.
3634      * Ideally, we'd return an error, but since we overflow for all other
3635      * arithmetic, consistency suggests otherwise, but what's the correct value
3636      * to use?  There is none.
3637      * The C standard guarantees overflow for unsigned arithmetic but signed
3638      * overflow invokes undefined behaviour; in practice, this is overflow
3639      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3640      * that long/longlong larger than int are available, or we could just work
3641      * with larger types.  We should consider whether to guarantee 32bit eval
3642      * and 64-bit working variables, with errors returned.  For now ...
3643      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3644      * can just let the other invalid results occur otherwise, as they have
3645      * until now.  For this one case, we can coerce.
3646      */
3647     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3648       {
3649       DEBUG(D_expand)
3650         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3651             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3652       x = EXIM_ARITH_MAX;
3653       continue;
3654       }
3655     if (op == '*')
3656       x *= y;
3657     else
3658       {
3659       if (y == 0)
3660         {
3661         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3662         x = 0;
3663         break;
3664         }
3665       if (op == '/')
3666         x /= y;
3667       else
3668         x %= y;
3669       }
3670     }
3671   }
3672 *sptr = s;
3673 return x;
3674 }
3675
3676
3677 static int_eximarith_t
3678 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3679 {
3680 uschar *s = *sptr;
3681 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3682 if (!*error)
3683   {
3684   while (*s == '+' || *s == '-')
3685     {
3686     int op = *s++;
3687     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3688     if (*error) break;
3689     if (  (x >=   EXIM_ARITH_MAX/2  && x >=   EXIM_ARITH_MAX/2)
3690        || (x <= -(EXIM_ARITH_MAX/2) && y <= -(EXIM_ARITH_MAX/2)))
3691       {                 /* over-conservative check */
3692       *error = op == '+'
3693         ? US"overflow in sum" : US"overflow in difference";
3694       break;
3695       }
3696     if (op == '+') x += y; else x -= y;
3697     }
3698   }
3699 *sptr = s;
3700 return x;
3701 }
3702
3703
3704 static int_eximarith_t
3705 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3706 {
3707 uschar *s = *sptr;
3708 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3709 if (*error == NULL)
3710   {
3711   while ((*s == '<' || *s == '>') && s[1] == s[0])
3712     {
3713     int_eximarith_t y;
3714     int op = *s++;
3715     s++;
3716     y = eval_op_sum(&s, decimal, error);
3717     if (*error != NULL) break;
3718     if (op == '<') x <<= y; else x >>= y;
3719     }
3720   }
3721 *sptr = s;
3722 return x;
3723 }
3724
3725
3726 static int_eximarith_t
3727 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3728 {
3729 uschar *s = *sptr;
3730 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3731 if (*error == NULL)
3732   {
3733   while (*s == '&')
3734     {
3735     int_eximarith_t y;
3736     s++;
3737     y = eval_op_shift(&s, decimal, error);
3738     if (*error != NULL) break;
3739     x &= y;
3740     }
3741   }
3742 *sptr = s;
3743 return x;
3744 }
3745
3746
3747 static int_eximarith_t
3748 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3749 {
3750 uschar *s = *sptr;
3751 int_eximarith_t x = eval_op_and(&s, decimal, error);
3752 if (*error == NULL)
3753   {
3754   while (*s == '^')
3755     {
3756     int_eximarith_t y;
3757     s++;
3758     y = eval_op_and(&s, decimal, error);
3759     if (*error != NULL) break;
3760     x ^= y;
3761     }
3762   }
3763 *sptr = s;
3764 return x;
3765 }
3766
3767
3768 static int_eximarith_t
3769 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3770 {
3771 uschar *s = *sptr;
3772 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3773 if (*error == NULL)
3774   {
3775   while (*s == '|')
3776     {
3777     int_eximarith_t y;
3778     s++;
3779     y = eval_op_xor(&s, decimal, error);
3780     if (*error != NULL) break;
3781     x |= y;
3782     }
3783   }
3784 *sptr = s;
3785 return x;
3786 }
3787
3788
3789
3790 /*************************************************
3791 *                 Expand string                  *
3792 *************************************************/
3793
3794 /* Returns either an unchanged string, or the expanded string in stacking pool
3795 store. Interpreted sequences are:
3796
3797    \...                    normal escaping rules
3798    $name                   substitutes the variable
3799    ${name}                 ditto
3800    ${op:string}            operates on the expanded string value
3801    ${item{arg1}{arg2}...}  expands the args and then does the business
3802                              some literal args are not enclosed in {}
3803
3804 There are now far too many operators and item types to make it worth listing
3805 them here in detail any more.
3806
3807 We use an internal routine recursively to handle embedded substrings. The
3808 external function follows. The yield is NULL if the expansion failed, and there
3809 are two cases: if something collapsed syntactically, or if "fail" was given
3810 as the action on a lookup failure. These can be distinguished by looking at the
3811 variable expand_string_forcedfail, which is TRUE in the latter case.
3812
3813 The skipping flag is set true when expanding a substring that isn't actually
3814 going to be used (after "if" or "lookup") and it prevents lookups from
3815 happening lower down.
3816
3817 Store usage: At start, a store block of the length of the input plus 64
3818 is obtained. This is expanded as necessary by string_cat(), which might have to
3819 get a new block, or might be able to expand the original. At the end of the
3820 function we can release any store above that portion of the yield block that
3821 was actually used. In many cases this will be optimal.
3822
3823 However: if the first item in the expansion is a variable name or header name,
3824 we reset the store before processing it; if the result is in fresh store, we
3825 use that without copying. This is helpful for expanding strings like
3826 $message_headers which can get very long.
3827
3828 There's a problem if a ${dlfunc item has side-effects that cause allocation,
3829 since resetting the store at the end of the expansion will free store that was
3830 allocated by the plugin code as well as the slop after the expanded string. So
3831 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
3832 and, given the acl condition, ${if }. This is an unfortunate consequence of
3833 string expansion becoming too powerful.
3834
3835 Arguments:
3836   string         the string to be expanded
3837   ket_ends       true if expansion is to stop at }
3838   left           if not NULL, a pointer to the first character after the
3839                  expansion is placed here (typically used with ket_ends)
3840   skipping       TRUE for recursive calls when the value isn't actually going
3841                  to be used (to allow for optimisation)
3842   honour_dollar  TRUE if $ is to be expanded,
3843                  FALSE if it's just another character
3844   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
3845                  the store.
3846
3847 Returns:         NULL if expansion fails:
3848                    expand_string_forcedfail is set TRUE if failure was forced
3849                    expand_string_message contains a textual error message
3850                  a pointer to the expanded string on success
3851 */
3852
3853 static uschar *
3854 expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left,
3855   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
3856 {
3857 int ptr = 0;
3858 int size = Ustrlen(string)+ 64;
3859 uschar *yield = store_get(size);
3860 int item_type;
3861 const uschar *s = string;
3862 uschar *save_expand_nstring[EXPAND_MAXN+1];
3863 int save_expand_nlength[EXPAND_MAXN+1];
3864 BOOL resetok = TRUE;
3865
3866 expand_level++;
3867 DEBUG(D_expand)
3868   debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n",
3869     skipping
3870     ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning"
3871     : "considering",
3872     string);
3873
3874 expand_string_forcedfail = FALSE;
3875 expand_string_message = US"";
3876
3877 while (*s != 0)
3878   {
3879   uschar *value;
3880   uschar name[256];
3881
3882   /* \ escapes the next character, which must exist, or else
3883   the expansion fails. There's a special escape, \N, which causes
3884   copying of the subject verbatim up to the next \N. Otherwise,
3885   the escapes are the standard set. */
3886
3887   if (*s == '\\')
3888     {
3889     if (s[1] == 0)
3890       {
3891       expand_string_message = US"\\ at end of string";
3892       goto EXPAND_FAILED;
3893       }
3894
3895     if (s[1] == 'N')
3896       {
3897       const uschar * t = s + 2;
3898       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
3899       yield = string_catn(yield, &size, &ptr, t, s - t);
3900       if (*s != 0) s += 2;
3901       }
3902
3903     else
3904       {
3905       uschar ch[1];
3906       ch[0] = string_interpret_escape(&s);
3907       s++;
3908       yield = string_catn(yield, &size, &ptr, ch, 1);
3909       }
3910
3911     continue;
3912     }
3913
3914   /*{*/
3915   /* Anything other than $ is just copied verbatim, unless we are
3916   looking for a terminating } character. */
3917
3918   /*{*/
3919   if (ket_ends && *s == '}') break;
3920
3921   if (*s != '$' || !honour_dollar)
3922     {
3923     yield = string_catn(yield, &size, &ptr, s++, 1);
3924     continue;
3925     }
3926
3927   /* No { after the $ - must be a plain name or a number for string
3928   match variable. There has to be a fudge for variables that are the
3929   names of header fields preceded by "$header_" because header field
3930   names can contain any printing characters except space and colon.
3931   For those that don't like typing this much, "$h_" is a synonym for
3932   "$header_". A non-existent header yields a NULL value; nothing is
3933   inserted. */  /*}*/
3934
3935   if (isalpha((*(++s))))
3936     {
3937     int len;
3938     int newsize = 0;
3939
3940     s = read_name(name, sizeof(name), s, US"_");
3941
3942     /* If this is the first thing to be expanded, release the pre-allocated
3943     buffer. */
3944
3945     if (ptr == 0 && yield != NULL)
3946       {
3947       if (resetok) store_reset(yield);
3948       yield = NULL;
3949       size = 0;
3950       }
3951
3952     /* Header */
3953
3954     if (Ustrncmp(name, "h_", 2) == 0 ||
3955         Ustrncmp(name, "rh_", 3) == 0 ||
3956         Ustrncmp(name, "bh_", 3) == 0 ||
3957         Ustrncmp(name, "header_", 7) == 0 ||
3958         Ustrncmp(name, "rheader_", 8) == 0 ||
3959         Ustrncmp(name, "bheader_", 8) == 0)
3960       {
3961       BOOL want_raw = (name[0] == 'r')? TRUE : FALSE;
3962       uschar *charset = (name[0] == 'b')? NULL : headers_charset;
3963       s = read_header_name(name, sizeof(name), s);
3964       value = find_header(name, FALSE, &newsize, want_raw, charset);
3965
3966       /* If we didn't find the header, and the header contains a closing brace
3967       character, this may be a user error where the terminating colon
3968       has been omitted. Set a flag to adjust the error message in this case.
3969       But there is no error here - nothing gets inserted. */
3970
3971       if (value == NULL)
3972         {
3973         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
3974         continue;
3975         }
3976       }
3977
3978     /* Variable */
3979
3980     else if (!(value = find_variable(name, FALSE, skipping, &newsize)))
3981       {
3982       expand_string_message =
3983         string_sprintf("unknown variable name \"%s\"", name);
3984         check_variable_error_message(name);
3985       goto EXPAND_FAILED;
3986       }
3987
3988     /* If the data is known to be in a new buffer, newsize will be set to the
3989     size of that buffer. If this is the first thing in an expansion string,
3990     yield will be NULL; just point it at the new store instead of copying. Many
3991     expansion strings contain just one reference, so this is a useful
3992     optimization, especially for humungous headers. */
3993
3994     len = Ustrlen(value);
3995     if (yield == NULL && newsize != 0)
3996       {
3997       yield = value;
3998       size = newsize;
3999       ptr = len;
4000       }
4001     else yield = string_catn(yield, &size, &ptr, value, len);
4002
4003     continue;
4004     }
4005
4006   if (isdigit(*s))
4007     {
4008     int n;
4009     s = read_cnumber(&n, s);
4010     if (n >= 0 && n <= expand_nmax)
4011       yield = string_catn(yield, &size, &ptr, expand_nstring[n],
4012         expand_nlength[n]);
4013     continue;
4014     }
4015
4016   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
4017
4018   if (*s != '{')                                                        /*}*/
4019     {
4020     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
4021     goto EXPAND_FAILED;
4022     }
4023
4024   /* After { there can be various things, but they all start with
4025   an initial word, except for a number for a string match variable. */
4026
4027   if (isdigit((*(++s))))
4028     {
4029     int n;
4030     s = read_cnumber(&n, s);            /*{*/
4031     if (*s++ != '}')
4032       {                                 /*{*/
4033       expand_string_message = US"} expected after number";
4034       goto EXPAND_FAILED;
4035       }
4036     if (n >= 0 && n <= expand_nmax)
4037       yield = string_catn(yield, &size, &ptr, expand_nstring[n],
4038         expand_nlength[n]);
4039     continue;
4040     }
4041
4042   if (!isalpha(*s))
4043     {
4044     expand_string_message = US"letter or digit expected after ${";      /*}*/
4045     goto EXPAND_FAILED;
4046     }
4047
4048   /* Allow "-" in names to cater for substrings with negative
4049   arguments. Since we are checking for known names after { this is
4050   OK. */
4051
4052   s = read_name(name, sizeof(name), s, US"_-");
4053   item_type = chop_match(name, item_table, nelem(item_table));
4054
4055   switch(item_type)
4056     {
4057     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
4058     If the ACL returns accept or reject we return content set by "message ="
4059     There is currently no limit on recursion; this would have us call
4060     acl_check_internal() directly and get a current level from somewhere.
4061     See also the acl expansion condition ECOND_ACL and the traditional
4062     acl modifier ACLC_ACL.
4063     Assume that the function has side-effects on the store that must be preserved.
4064     */
4065
4066     case EITEM_ACL:
4067       /* ${acl {name} {arg1}{arg2}...} */
4068       {
4069       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
4070       uschar *user_msg;
4071
4072       switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, US"acl",
4073                       &resetok))
4074         {
4075         case 1: goto EXPAND_FAILED_CURLY;
4076         case 2:
4077         case 3: goto EXPAND_FAILED;
4078         }
4079       if (skipping) continue;
4080
4081       resetok = FALSE;
4082       switch(eval_acl(sub, nelem(sub), &user_msg))
4083         {
4084         case OK:
4085         case FAIL:
4086           DEBUG(D_expand)
4087             debug_printf_indent("acl expansion yield: %s\n", user_msg);
4088           if (user_msg)
4089             yield = string_cat(yield, &size, &ptr, user_msg);
4090           continue;
4091
4092         case DEFER:
4093           expand_string_forcedfail = TRUE;
4094           /*FALLTHROUGH*/
4095         default:
4096           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
4097           goto EXPAND_FAILED;
4098         }
4099       }
4100
4101     /* Handle conditionals - preserve the values of the numerical expansion
4102     variables in case they get changed by a regular expression match in the
4103     condition. If not, they retain their external settings. At the end
4104     of this "if" section, they get restored to their previous values. */
4105
4106     case EITEM_IF:
4107       {
4108       BOOL cond = FALSE;
4109       const uschar *next_s;
4110       int save_expand_nmax =
4111         save_expand_strings(save_expand_nstring, save_expand_nlength);
4112
4113       while (isspace(*s)) s++;
4114       next_s = eval_condition(s, &resetok, skipping ? NULL : &cond);
4115       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
4116
4117       DEBUG(D_expand)
4118         {
4119         debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4120           "condition: %.*s\n",
4121           (int)(next_s - s), s);
4122         debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4123           UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
4124           "result: %s\n",
4125           cond ? "true" : "false");
4126         }
4127
4128       s = next_s;
4129
4130       /* The handling of "yes" and "no" result strings is now in a separate
4131       function that is also used by ${lookup} and ${extract} and ${run}. */
4132
4133       switch(process_yesno(
4134                skipping,                     /* were previously skipping */
4135                cond,                         /* success/failure indicator */
4136                lookup_value,                 /* value to reset for string2 */
4137                &s,                           /* input pointer */
4138                &yield,                       /* output pointer */
4139                &size,                        /* output size */
4140                &ptr,                         /* output current point */
4141                US"if",                       /* condition type */
4142                &resetok))
4143         {
4144         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4145         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4146         }
4147
4148       /* Restore external setting of expansion variables for continuation
4149       at this level. */
4150
4151       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4152         save_expand_nlength);
4153       continue;
4154       }
4155
4156 #ifdef SUPPORT_I18N
4157     case EITEM_IMAPFOLDER:
4158       {                         /* ${imapfolder {name}{sep]{specials}} */
4159       uschar *sub_arg[3];
4160       uschar *encoded;
4161
4162       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4163                       &resetok))
4164         {
4165         case 1: goto EXPAND_FAILED_CURLY;
4166         case 2:
4167         case 3: goto EXPAND_FAILED;
4168         }
4169
4170       if (sub_arg[1] == NULL)           /* One argument */
4171         {
4172         sub_arg[1] = US"/";             /* default separator */
4173         sub_arg[2] = NULL;
4174         }
4175       else if (Ustrlen(sub_arg[1]) != 1)
4176         {
4177         expand_string_message =
4178           string_sprintf(
4179                 "IMAP folder separator must be one character, found \"%s\"",
4180                 sub_arg[1]);
4181         goto EXPAND_FAILED;
4182         }
4183
4184       if (!skipping)
4185         {
4186         if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
4187                             sub_arg[1][0], sub_arg[2], &expand_string_message)))
4188           goto EXPAND_FAILED;
4189         yield = string_cat(yield, &size, &ptr, encoded);
4190         }
4191       continue;
4192       }
4193 #endif
4194
4195     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4196     expanding an internal string that isn't actually going to be used. All we
4197     need to do is check the syntax, so don't do a lookup at all. Preserve the
4198     values of the numerical expansion variables in case they get changed by a
4199     partial lookup. If not, they retain their external settings. At the end
4200     of this "lookup" section, they get restored to their previous values. */
4201
4202     case EITEM_LOOKUP:
4203       {
4204       int stype, partial, affixlen, starflags;
4205       int expand_setup = 0;
4206       int nameptr = 0;
4207       uschar *key, *filename;
4208       const uschar *affix;
4209       uschar *save_lookup_value = lookup_value;
4210       int save_expand_nmax =
4211         save_expand_strings(save_expand_nstring, save_expand_nlength);
4212
4213       if ((expand_forbid & RDO_LOOKUP) != 0)
4214         {
4215         expand_string_message = US"lookup expansions are not permitted";
4216         goto EXPAND_FAILED;
4217         }
4218
4219       /* Get the key we are to look up for single-key+file style lookups.
4220       Otherwise set the key NULL pro-tem. */
4221
4222       while (isspace(*s)) s++;
4223       if (*s == '{')                                    /*}*/
4224         {
4225         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4226         if (!key) goto EXPAND_FAILED;                   /*{{*/
4227         if (*s++ != '}')
4228           {
4229           expand_string_message = US"missing '}' after lookup key";
4230           goto EXPAND_FAILED_CURLY;
4231           }
4232         while (isspace(*s)) s++;
4233         }
4234       else key = NULL;
4235
4236       /* Find out the type of database */
4237
4238       if (!isalpha(*s))
4239         {
4240         expand_string_message = US"missing lookup type";
4241         goto EXPAND_FAILED;
4242         }
4243
4244       /* The type is a string that may contain special characters of various
4245       kinds. Allow everything except space or { to appear; the actual content
4246       is checked by search_findtype_partial. */         /*}*/
4247
4248       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4249         {
4250         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4251         s++;
4252         }
4253       name[nameptr] = 0;
4254       while (isspace(*s)) s++;
4255
4256       /* Now check for the individual search type and any partial or default
4257       options. Only those types that are actually in the binary are valid. */
4258
4259       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4260         &starflags);
4261       if (stype < 0)
4262         {
4263         expand_string_message = search_error_message;
4264         goto EXPAND_FAILED;
4265         }
4266
4267       /* Check that a key was provided for those lookup types that need it,
4268       and was not supplied for those that use the query style. */
4269
4270       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4271         {
4272         if (key == NULL)
4273           {
4274           expand_string_message = string_sprintf("missing {key} for single-"
4275             "key \"%s\" lookup", name);
4276           goto EXPAND_FAILED;
4277           }
4278         }
4279       else
4280         {
4281         if (key != NULL)
4282           {
4283           expand_string_message = string_sprintf("a single key was given for "
4284             "lookup type \"%s\", which is not a single-key lookup type", name);
4285           goto EXPAND_FAILED;
4286           }
4287         }
4288
4289       /* Get the next string in brackets and expand it. It is the file name for
4290       single-key+file lookups, and the whole query otherwise. In the case of
4291       queries that also require a file name (e.g. sqlite), the file name comes
4292       first. */
4293
4294       if (*s != '{')
4295         {
4296         expand_string_message = US"missing '{' for lookup file-or-query arg";
4297         goto EXPAND_FAILED_CURLY;
4298         }
4299       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4300       if (filename == NULL) goto EXPAND_FAILED;
4301       if (*s++ != '}')
4302         {
4303         expand_string_message = US"missing '}' closing lookup file-or-query arg";
4304         goto EXPAND_FAILED_CURLY;
4305         }
4306       while (isspace(*s)) s++;
4307
4308       /* If this isn't a single-key+file lookup, re-arrange the variables
4309       to be appropriate for the search_ functions. For query-style lookups,
4310       there is just a "key", and no file name. For the special query-style +
4311       file types, the query (i.e. "key") starts with a file name. */
4312
4313       if (!key)
4314         {
4315         while (isspace(*filename)) filename++;
4316         key = filename;
4317
4318         if (mac_islookup(stype, lookup_querystyle))
4319           filename = NULL;
4320         else
4321           {
4322           if (*filename != '/')
4323             {
4324             expand_string_message = string_sprintf(
4325               "absolute file name expected for \"%s\" lookup", name);
4326             goto EXPAND_FAILED;
4327             }
4328           while (*key != 0 && !isspace(*key)) key++;
4329           if (*key != 0) *key++ = 0;
4330           }
4331         }
4332
4333       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4334       the entry was not found. Note that there is no search_close() function.
4335       Files are left open in case of re-use. At suitable places in higher logic,
4336       search_tidyup() is called to tidy all open files. This can save opening
4337       the same file several times. However, files may also get closed when
4338       others are opened, if too many are open at once. The rule is that a
4339       handle should not be used after a second search_open().
4340
4341       Request that a partial search sets up $1 and maybe $2 by passing
4342       expand_setup containing zero. If its value changes, reset expand_nmax,
4343       since new variables will have been set. Note that at the end of this
4344       "lookup" section, the old numeric variables are restored. */
4345
4346       if (skipping)
4347         lookup_value = NULL;
4348       else
4349         {
4350         void *handle = search_open(filename, stype, 0, NULL, NULL);
4351         if (handle == NULL)
4352           {
4353           expand_string_message = search_error_message;
4354           goto EXPAND_FAILED;
4355           }
4356         lookup_value = search_find(handle, filename, key, partial, affix,
4357           affixlen, starflags, &expand_setup);
4358         if (search_find_defer)
4359           {
4360           expand_string_message =
4361             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4362               string_printing2(key, FALSE), search_error_message);
4363           goto EXPAND_FAILED;
4364           }
4365         if (expand_setup > 0) expand_nmax = expand_setup;
4366         }
4367
4368       /* The handling of "yes" and "no" result strings is now in a separate
4369       function that is also used by ${if} and ${extract}. */
4370
4371       switch(process_yesno(
4372                skipping,                     /* were previously skipping */
4373                lookup_value != NULL,         /* success/failure indicator */
4374                save_lookup_value,            /* value to reset for string2 */
4375                &s,                           /* input pointer */
4376                &yield,                       /* output pointer */
4377                &size,                        /* output size */
4378                &ptr,                         /* output current point */
4379                US"lookup",                   /* condition type */
4380                &resetok))
4381         {
4382         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4383         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4384         }
4385
4386       /* Restore external setting of expansion variables for carrying on
4387       at this level, and continue. */
4388
4389       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4390         save_expand_nlength);
4391       continue;
4392       }
4393
4394     /* If Perl support is configured, handle calling embedded perl subroutines,
4395     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4396     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4397     arguments (defined below). */
4398
4399     #define EXIM_PERL_MAX_ARGS 8
4400
4401     case EITEM_PERL:
4402     #ifndef EXIM_PERL
4403     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4404       "is not included in this binary";
4405     goto EXPAND_FAILED;
4406
4407     #else   /* EXIM_PERL */
4408       {
4409       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4410       uschar *new_yield;
4411
4412       if ((expand_forbid & RDO_PERL) != 0)
4413         {
4414         expand_string_message = US"Perl calls are not permitted";
4415         goto EXPAND_FAILED;
4416         }
4417
4418       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4419            US"perl", &resetok))
4420         {
4421         case 1: goto EXPAND_FAILED_CURLY;
4422         case 2:
4423         case 3: goto EXPAND_FAILED;
4424         }
4425
4426       /* If skipping, we don't actually do anything */
4427
4428       if (skipping) continue;
4429
4430       /* Start the interpreter if necessary */
4431
4432       if (!opt_perl_started)
4433         {
4434         uschar *initerror;
4435         if (opt_perl_startup == NULL)
4436           {
4437           expand_string_message = US"A setting of perl_startup is needed when "
4438             "using the Perl interpreter";
4439           goto EXPAND_FAILED;
4440           }
4441         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4442         initerror = init_perl(opt_perl_startup);
4443         if (initerror != NULL)
4444           {
4445           expand_string_message =
4446             string_sprintf("error in perl_startup code: %s\n", initerror);
4447           goto EXPAND_FAILED;
4448           }
4449         opt_perl_started = TRUE;
4450         }
4451
4452       /* Call the function */
4453
4454       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4455       new_yield = call_perl_cat(yield, &size, &ptr, &expand_string_message,
4456         sub_arg[0], sub_arg + 1);
4457
4458       /* NULL yield indicates failure; if the message pointer has been set to
4459       NULL, the yield was undef, indicating a forced failure. Otherwise the
4460       message will indicate some kind of Perl error. */
4461
4462       if (new_yield == NULL)
4463         {
4464         if (expand_string_message == NULL)
4465           {
4466           expand_string_message =
4467             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4468               "failure", sub_arg[0]);
4469           expand_string_forcedfail = TRUE;
4470           }
4471         goto EXPAND_FAILED;
4472         }
4473
4474       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4475       set during a callback from Perl. */
4476
4477       expand_string_forcedfail = FALSE;
4478       yield = new_yield;
4479       continue;
4480       }
4481     #endif /* EXIM_PERL */
4482
4483     /* Transform email address to "prvs" scheme to use
4484        as BATV-signed return path */
4485
4486     case EITEM_PRVS:
4487       {
4488       uschar *sub_arg[3];
4489       uschar *p,*domain;
4490
4491       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4492         {
4493         case 1: goto EXPAND_FAILED_CURLY;
4494         case 2:
4495         case 3: goto EXPAND_FAILED;
4496         }
4497
4498       /* If skipping, we don't actually do anything */
4499       if (skipping) continue;
4500
4501       /* sub_arg[0] is the address */
4502       domain = Ustrrchr(sub_arg[0],'@');
4503       if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) )
4504         {
4505         expand_string_message = US"prvs first argument must be a qualified email address";
4506         goto EXPAND_FAILED;
4507         }
4508
4509       /* Calculate the hash. The second argument must be a single-digit
4510       key number, or unset. */
4511
4512       if (sub_arg[2] != NULL &&
4513           (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4514         {
4515         expand_string_message = US"prvs second argument must be a single digit";
4516         goto EXPAND_FAILED;
4517         }
4518
4519       p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7));
4520       if (p == NULL)
4521         {
4522         expand_string_message = US"prvs hmac-sha1 conversion failed";
4523         goto EXPAND_FAILED;
4524         }
4525
4526       /* Now separate the domain from the local part */
4527       *domain++ = '\0';
4528
4529       yield = string_catn(yield, &size, &ptr, US"prvs=", 5);
4530       yield = string_catn(yield, &size, &ptr, sub_arg[2] ? sub_arg[2] : US"0", 1);
4531       yield = string_catn(yield, &size, &ptr, prvs_daystamp(7), 3);
4532       yield = string_catn(yield, &size, &ptr, p, 6);
4533       yield = string_catn(yield, &size, &ptr, US"=", 1);
4534       yield = string_cat (yield, &size, &ptr, sub_arg[0]);
4535       yield = string_catn(yield, &size, &ptr, US"@", 1);
4536       yield = string_cat (yield, &size, &ptr, domain);
4537
4538       continue;
4539       }
4540
4541     /* Check a prvs-encoded address for validity */
4542
4543     case EITEM_PRVSCHECK:
4544       {
4545       uschar *sub_arg[3];
4546       int mysize = 0, myptr = 0;
4547       const pcre *re;
4548       uschar *p;
4549
4550       /* TF: Ugliness: We want to expand parameter 1 first, then set
4551          up expansion variables that are used in the expansion of
4552          parameter 2. So we clone the string for the first
4553          expansion, where we only expand parameter 1.
4554
4555          PH: Actually, that isn't necessary. The read_subs() function is
4556          designed to work this way for the ${if and ${lookup expansions. I've
4557          tidied the code.
4558       */
4559
4560       /* Reset expansion variables */
4561       prvscheck_result = NULL;
4562       prvscheck_address = NULL;
4563       prvscheck_keynum = NULL;
4564
4565       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4566         {
4567         case 1: goto EXPAND_FAILED_CURLY;
4568         case 2:
4569         case 3: goto EXPAND_FAILED;
4570         }
4571
4572       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4573                               TRUE,FALSE);
4574
4575       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4576         {
4577         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4578         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4579         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4580         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4581         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4582
4583         DEBUG(D_expand) debug_printf_indent("prvscheck localpart: %s\n", local_part);
4584         DEBUG(D_expand) debug_printf_indent("prvscheck key number: %s\n", key_num);
4585         DEBUG(D_expand) debug_printf_indent("prvscheck daystamp: %s\n", daystamp);
4586         DEBUG(D_expand) debug_printf_indent("prvscheck hash: %s\n", hash);
4587         DEBUG(D_expand) debug_printf_indent("prvscheck domain: %s\n", domain);
4588
4589         /* Set up expansion variables */
4590         prvscheck_address = string_cat (NULL, &mysize, &myptr, local_part);
4591         prvscheck_address = string_catn(prvscheck_address, &mysize, &myptr, US"@", 1);
4592         prvscheck_address = string_cat (prvscheck_address, &mysize, &myptr, domain);
4593         prvscheck_address[myptr] = '\0';
4594         prvscheck_keynum = string_copy(key_num);
4595
4596         /* Now expand the second argument */
4597         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4598           {
4599           case 1: goto EXPAND_FAILED_CURLY;
4600           case 2:
4601           case 3: goto EXPAND_FAILED;
4602           }
4603
4604         /* Now we have the key and can check the address. */
4605
4606         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4607           daystamp);
4608
4609         if (p == NULL)
4610           {
4611           expand_string_message = US"hmac-sha1 conversion failed";
4612           goto EXPAND_FAILED;
4613           }
4614
4615         DEBUG(D_expand) debug_printf_indent("prvscheck: received hash is %s\n", hash);
4616         DEBUG(D_expand) debug_printf_indent("prvscheck:      own hash is %s\n", p);
4617
4618         if (Ustrcmp(p,hash) == 0)
4619           {
4620           /* Success, valid BATV address. Now check the expiry date. */
4621           uschar *now = prvs_daystamp(0);
4622           unsigned int inow = 0,iexpire = 1;
4623
4624           (void)sscanf(CS now,"%u",&inow);
4625           (void)sscanf(CS daystamp,"%u",&iexpire);
4626
4627           /* When "iexpire" is < 7, a "flip" has occured.
4628              Adjust "inow" accordingly. */
4629           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4630
4631           if (iexpire >= inow)
4632             {
4633             prvscheck_result = US"1";
4634             DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n");
4635             }
4636             else
4637             {
4638             prvscheck_result = NULL;
4639             DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n");
4640             }
4641           }
4642         else
4643           {
4644           prvscheck_result = NULL;
4645           DEBUG(D_expand) debug_printf_indent("prvscheck: hash failure, $pvrs_result unset\n");
4646           }
4647
4648         /* Now expand the final argument. We leave this till now so that
4649         it can include $prvscheck_result. */
4650
4651         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4652           {
4653           case 1: goto EXPAND_FAILED_CURLY;
4654           case 2:
4655           case 3: goto EXPAND_FAILED;
4656           }
4657
4658         yield = string_cat(yield, &size, &ptr,
4659           !sub_arg[0] || !*sub_arg[0] ? prvscheck_address : sub_arg[0]);
4660
4661         /* Reset the "internal" variables afterwards, because they are in
4662         dynamic store that will be reclaimed if the expansion succeeded. */
4663
4664         prvscheck_address = NULL;
4665         prvscheck_keynum = NULL;
4666         }
4667       else
4668         /* Does not look like a prvs encoded address, return the empty string.
4669            We need to make sure all subs are expanded first, so as to skip over
4670            the entire item. */
4671
4672         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4673           {
4674           case 1: goto EXPAND_FAILED_CURLY;
4675           case 2:
4676           case 3: goto EXPAND_FAILED;
4677           }
4678
4679       continue;
4680       }
4681
4682     /* Handle "readfile" to insert an entire file */
4683
4684     case EITEM_READFILE:
4685       {
4686       FILE *f;
4687       uschar *sub_arg[2];
4688
4689       if ((expand_forbid & RDO_READFILE) != 0)
4690         {
4691         expand_string_message = US"file insertions are not permitted";
4692         goto EXPAND_FAILED;
4693         }
4694
4695       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4696         {
4697         case 1: goto EXPAND_FAILED_CURLY;
4698         case 2:
4699         case 3: goto EXPAND_FAILED;
4700         }
4701
4702       /* If skipping, we don't actually do anything */
4703
4704       if (skipping) continue;
4705
4706       /* Open the file and read it */
4707
4708       f = Ufopen(sub_arg[0], "rb");
4709       if (f == NULL)
4710         {
4711         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4712         goto EXPAND_FAILED;
4713         }
4714
4715       yield = cat_file(f, yield, &size, &ptr, sub_arg[1]);
4716       (void)fclose(f);
4717       continue;
4718       }
4719
4720     /* Handle "readsocket" to insert data from a Unix domain socket */
4721
4722     case EITEM_READSOCK:
4723       {
4724       int fd;
4725       int timeout = 5;
4726       int save_ptr = ptr;
4727       FILE *f;
4728       struct sockaddr_un sockun;         /* don't call this "sun" ! */
4729       uschar *arg;
4730       uschar *sub_arg[4];
4731       BOOL do_shutdown = TRUE;
4732
4733       if (expand_forbid & RDO_READSOCK)
4734         {
4735         expand_string_message = US"socket insertions are not permitted";
4736         goto EXPAND_FAILED;
4737         }
4738
4739       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4740       because there may be a string for expansion on failure. */
4741
4742       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4743         {
4744         case 1: goto EXPAND_FAILED_CURLY;
4745         case 2:                             /* Won't occur: no end check */
4746         case 3: goto EXPAND_FAILED;
4747         }
4748
4749       /* Sort out timeout, if given.  The second arg is a list with the first element
4750       being a time value.  Any more are options of form "name=value".  Currently the
4751       only option recognised is "shutdown". */
4752
4753       if (sub_arg[2])
4754         {
4755         const uschar * list = sub_arg[2];
4756         uschar * item;
4757         int sep = 0;
4758
4759         item = string_nextinlist(&list, &sep, NULL, 0);
4760         if ((timeout = readconf_readtime(item, 0, FALSE)) < 0)
4761           {
4762           expand_string_message = string_sprintf("bad time value %s", item);
4763           goto EXPAND_FAILED;
4764           }
4765
4766         while ((item = string_nextinlist(&list, &sep, NULL, 0)))
4767           if (Ustrncmp(item, US"shutdown=", 9) == 0)
4768             if (Ustrcmp(item + 9, US"no") == 0)
4769               do_shutdown = FALSE;
4770         }
4771       else sub_arg[3] = NULL;                     /* No eol if no timeout */
4772
4773       /* If skipping, we don't actually do anything. Otherwise, arrange to
4774       connect to either an IP or a Unix socket. */
4775
4776       if (!skipping)
4777         {
4778         /* Handle an IP (internet) domain */
4779
4780         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
4781           {
4782           int port;
4783           uschar *server_name = sub_arg[0] + 5;
4784           uschar *port_name = Ustrrchr(server_name, ':');
4785
4786           /* Sort out the port */
4787
4788           if (port_name == NULL)
4789             {
4790             expand_string_message =
4791               string_sprintf("missing port for readsocket %s", sub_arg[0]);
4792             goto EXPAND_FAILED;
4793             }
4794           *port_name++ = 0;           /* Terminate server name */
4795
4796           if (isdigit(*port_name))
4797             {
4798             uschar *end;
4799             port = Ustrtol(port_name, &end, 0);
4800             if (end != port_name + Ustrlen(port_name))
4801               {
4802               expand_string_message =
4803                 string_sprintf("invalid port number %s", port_name);
4804               goto EXPAND_FAILED;
4805               }
4806             }
4807           else
4808             {
4809             struct servent *service_info = getservbyname(CS port_name, "tcp");
4810             if (service_info == NULL)
4811               {
4812               expand_string_message = string_sprintf("unknown port \"%s\"",
4813                 port_name);
4814               goto EXPAND_FAILED;
4815               }
4816             port = ntohs(service_info->s_port);
4817             }
4818
4819           fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
4820                   timeout, NULL, &expand_string_message);
4821           callout_address = NULL;
4822           if (fd < 0)
4823               goto SOCK_FAIL;
4824           }
4825
4826         /* Handle a Unix domain socket */
4827
4828         else
4829           {
4830           int rc;
4831           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
4832             {
4833             expand_string_message = string_sprintf("failed to create socket: %s",
4834               strerror(errno));
4835             goto SOCK_FAIL;
4836             }
4837
4838           sockun.sun_family = AF_UNIX;
4839           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
4840             sub_arg[0]);
4841
4842           sigalrm_seen = FALSE;
4843           alarm(timeout);
4844           rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun));
4845           alarm(0);
4846           if (sigalrm_seen)
4847             {
4848             expand_string_message = US "socket connect timed out";
4849             goto SOCK_FAIL;
4850             }
4851           if (rc < 0)
4852             {
4853             expand_string_message = string_sprintf("failed to connect to socket "
4854               "%s: %s", sub_arg[0], strerror(errno));
4855             goto SOCK_FAIL;
4856             }
4857           }
4858
4859         DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]);
4860
4861         /* Allow sequencing of test actions */
4862         if (running_in_test_harness) millisleep(100);
4863
4864         /* Write the request string, if not empty */
4865
4866         if (sub_arg[1][0] != 0)
4867           {
4868           int len = Ustrlen(sub_arg[1]);
4869           DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n",
4870             sub_arg[1]);
4871           if (write(fd, sub_arg[1], len) != len)
4872             {
4873             expand_string_message = string_sprintf("request write to socket "
4874               "failed: %s", strerror(errno));
4875             goto SOCK_FAIL;
4876             }
4877           }
4878
4879         /* Shut down the sending side of the socket. This helps some servers to
4880         recognise that it is their turn to do some work. Just in case some
4881         system doesn't have this function, make it conditional. */
4882
4883 #ifdef SHUT_WR
4884         if (do_shutdown) shutdown(fd, SHUT_WR);
4885 #endif
4886
4887         if (running_in_test_harness) millisleep(100);
4888
4889         /* Now we need to read from the socket, under a timeout. The function
4890         that reads a file can be used. */
4891
4892         f = fdopen(fd, "rb");
4893         sigalrm_seen = FALSE;
4894         alarm(timeout);
4895         yield = cat_file(f, yield, &size, &ptr, sub_arg[3]);
4896         alarm(0);
4897         (void)fclose(f);
4898
4899         /* After a timeout, we restore the pointer in the result, that is,
4900         make sure we add nothing from the socket. */
4901
4902         if (sigalrm_seen)
4903           {
4904           ptr = save_ptr;
4905           expand_string_message = US "socket read timed out";
4906           goto SOCK_FAIL;
4907           }
4908         }
4909
4910       /* The whole thing has worked (or we were skipping). If there is a
4911       failure string following, we need to skip it. */
4912
4913       if (*s == '{')
4914         {
4915         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
4916           goto EXPAND_FAILED;
4917         if (*s++ != '}')
4918           {
4919           expand_string_message = US"missing '}' closing failstring for readsocket";
4920           goto EXPAND_FAILED_CURLY;
4921           }
4922         while (isspace(*s)) s++;
4923         }
4924
4925     READSOCK_DONE:
4926       if (*s++ != '}')
4927         {
4928         expand_string_message = US"missing '}' closing readsocket";
4929         goto EXPAND_FAILED_CURLY;
4930         }
4931       continue;
4932
4933       /* Come here on failure to create socket, connect socket, write to the
4934       socket, or timeout on reading. If another substring follows, expand and
4935       use it. Otherwise, those conditions give expand errors. */
4936
4937     SOCK_FAIL:
4938       if (*s != '{') goto EXPAND_FAILED;
4939       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
4940       if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok)))
4941         goto EXPAND_FAILED;
4942       yield = string_cat(yield, &size, &ptr, arg);
4943       if (*s++ != '}')
4944         {
4945         expand_string_message = US"missing '}' closing failstring for readsocket";
4946         goto EXPAND_FAILED_CURLY;
4947         }
4948       while (isspace(*s)) s++;
4949       goto READSOCK_DONE;
4950       }
4951
4952     /* Handle "run" to execute a program. */
4953
4954     case EITEM_RUN:
4955       {
4956       FILE *f;
4957       uschar *arg;
4958       const uschar **argv;
4959       pid_t pid;
4960       int fd_in, fd_out;
4961       int lsize = 0, lptr = 0;
4962
4963       if ((expand_forbid & RDO_RUN) != 0)
4964         {
4965         expand_string_message = US"running a command is not permitted";
4966         goto EXPAND_FAILED;
4967         }
4968
4969       while (isspace(*s)) s++;
4970       if (*s != '{')
4971         {
4972         expand_string_message = US"missing '{' for command arg of run";
4973         goto EXPAND_FAILED_CURLY;
4974         }
4975       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4976       if (arg == NULL) goto EXPAND_FAILED;
4977       while (isspace(*s)) s++;
4978       if (*s++ != '}')
4979         {
4980         expand_string_message = US"missing '}' closing command arg of run";
4981         goto EXPAND_FAILED_CURLY;
4982         }
4983
4984       if (skipping)   /* Just pretend it worked when we're skipping */
4985         {
4986         runrc = 0;
4987         lookup_value = NULL;
4988         }
4989       else
4990         {
4991         if (!transport_set_up_command(&argv,    /* anchor for arg list */
4992             arg,                                /* raw command */
4993             FALSE,                              /* don't expand the arguments */
4994             0,                                  /* not relevant when... */
4995             NULL,                               /* no transporting address */
4996             US"${run} expansion",               /* for error messages */
4997             &expand_string_message))            /* where to put error message */
4998           goto EXPAND_FAILED;
4999
5000         /* Create the child process, making it a group leader. */
5001
5002         if ((pid = child_open(USS argv, NULL, 0077, &fd_in, &fd_out, TRUE)) < 0)
5003           {
5004           expand_string_message =
5005             string_sprintf("couldn't create child process: %s", strerror(errno));
5006           goto EXPAND_FAILED;
5007           }
5008
5009         /* Nothing is written to the standard input. */
5010
5011         (void)close(fd_in);
5012
5013         /* Read the pipe to get the command's output into $value (which is kept
5014         in lookup_value). Read during execution, so that if the output exceeds
5015         the OS pipe buffer limit, we don't block forever. Remember to not release
5016         memory just allocated for $value. */
5017
5018         resetok = FALSE;
5019         f = fdopen(fd_out, "rb");
5020         sigalrm_seen = FALSE;
5021         alarm(60);
5022         lookup_value = cat_file(f, NULL, &lsize, &lptr, NULL);
5023         alarm(0);
5024         (void)fclose(f);
5025
5026         /* Wait for the process to finish, applying the timeout, and inspect its
5027         return code for serious disasters. Simple non-zero returns are passed on.
5028         */
5029
5030         if (sigalrm_seen || (runrc = child_close(pid, 30)) < 0)
5031           {
5032           if (sigalrm_seen || runrc == -256)
5033             {
5034             expand_string_message = string_sprintf("command timed out");
5035             killpg(pid, SIGKILL);       /* Kill the whole process group */
5036             }
5037
5038           else if (runrc == -257)
5039             expand_string_message = string_sprintf("wait() failed: %s",
5040               strerror(errno));
5041
5042           else
5043             expand_string_message = string_sprintf("command killed by signal %d",
5044               -runrc);
5045
5046           goto EXPAND_FAILED;
5047           }
5048         }
5049
5050       /* Process the yes/no strings; $value may be useful in both cases */
5051
5052       switch(process_yesno(
5053                skipping,                     /* were previously skipping */
5054                runrc == 0,                   /* success/failure indicator */
5055                lookup_value,                 /* value to reset for string2 */
5056                &s,                           /* input pointer */
5057                &yield,                       /* output pointer */
5058                &size,                        /* output size */
5059                &ptr,                         /* output current point */
5060                US"run",                      /* condition type */
5061                &resetok))
5062         {
5063         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5064         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5065         }
5066
5067       continue;
5068       }
5069
5070     /* Handle character translation for "tr" */
5071
5072     case EITEM_TR:
5073       {
5074       int oldptr = ptr;
5075       int o2m;
5076       uschar *sub[3];
5077
5078       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
5079         {
5080         case 1: goto EXPAND_FAILED_CURLY;
5081         case 2:
5082         case 3: goto EXPAND_FAILED;
5083         }
5084
5085       yield = string_cat(yield, &size, &ptr, sub[0]);
5086       o2m = Ustrlen(sub[2]) - 1;
5087
5088       if (o2m >= 0) for (; oldptr < ptr; oldptr++)
5089         {
5090         uschar *m = Ustrrchr(sub[1], yield[oldptr]);
5091         if (m != NULL)
5092           {
5093           int o = m - sub[1];
5094           yield[oldptr] = sub[2][(o < o2m)? o : o2m];
5095           }
5096         }
5097
5098       continue;
5099       }
5100
5101     /* Handle "hash", "length", "nhash", and "substr" when they are given with
5102     expanded arguments. */
5103
5104     case EITEM_HASH:
5105     case EITEM_LENGTH:
5106     case EITEM_NHASH:
5107     case EITEM_SUBSTR:
5108       {
5109       int i;
5110       int len;
5111       uschar *ret;
5112       int val[2] = { 0, -1 };
5113       uschar *sub[3];
5114
5115       /* "length" takes only 2 arguments whereas the others take 2 or 3.
5116       Ensure that sub[2] is set in the ${length } case. */
5117
5118       sub[2] = NULL;
5119       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
5120              TRUE, name, &resetok))
5121         {
5122         case 1: goto EXPAND_FAILED_CURLY;
5123         case 2:
5124         case 3: goto EXPAND_FAILED;
5125         }
5126
5127       /* Juggle the arguments if there are only two of them: always move the
5128       string to the last position and make ${length{n}{str}} equivalent to
5129       ${substr{0}{n}{str}}. See the defaults for val[] above. */
5130
5131       if (sub[2] == NULL)
5132         {
5133         sub[2] = sub[1];
5134         sub[1] = NULL;
5135         if (item_type == EITEM_LENGTH)
5136           {
5137           sub[1] = sub[0];
5138           sub[0] = NULL;
5139           }
5140         }
5141
5142       for (i = 0; i < 2; i++)
5143         {
5144         if (sub[i] == NULL) continue;
5145         val[i] = (int)Ustrtol(sub[i], &ret, 10);
5146         if (*ret != 0 || (i != 0 && val[i] < 0))
5147           {
5148           expand_string_message = string_sprintf("\"%s\" is not a%s number "
5149             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
5150           goto EXPAND_FAILED;
5151           }
5152         }
5153
5154       ret =
5155         (item_type == EITEM_HASH)?
5156           compute_hash(sub[2], val[0], val[1], &len) :
5157         (item_type == EITEM_NHASH)?
5158           compute_nhash(sub[2], val[0], val[1], &len) :
5159           extract_substr(sub[2], val[0], val[1], &len);
5160
5161       if (ret == NULL) goto EXPAND_FAILED;
5162       yield = string_catn(yield, &size, &ptr, ret, len);
5163       continue;
5164       }
5165
5166     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
5167     This code originally contributed by Steve Haslam. It currently supports
5168     the use of MD5 and SHA-1 hashes.
5169
5170     We need some workspace that is large enough to handle all the supported
5171     hash types. Use macros to set the sizes rather than be too elaborate. */
5172
5173     #define MAX_HASHLEN      20
5174     #define MAX_HASHBLOCKLEN 64
5175
5176     case EITEM_HMAC:
5177       {
5178       uschar *sub[3];
5179       md5 md5_base;
5180       hctx sha1_ctx;
5181       void *use_base;
5182       int type, i;
5183       int hashlen;      /* Number of octets for the hash algorithm's output */
5184       int hashblocklen; /* Number of octets the hash algorithm processes */
5185       uschar *keyptr, *p;
5186       unsigned int keylen;
5187
5188       uschar keyhash[MAX_HASHLEN];
5189       uschar innerhash[MAX_HASHLEN];
5190       uschar finalhash[MAX_HASHLEN];
5191       uschar finalhash_hex[2*MAX_HASHLEN];
5192       uschar innerkey[MAX_HASHBLOCKLEN];
5193       uschar outerkey[MAX_HASHBLOCKLEN];
5194
5195       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
5196         {
5197         case 1: goto EXPAND_FAILED_CURLY;
5198         case 2:
5199         case 3: goto EXPAND_FAILED;
5200         }
5201
5202       if (!skipping)
5203         {
5204         if (Ustrcmp(sub[0], "md5") == 0)
5205           {
5206           type = HMAC_MD5;
5207           use_base = &md5_base;
5208           hashlen = 16;
5209           hashblocklen = 64;
5210           }
5211         else if (Ustrcmp(sub[0], "sha1") == 0)
5212           {
5213           type = HMAC_SHA1;
5214           use_base = &sha1_ctx;
5215           hashlen = 20;
5216           hashblocklen = 64;
5217           }
5218         else
5219           {
5220           expand_string_message =
5221             string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
5222           goto EXPAND_FAILED;
5223           }
5224
5225         keyptr = sub[1];
5226         keylen = Ustrlen(keyptr);
5227
5228         /* If the key is longer than the hash block length, then hash the key
5229         first */
5230
5231         if (keylen > hashblocklen)
5232           {
5233           chash_start(type, use_base);
5234           chash_end(type, use_base, keyptr, keylen, keyhash);
5235           keyptr = keyhash;
5236           keylen = hashlen;
5237           }
5238
5239         /* Now make the inner and outer key values */
5240
5241         memset(innerkey, 0x36, hashblocklen);
5242         memset(outerkey, 0x5c, hashblocklen);
5243
5244         for (i = 0; i < keylen; i++)
5245           {
5246           innerkey[i] ^= keyptr[i];
5247           outerkey[i] ^= keyptr[i];
5248           }
5249
5250         /* Now do the hashes */
5251
5252         chash_start(type, use_base);
5253         chash_mid(type, use_base, innerkey);
5254         chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5255
5256         chash_start(type, use_base);
5257         chash_mid(type, use_base, outerkey);
5258         chash_end(type, use_base, innerhash, hashlen, finalhash);
5259
5260         /* Encode the final hash as a hex string */
5261
5262         p = finalhash_hex;
5263         for (i = 0; i < hashlen; i++)
5264           {
5265           *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5266           *p++ = hex_digits[finalhash[i] & 0x0f];
5267           }
5268
5269         DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%s)=%.*s\n",
5270           sub[0], (int)keylen, keyptr, sub[2], hashlen*2, finalhash_hex);
5271
5272         yield = string_catn(yield, &size, &ptr, finalhash_hex, hashlen*2);
5273         }
5274       continue;
5275       }
5276
5277     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5278     We have to save the numerical variables and restore them afterwards. */
5279
5280     case EITEM_SG:
5281       {
5282       const pcre *re;
5283       int moffset, moffsetextra, slen;
5284       int roffset;
5285       int emptyopt;
5286       const uschar *rerror;
5287       uschar *subject;
5288       uschar *sub[3];
5289       int save_expand_nmax =
5290         save_expand_strings(save_expand_nstring, save_expand_nlength);
5291
5292       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5293         {
5294         case 1: goto EXPAND_FAILED_CURLY;
5295         case 2:
5296         case 3: goto EXPAND_FAILED;
5297         }
5298
5299       /* Compile the regular expression */
5300
5301       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5302         NULL);
5303
5304       if (re == NULL)
5305         {
5306         expand_string_message = string_sprintf("regular expression error in "
5307           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5308         goto EXPAND_FAILED;
5309         }
5310
5311       /* Now run a loop to do the substitutions as often as necessary. It ends
5312       when there are no more matches. Take care over matches of the null string;
5313       do the same thing as Perl does. */
5314
5315       subject = sub[0];
5316       slen = Ustrlen(sub[0]);
5317       moffset = moffsetextra = 0;
5318       emptyopt = 0;
5319
5320       for (;;)
5321         {
5322         int ovector[3*(EXPAND_MAXN+1)];
5323         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5324           PCRE_EOPT | emptyopt, ovector, nelem(ovector));
5325         int nn;
5326         uschar *insert;
5327
5328         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5329         is not necessarily the end. We want to repeat the match from one
5330         character further along, but leaving the basic offset the same (for
5331         copying below). We can't be at the end of the string - that was checked
5332         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5333         finished; copy the remaining string and end the loop. */
5334
5335         if (n < 0)
5336           {
5337           if (emptyopt != 0)
5338             {
5339             moffsetextra = 1;
5340             emptyopt = 0;
5341             continue;
5342             }
5343           yield = string_catn(yield, &size, &ptr, subject+moffset, slen-moffset);
5344           break;
5345           }
5346
5347         /* Match - set up for expanding the replacement. */
5348
5349         if (n == 0) n = EXPAND_MAXN + 1;
5350         expand_nmax = 0;
5351         for (nn = 0; nn < n*2; nn += 2)
5352           {
5353           expand_nstring[expand_nmax] = subject + ovector[nn];
5354           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5355           }
5356         expand_nmax--;
5357
5358         /* Copy the characters before the match, plus the expanded insertion. */
5359
5360         yield = string_catn(yield, &size, &ptr, subject + moffset,
5361           ovector[0] - moffset);
5362         insert = expand_string(sub[2]);
5363         if (insert == NULL) goto EXPAND_FAILED;
5364         yield = string_cat(yield, &size, &ptr, insert);
5365
5366         moffset = ovector[1];
5367         moffsetextra = 0;
5368         emptyopt = 0;
5369
5370         /* If we have matched an empty string, first check to see if we are at
5371         the end of the subject. If so, the loop is over. Otherwise, mimic
5372         what Perl's /g options does. This turns out to be rather cunning. First
5373         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5374         string at the same point. If this fails (picked up above) we advance to
5375         the next character. */
5376
5377         if (ovector[0] == ovector[1])
5378           {
5379           if (ovector[0] == slen) break;
5380           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5381           }
5382         }
5383
5384       /* All done - restore numerical variables. */
5385
5386       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5387         save_expand_nlength);
5388       continue;
5389       }
5390
5391     /* Handle keyed and numbered substring extraction. If the first argument
5392     consists entirely of digits, then a numerical extraction is assumed. */
5393
5394     case EITEM_EXTRACT:
5395       {
5396       int i;
5397       int j;
5398       int field_number = 1;
5399       BOOL field_number_set = FALSE;
5400       uschar *save_lookup_value = lookup_value;
5401       uschar *sub[3];
5402       int save_expand_nmax =
5403         save_expand_strings(save_expand_nstring, save_expand_nlength);
5404
5405       /* While skipping we cannot rely on the data for expansions being
5406       available (eg. $item) hence cannot decide on numeric vs. keyed.
5407       Read a maximum of 5 arguments (including the yes/no) */
5408
5409       if (skipping)
5410         {
5411         while (isspace(*s)) s++;
5412         for (j = 5; j > 0 && *s == '{'; j--)
5413           {
5414           if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))
5415             goto EXPAND_FAILED;                                 /*{*/
5416           if (*s++ != '}')
5417             {
5418             expand_string_message = US"missing '{' for arg of extract";
5419             goto EXPAND_FAILED_CURLY;
5420             }
5421           while (isspace(*s)) s++;
5422           }
5423         if (  Ustrncmp(s, "fail", 4) == 0
5424            && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4])
5425            )
5426           {
5427           s += 4;
5428           while (isspace(*s)) s++;
5429           }
5430         if (*s != '}')
5431           {
5432           expand_string_message = US"missing '}' closing extract";
5433           goto EXPAND_FAILED_CURLY;
5434           }
5435         }
5436
5437       else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */
5438         {
5439         while (isspace(*s)) s++;
5440         if (*s == '{')                                          /*}*/
5441           {
5442           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5443           if (sub[i] == NULL) goto EXPAND_FAILED;               /*{*/
5444           if (*s++ != '}')
5445             {
5446             expand_string_message = string_sprintf(
5447               "missing '}' closing arg %d of extract", i+1);
5448             goto EXPAND_FAILED_CURLY;
5449             }
5450
5451           /* After removal of leading and trailing white space, the first
5452           argument must not be empty; if it consists entirely of digits
5453           (optionally preceded by a minus sign), this is a numerical
5454           extraction, and we expect 3 arguments. */
5455
5456           if (i == 0)
5457             {
5458             int len;
5459             int x = 0;
5460             uschar *p = sub[0];
5461
5462             while (isspace(*p)) p++;
5463             sub[0] = p;
5464
5465             len = Ustrlen(p);
5466             while (len > 0 && isspace(p[len-1])) len--;
5467             p[len] = 0;
5468
5469             if (*p == 0)
5470               {
5471               expand_string_message = US"first argument of \"extract\" must "
5472                 "not be empty";
5473               goto EXPAND_FAILED;
5474               }
5475
5476             if (*p == '-')
5477               {
5478               field_number = -1;
5479               p++;
5480               }
5481             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5482             if (*p == 0)
5483               {
5484               field_number *= x;
5485               j = 3;               /* Need 3 args */
5486               field_number_set = TRUE;
5487               }
5488             }
5489           }
5490         else
5491           {
5492           expand_string_message = string_sprintf(
5493             "missing '{' for arg %d of extract", i+1);
5494           goto EXPAND_FAILED_CURLY;
5495           }
5496         }
5497
5498       /* Extract either the numbered or the keyed substring into $value. If
5499       skipping, just pretend the extraction failed. */
5500
5501       lookup_value = skipping? NULL : field_number_set?
5502         expand_gettokened(field_number, sub[1], sub[2]) :
5503         expand_getkeyed(sub[0], sub[1]);
5504
5505       /* If no string follows, $value gets substituted; otherwise there can
5506       be yes/no strings, as for lookup or if. */
5507
5508       switch(process_yesno(
5509                skipping,                     /* were previously skipping */
5510                lookup_value != NULL,         /* success/failure indicator */
5511                save_lookup_value,            /* value to reset for string2 */
5512                &s,                           /* input pointer */
5513                &yield,                       /* output pointer */
5514                &size,                        /* output size */
5515                &ptr,                         /* output current point */
5516                US"extract",                  /* condition type */
5517                &resetok))
5518         {
5519         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5520         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5521         }
5522
5523       /* All done - restore numerical variables. */
5524
5525       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5526         save_expand_nlength);
5527
5528       continue;
5529       }
5530
5531     /* return the Nth item from a list */
5532
5533     case EITEM_LISTEXTRACT:
5534       {
5535       int i;
5536       int field_number = 1;
5537       uschar *save_lookup_value = lookup_value;
5538       uschar *sub[2];
5539       int save_expand_nmax =
5540         save_expand_strings(save_expand_nstring, save_expand_nlength);
5541
5542       /* Read the field & list arguments */
5543
5544       for (i = 0; i < 2; i++)
5545         {
5546         while (isspace(*s)) s++;
5547         if (*s != '{')                                  /*}*/
5548           {
5549           expand_string_message = string_sprintf(
5550             "missing '{' for arg %d of listextract", i+1);
5551           goto EXPAND_FAILED_CURLY;
5552           }
5553
5554         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5555         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5556         if (*s++ != '}')
5557           {
5558           expand_string_message = string_sprintf(
5559             "missing '}' closing arg %d of listextract", i+1);
5560           goto EXPAND_FAILED_CURLY;
5561           }
5562
5563         /* After removal of leading and trailing white space, the first
5564         argument must be numeric and nonempty. */
5565
5566         if (i == 0)
5567           {
5568           int len;
5569           int x = 0;
5570           uschar *p = sub[0];
5571
5572           while (isspace(*p)) p++;
5573           sub[0] = p;
5574
5575           len = Ustrlen(p);
5576           while (len > 0 && isspace(p[len-1])) len--;
5577           p[len] = 0;
5578
5579           if (!*p && !skipping)
5580             {
5581             expand_string_message = US"first argument of \"listextract\" must "
5582               "not be empty";
5583             goto EXPAND_FAILED;
5584             }
5585
5586           if (*p == '-')
5587             {
5588             field_number = -1;
5589             p++;
5590             }
5591           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5592           if (*p)
5593             {
5594             expand_string_message = US"first argument of \"listextract\" must "
5595               "be numeric";
5596             goto EXPAND_FAILED;
5597             }
5598           field_number *= x;
5599           }
5600         }
5601
5602       /* Extract the numbered element into $value. If
5603       skipping, just pretend the extraction failed. */
5604
5605       lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]);
5606
5607       /* If no string follows, $value gets substituted; otherwise there can
5608       be yes/no strings, as for lookup or if. */
5609
5610       switch(process_yesno(
5611                skipping,                     /* were previously skipping */
5612                lookup_value != NULL,         /* success/failure indicator */
5613                save_lookup_value,            /* value to reset for string2 */
5614                &s,                           /* input pointer */
5615                &yield,                       /* output pointer */
5616                &size,                        /* output size */
5617                &ptr,                         /* output current point */
5618                US"listextract",              /* condition type */
5619                &resetok))
5620         {
5621         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5622         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5623         }
5624
5625       /* All done - restore numerical variables. */
5626
5627       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5628         save_expand_nlength);
5629
5630       continue;
5631       }
5632
5633 #ifdef SUPPORT_TLS
5634     case EITEM_CERTEXTRACT:
5635       {
5636       uschar *save_lookup_value = lookup_value;
5637       uschar *sub[2];
5638       int save_expand_nmax =
5639         save_expand_strings(save_expand_nstring, save_expand_nlength);
5640
5641       /* Read the field argument */
5642       while (isspace(*s)) s++;
5643       if (*s != '{')                                    /*}*/
5644         {
5645         expand_string_message = US"missing '{' for field arg of certextract";
5646         goto EXPAND_FAILED_CURLY;
5647         }
5648       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5649       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
5650       if (*s++ != '}')
5651         {
5652         expand_string_message = US"missing '}' closing field arg of certextract";
5653         goto EXPAND_FAILED_CURLY;
5654         }
5655       /* strip spaces fore & aft */
5656       {
5657       int len;
5658       uschar *p = sub[0];
5659
5660       while (isspace(*p)) p++;
5661       sub[0] = p;
5662
5663       len = Ustrlen(p);
5664       while (len > 0 && isspace(p[len-1])) len--;
5665       p[len] = 0;
5666       }
5667
5668       /* inspect the cert argument */
5669       while (isspace(*s)) s++;
5670       if (*s != '{')                                    /*}*/
5671         {
5672         expand_string_message = US"missing '{' for cert variable arg of certextract";
5673         goto EXPAND_FAILED_CURLY;
5674         }
5675       if (*++s != '$')
5676         {
5677         expand_string_message = US"second argument of \"certextract\" must "
5678           "be a certificate variable";
5679         goto EXPAND_FAILED;
5680         }
5681       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
5682       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
5683       if (*s++ != '}')
5684         {
5685         expand_string_message = US"missing '}' closing cert variable arg of certextract";
5686         goto EXPAND_FAILED_CURLY;
5687         }
5688
5689       if (skipping)
5690         lookup_value = NULL;
5691       else
5692         {
5693         lookup_value = expand_getcertele(sub[0], sub[1]);
5694         if (*expand_string_message) goto EXPAND_FAILED;
5695         }
5696       switch(process_yesno(
5697                skipping,                     /* were previously skipping */
5698                lookup_value != NULL,         /* success/failure indicator */
5699                save_lookup_value,            /* value to reset for string2 */
5700                &s,                           /* input pointer */
5701                &yield,                       /* output pointer */
5702                &size,                        /* output size */
5703                &ptr,                         /* output current point */
5704                US"certextract",              /* condition type */
5705                &resetok))
5706         {
5707         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5708         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5709         }
5710
5711       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5712         save_expand_nlength);
5713       continue;
5714       }
5715 #endif  /*SUPPORT_TLS*/
5716
5717     /* Handle list operations */
5718
5719     case EITEM_FILTER:
5720     case EITEM_MAP:
5721     case EITEM_REDUCE:
5722       {
5723       int sep = 0;
5724       int save_ptr = ptr;
5725       uschar outsep[2] = { '\0', '\0' };
5726       const uschar *list, *expr, *temp;
5727       uschar *save_iterate_item = iterate_item;
5728       uschar *save_lookup_value = lookup_value;
5729
5730       while (isspace(*s)) s++;
5731       if (*s++ != '{')
5732         {
5733         expand_string_message =
5734           string_sprintf("missing '{' for first arg of %s", name);
5735         goto EXPAND_FAILED_CURLY;
5736         }
5737
5738       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5739       if (list == NULL) goto EXPAND_FAILED;
5740       if (*s++ != '}')
5741         {
5742         expand_string_message =
5743           string_sprintf("missing '}' closing first arg of %s", name);
5744         goto EXPAND_FAILED_CURLY;
5745         }
5746
5747       if (item_type == EITEM_REDUCE)
5748         {
5749         uschar * t;
5750         while (isspace(*s)) s++;
5751         if (*s++ != '{')
5752           {
5753           expand_string_message = US"missing '{' for second arg of reduce";
5754           goto EXPAND_FAILED_CURLY;
5755           }
5756         t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5757         if (!t) goto EXPAND_FAILED;
5758         lookup_value = t;
5759         if (*s++ != '}')
5760           {
5761           expand_string_message = US"missing '}' closing second arg of reduce";
5762           goto EXPAND_FAILED_CURLY;
5763           }
5764         }
5765
5766       while (isspace(*s)) s++;
5767       if (*s++ != '{')
5768         {
5769         expand_string_message =
5770           string_sprintf("missing '{' for last arg of %s", name);
5771         goto EXPAND_FAILED_CURLY;
5772         }
5773
5774       expr = s;
5775
5776       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
5777       if scanning a "false" part). This allows us to find the end of the
5778       condition, because if the list is empty, we won't actually evaluate the
5779       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
5780       the normal internal expansion function. */
5781
5782       if (item_type == EITEM_FILTER)
5783         {
5784         temp = eval_condition(expr, &resetok, NULL);
5785         if (temp != NULL) s = temp;
5786         }
5787       else
5788         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5789
5790       if (temp == NULL)
5791         {
5792         expand_string_message = string_sprintf("%s inside \"%s\" item",
5793           expand_string_message, name);
5794         goto EXPAND_FAILED;
5795         }
5796
5797       while (isspace(*s)) s++;
5798       if (*s++ != '}')
5799         {                                               /*{*/
5800         expand_string_message = string_sprintf("missing } at end of condition "
5801           "or expression inside \"%s\"", name);
5802         goto EXPAND_FAILED;
5803         }
5804
5805       while (isspace(*s)) s++;                          /*{*/
5806       if (*s++ != '}')
5807         {                                               /*{*/
5808         expand_string_message = string_sprintf("missing } at end of \"%s\"",
5809           name);
5810         goto EXPAND_FAILED;
5811         }
5812
5813       /* If we are skipping, we can now just move on to the next item. When
5814       processing for real, we perform the iteration. */
5815
5816       if (skipping) continue;
5817       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
5818         {
5819         *outsep = (uschar)sep;      /* Separator as a string */
5820
5821         DEBUG(D_expand) debug_printf_indent("%s: $item = '%s'  $value = '%s'\n",
5822                           name, iterate_item, lookup_value);
5823
5824         if (item_type == EITEM_FILTER)
5825           {
5826           BOOL condresult;
5827           if (eval_condition(expr, &resetok, &condresult) == NULL)
5828             {
5829             iterate_item = save_iterate_item;
5830             lookup_value = save_lookup_value;
5831             expand_string_message = string_sprintf("%s inside \"%s\" condition",
5832               expand_string_message, name);
5833             goto EXPAND_FAILED;
5834             }
5835           DEBUG(D_expand) debug_printf_indent("%s: condition is %s\n", name,
5836             condresult? "true":"false");
5837           if (condresult)
5838             temp = iterate_item;    /* TRUE => include this item */
5839           else
5840             continue;               /* FALSE => skip this item */
5841           }
5842
5843         /* EITEM_MAP and EITEM_REDUCE */
5844
5845         else
5846           {
5847           uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
5848           temp = t;
5849           if (temp == NULL)
5850             {
5851             iterate_item = save_iterate_item;
5852             expand_string_message = string_sprintf("%s inside \"%s\" item",
5853               expand_string_message, name);
5854             goto EXPAND_FAILED;
5855             }
5856           if (item_type == EITEM_REDUCE)
5857             {
5858             lookup_value = t;         /* Update the value of $value */
5859             continue;                 /* and continue the iteration */
5860             }
5861           }
5862
5863         /* We reach here for FILTER if the condition is true, always for MAP,
5864         and never for REDUCE. The value in "temp" is to be added to the output
5865         list that is being created, ensuring that any occurrences of the
5866         separator character are doubled. Unless we are dealing with the first
5867         item of the output list, add in a space if the new item begins with the
5868         separator character, or is an empty string. */
5869
5870         if (ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
5871           yield = string_catn(yield, &size, &ptr, US" ", 1);
5872
5873         /* Add the string in "temp" to the output list that we are building,
5874         This is done in chunks by searching for the separator character. */
5875
5876         for (;;)
5877           {
5878           size_t seglen = Ustrcspn(temp, outsep);
5879
5880           yield = string_catn(yield, &size, &ptr, temp, seglen + 1);
5881
5882           /* If we got to the end of the string we output one character
5883           too many; backup and end the loop. Otherwise arrange to double the
5884           separator. */
5885
5886           if (temp[seglen] == '\0') { ptr--; break; }
5887           yield = string_catn(yield, &size, &ptr, outsep, 1);
5888           temp += seglen + 1;
5889           }
5890
5891         /* Output a separator after the string: we will remove the redundant
5892         final one at the end. */
5893
5894         yield = string_catn(yield, &size, &ptr, outsep, 1);
5895         }   /* End of iteration over the list loop */
5896
5897       /* REDUCE has generated no output above: output the final value of
5898       $value. */
5899
5900       if (item_type == EITEM_REDUCE)
5901         {
5902         yield = string_cat(yield, &size, &ptr, lookup_value);
5903         lookup_value = save_lookup_value;  /* Restore $value */
5904         }
5905
5906       /* FILTER and MAP generate lists: if they have generated anything, remove
5907       the redundant final separator. Even though an empty item at the end of a
5908       list does not count, this is tidier. */
5909
5910       else if (ptr != save_ptr) ptr--;
5911
5912       /* Restore preserved $item */
5913
5914       iterate_item = save_iterate_item;
5915       continue;
5916       }
5917
5918     case EITEM_SORT:
5919       {
5920       int sep = 0;
5921       const uschar *srclist, *cmp, *xtract;
5922       uschar *srcitem;
5923       const uschar *dstlist = NULL, *dstkeylist = NULL;
5924       uschar * tmp;
5925       uschar *save_iterate_item = iterate_item;
5926
5927       while (isspace(*s)) s++;
5928       if (*s++ != '{')
5929         {
5930         expand_string_message = US"missing '{' for list arg of sort";
5931         goto EXPAND_FAILED_CURLY;
5932         }
5933
5934       srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5935       if (!srclist) goto EXPAND_FAILED;
5936       if (*s++ != '}')
5937         {
5938         expand_string_message = US"missing '}' closing list arg of sort";
5939         goto EXPAND_FAILED_CURLY;
5940         }
5941
5942       while (isspace(*s)) s++;
5943       if (*s++ != '{')
5944         {
5945         expand_string_message = US"missing '{' for comparator arg of sort";
5946         goto EXPAND_FAILED_CURLY;
5947         }
5948
5949       cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok);
5950       if (!cmp) goto EXPAND_FAILED;
5951       if (*s++ != '}')
5952         {
5953         expand_string_message = US"missing '}' closing comparator arg of sort";
5954         goto EXPAND_FAILED_CURLY;
5955         }
5956
5957       while (isspace(*s)) s++;
5958       if (*s++ != '{')
5959         {
5960         expand_string_message = US"missing '{' for extractor arg of sort";
5961         goto EXPAND_FAILED_CURLY;
5962         }
5963
5964       xtract = s;
5965       tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5966       if (!tmp) goto EXPAND_FAILED;
5967       xtract = string_copyn(xtract, s - xtract);
5968
5969       if (*s++ != '}')
5970         {
5971         expand_string_message = US"missing '}' closing extractor arg of sort";
5972         goto EXPAND_FAILED_CURLY;
5973         }
5974                                                         /*{*/
5975       if (*s++ != '}')
5976         {                                               /*{*/
5977         expand_string_message = US"missing } at end of \"sort\"";
5978         goto EXPAND_FAILED;
5979         }
5980
5981       if (skipping) continue;
5982
5983       while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0)))
5984         {
5985         uschar * dstitem;
5986         uschar * newlist = NULL;
5987         int size = 0, len = 0;
5988         uschar * newkeylist = NULL;
5989         int ksize = 0, klen = 0;
5990         uschar * srcfield;
5991
5992         DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, srcitem);
5993
5994         /* extract field for comparisons */
5995         iterate_item = srcitem;
5996         if (  !(srcfield = expand_string_internal(xtract, FALSE, NULL, FALSE,
5997                                           TRUE, &resetok))
5998            || !*srcfield)
5999           {
6000           expand_string_message = string_sprintf(
6001               "field-extract in sort: \"%s\"", xtract);
6002           goto EXPAND_FAILED;
6003           }
6004
6005         /* Insertion sort */
6006
6007         /* copy output list until new-item < list-item */
6008         while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6009           {
6010           uschar * dstfield;
6011           uschar * expr;
6012           BOOL before;
6013
6014           /* field for comparison */
6015           if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6016             goto sort_mismatch;
6017
6018           /* build and run condition string */
6019           expr = string_sprintf("%s{%s}{%s}", cmp, srcfield, dstfield);
6020
6021           DEBUG(D_expand) debug_printf_indent("%s: cond = \"%s\"\n", name, expr);
6022           if (!eval_condition(expr, &resetok, &before))
6023             {
6024             expand_string_message = string_sprintf("comparison in sort: %s",
6025                 expr);
6026             goto EXPAND_FAILED;
6027             }
6028
6029           if (before)
6030             {
6031             /* New-item sorts before this dst-item.  Append new-item,
6032             then dst-item, then remainder of dst list. */
6033
6034             newlist = string_append_listele(newlist, &size, &len, sep, srcitem);
6035             newkeylist = string_append_listele(newkeylist, &ksize, &klen, sep, srcfield);
6036             srcitem = NULL;
6037
6038             newlist = string_append_listele(newlist, &size, &len, sep, dstitem);
6039             newkeylist = string_append_listele(newkeylist, &ksize, &klen, sep, dstfield);
6040
6041             while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6042               {
6043               if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6044                 goto sort_mismatch;
6045               newlist = string_append_listele(newlist, &size, &len, sep, dstitem);
6046               newkeylist = string_append_listele(newkeylist, &ksize, &klen, sep, dstfield);
6047               }
6048
6049             break;
6050             }
6051
6052           newlist = string_append_listele(newlist, &size, &len, sep, dstitem);
6053           newkeylist = string_append_listele(newkeylist, &ksize, &klen, sep, dstfield);
6054           }
6055
6056         /* If we ran out of dstlist without consuming srcitem, append it */
6057         if (srcitem)
6058           {
6059           newlist = string_append_listele(newlist, &size, &len, sep, srcitem);
6060           newkeylist = string_append_listele(newkeylist, &ksize, &klen, sep, srcfield);
6061           }
6062
6063         dstlist = newlist;
6064         dstkeylist = newkeylist;
6065
6066         DEBUG(D_expand) debug_printf_indent("%s: dstlist = \"%s\"\n", name, dstlist);
6067         DEBUG(D_expand) debug_printf_indent("%s: dstkeylist = \"%s\"\n", name, dstkeylist);
6068         }
6069
6070       if (dstlist)
6071         yield = string_cat(yield, &size, &ptr, dstlist);
6072
6073       /* Restore preserved $item */
6074       iterate_item = save_iterate_item;
6075       continue;
6076
6077       sort_mismatch:
6078         expand_string_message = US"Internal error in sort (list mismatch)";
6079         goto EXPAND_FAILED;
6080       }
6081
6082
6083     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
6084     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
6085     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
6086     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
6087
6088     #define EXPAND_DLFUNC_MAX_ARGS 8
6089
6090     case EITEM_DLFUNC:
6091 #ifndef EXPAND_DLFUNC
6092       expand_string_message = US"\"${dlfunc\" encountered, but this facility "  /*}*/
6093         "is not included in this binary";
6094       goto EXPAND_FAILED;
6095
6096 #else   /* EXPAND_DLFUNC */
6097       {
6098       tree_node *t;
6099       exim_dlfunc_t *func;
6100       uschar *result;
6101       int status, argc;
6102       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
6103
6104       if ((expand_forbid & RDO_DLFUNC) != 0)
6105         {
6106         expand_string_message =
6107           US"dynamically-loaded functions are not permitted";
6108         goto EXPAND_FAILED;
6109         }
6110
6111       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
6112            TRUE, US"dlfunc", &resetok))
6113         {
6114         case 1: goto EXPAND_FAILED_CURLY;
6115         case 2:
6116         case 3: goto EXPAND_FAILED;
6117         }
6118
6119       /* If skipping, we don't actually do anything */
6120
6121       if (skipping) continue;
6122
6123       /* Look up the dynamically loaded object handle in the tree. If it isn't
6124       found, dlopen() the file and put the handle in the tree for next time. */
6125
6126       t = tree_search(dlobj_anchor, argv[0]);
6127       if (t == NULL)
6128         {
6129         void *handle = dlopen(CS argv[0], RTLD_LAZY);
6130         if (handle == NULL)
6131           {
6132           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
6133             argv[0], dlerror());
6134           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6135           goto EXPAND_FAILED;
6136           }
6137         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
6138         Ustrcpy(t->name, argv[0]);
6139         t->data.ptr = handle;
6140         (void)tree_insertnode(&dlobj_anchor, t);
6141         }
6142
6143       /* Having obtained the dynamically loaded object handle, look up the
6144       function pointer. */
6145
6146       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
6147       if (func == NULL)
6148         {
6149         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
6150           "%s", argv[1], argv[0], dlerror());
6151         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6152         goto EXPAND_FAILED;
6153         }
6154
6155       /* Call the function and work out what to do with the result. If it
6156       returns OK, we have a replacement string; if it returns DEFER then
6157       expansion has failed in a non-forced manner; if it returns FAIL then
6158       failure was forced; if it returns ERROR or any other value there's a
6159       problem, so panic slightly. In any case, assume that the function has
6160       side-effects on the store that must be preserved. */
6161
6162       resetok = FALSE;
6163       result = NULL;
6164       for (argc = 0; argv[argc] != NULL; argc++);
6165       status = func(&result, argc - 2, &argv[2]);
6166       if(status == OK)
6167         {
6168         if (result == NULL) result = US"";
6169         yield = string_cat(yield, &size, &ptr, result);
6170         continue;
6171         }
6172       else
6173         {
6174         expand_string_message = result == NULL ? US"(no message)" : result;
6175         if(status == FAIL_FORCED) expand_string_forcedfail = TRUE;
6176           else if(status != FAIL)
6177             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
6178               argv[0], argv[1], status, expand_string_message);
6179         goto EXPAND_FAILED;
6180         }
6181       }
6182 #endif /* EXPAND_DLFUNC */
6183
6184     case EITEM_ENV:     /* ${env {name} {val_if_found} {val_if_unfound}} */
6185       {
6186       uschar * key;
6187       uschar *save_lookup_value = lookup_value;
6188
6189       while (isspace(*s)) s++;
6190       if (*s != '{')                                    /*}*/
6191         goto EXPAND_FAILED;
6192
6193       key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6194       if (!key) goto EXPAND_FAILED;                     /*{*/
6195       if (*s++ != '}')
6196         {
6197         expand_string_message = US"missing '{' for name arg of env";
6198         goto EXPAND_FAILED_CURLY;
6199         }
6200
6201       lookup_value = US getenv(CS key);
6202
6203       switch(process_yesno(
6204                skipping,                     /* were previously skipping */
6205                lookup_value != NULL,         /* success/failure indicator */
6206                save_lookup_value,            /* value to reset for string2 */
6207                &s,                           /* input pointer */
6208                &yield,                       /* output pointer */
6209                &size,                        /* output size */
6210                &ptr,                         /* output current point */
6211                US"env",                      /* condition type */
6212                &resetok))
6213         {
6214         case 1: goto EXPAND_FAILED;          /* when all is well, the */
6215         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
6216         }
6217       continue;
6218       }
6219     }   /* EITEM_* switch */
6220
6221   /* Control reaches here if the name is not recognized as one of the more
6222   complicated expansion items. Check for the "operator" syntax (name terminated
6223   by a colon). Some of the operators have arguments, separated by _ from the
6224   name. */
6225
6226   if (*s == ':')
6227     {
6228     int c;
6229     uschar *arg = NULL;
6230     uschar *sub;
6231     var_entry *vp = NULL;
6232
6233     /* Owing to an historical mis-design, an underscore may be part of the
6234     operator name, or it may introduce arguments.  We therefore first scan the
6235     table of names that contain underscores. If there is no match, we cut off
6236     the arguments and then scan the main table. */
6237
6238     if ((c = chop_match(name, op_table_underscore,
6239                         nelem(op_table_underscore))) < 0)
6240       {
6241       arg = Ustrchr(name, '_');
6242       if (arg != NULL) *arg = 0;
6243       c = chop_match(name, op_table_main, nelem(op_table_main));
6244       if (c >= 0) c += nelem(op_table_underscore);
6245       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
6246       }
6247
6248     /* Deal specially with operators that might take a certificate variable
6249     as we do not want to do the usual expansion. For most, expand the string.*/
6250     switch(c)
6251       {
6252 #ifdef SUPPORT_TLS
6253       case EOP_MD5:
6254       case EOP_SHA1:
6255       case EOP_SHA256:
6256       case EOP_BASE64:
6257         if (s[1] == '$')
6258           {
6259           const uschar * s1 = s;
6260           sub = expand_string_internal(s+2, TRUE, &s1, skipping,
6261                   FALSE, &resetok);
6262           if (!sub)       goto EXPAND_FAILED;           /*{*/
6263           if (*s1 != '}')
6264             {
6265             expand_string_message =
6266               string_sprintf("missing '}' closing cert arg of %s", name);
6267             goto EXPAND_FAILED_CURLY;
6268             }
6269           if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
6270             {
6271             s = s1+1;
6272             break;
6273             }
6274           vp = NULL;
6275           }
6276         /*FALLTHROUGH*/
6277 #endif
6278       default:
6279         sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6280         if (!sub) goto EXPAND_FAILED;
6281         s++;
6282         break;
6283       }
6284
6285     /* If we are skipping, we don't need to perform the operation at all.
6286     This matters for operations like "mask", because the data may not be
6287     in the correct format when skipping. For example, the expression may test
6288     for the existence of $sender_host_address before trying to mask it. For
6289     other operations, doing them may not fail, but it is a waste of time. */
6290
6291     if (skipping && c >= 0) continue;
6292
6293     /* Otherwise, switch on the operator type */
6294
6295     switch(c)
6296       {
6297       case EOP_BASE32:
6298         {
6299         uschar *t;
6300         unsigned long int n = Ustrtoul(sub, &t, 10);
6301         uschar * s = NULL;
6302         int sz = 0, i = 0;
6303
6304         if (*t != 0)
6305           {
6306           expand_string_message = string_sprintf("argument for base32 "
6307             "operator is \"%s\", which is not a decimal number", sub);
6308           goto EXPAND_FAILED;
6309           }
6310         for ( ; n; n >>= 5)
6311           s = string_catn(s, &sz, &i, &base32_chars[n & 0x1f], 1);
6312
6313         while (i > 0) yield = string_catn(yield, &size, &ptr, &s[--i], 1);
6314         continue;
6315         }
6316
6317       case EOP_BASE32D:
6318         {
6319         uschar *tt = sub;
6320         unsigned long int n = 0;
6321         uschar * s;
6322         while (*tt)
6323           {
6324           uschar * t = Ustrchr(base32_chars, *tt++);
6325           if (t == NULL)
6326             {
6327             expand_string_message = string_sprintf("argument for base32d "
6328               "operator is \"%s\", which is not a base 32 number", sub);
6329             goto EXPAND_FAILED;
6330             }
6331           n = n * 32 + (t - base32_chars);
6332           }
6333         s = string_sprintf("%ld", n);
6334         yield = string_cat(yield, &size, &ptr, s);
6335         continue;
6336         }
6337
6338       case EOP_BASE62:
6339         {
6340         uschar *t;
6341         unsigned long int n = Ustrtoul(sub, &t, 10);
6342         if (*t != 0)
6343           {
6344           expand_string_message = string_sprintf("argument for base62 "
6345             "operator is \"%s\", which is not a decimal number", sub);
6346           goto EXPAND_FAILED;
6347           }
6348         t = string_base62(n);
6349         yield = string_cat(yield, &size, &ptr, t);
6350         continue;
6351         }
6352
6353       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
6354
6355       case EOP_BASE62D:
6356         {
6357         uschar buf[16];
6358         uschar *tt = sub;
6359         unsigned long int n = 0;
6360         while (*tt != 0)
6361           {
6362           uschar *t = Ustrchr(base62_chars, *tt++);
6363           if (t == NULL)
6364             {
6365             expand_string_message = string_sprintf("argument for base62d "
6366               "operator is \"%s\", which is not a base %d number", sub,
6367               BASE_62);
6368             goto EXPAND_FAILED;
6369             }
6370           n = n * BASE_62 + (t - base62_chars);
6371           }
6372         (void)sprintf(CS buf, "%ld", n);
6373         yield = string_cat(yield, &size, &ptr, buf);
6374         continue;
6375         }
6376
6377       case EOP_EXPAND:
6378         {
6379         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
6380         if (expanded == NULL)
6381           {
6382           expand_string_message =
6383             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
6384               expand_string_message);
6385           goto EXPAND_FAILED;
6386           }
6387         yield = string_cat(yield, &size, &ptr, expanded);
6388         continue;
6389         }
6390
6391       case EOP_LC:
6392         {
6393         int count = 0;
6394         uschar *t = sub - 1;
6395         while (*(++t) != 0) { *t = tolower(*t); count++; }
6396         yield = string_catn(yield, &size, &ptr, sub, count);
6397         continue;
6398         }
6399
6400       case EOP_UC:
6401         {
6402         int count = 0;
6403         uschar *t = sub - 1;
6404         while (*(++t) != 0) { *t = toupper(*t); count++; }
6405         yield = string_catn(yield, &size, &ptr, sub, count);
6406         continue;
6407         }
6408
6409       case EOP_MD5:
6410 #ifdef SUPPORT_TLS
6411         if (vp && *(void **)vp->value)
6412           {
6413           uschar * cp = tls_cert_fprt_md5(*(void **)vp->value);
6414           yield = string_cat(yield, &size, &ptr, cp);
6415           }
6416         else
6417 #endif
6418           {
6419           md5 base;
6420           uschar digest[16];
6421           int j;
6422           char st[33];
6423           md5_start(&base);
6424           md5_end(&base, sub, Ustrlen(sub), digest);
6425           for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]);
6426           yield = string_cat(yield, &size, &ptr, US st);
6427           }
6428         continue;
6429
6430       case EOP_SHA1:
6431 #ifdef SUPPORT_TLS
6432         if (vp && *(void **)vp->value)
6433           {
6434           uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value);
6435           yield = string_cat(yield, &size, &ptr, cp);
6436           }
6437         else
6438 #endif
6439           {
6440           hctx h;
6441           uschar digest[20];
6442           int j;
6443           char st[41];
6444           sha1_start(&h);
6445           sha1_end(&h, sub, Ustrlen(sub), digest);
6446           for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
6447           yield = string_catn(yield, &size, &ptr, US st, 40);
6448           }
6449         continue;
6450
6451       case EOP_SHA256:
6452 #ifdef EXIM_HAVE_SHA2
6453         if (vp && *(void **)vp->value)
6454           {
6455           uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value);
6456           yield = string_cat(yield, &size, &ptr, cp);
6457           }
6458         else
6459           {
6460           hctx h;
6461           blob b;
6462           char st[3];
6463
6464           if (!exim_sha_init(&h, HASH_SHA256))
6465             {
6466             expand_string_message = US"unrecognised sha256 variant";
6467             goto EXPAND_FAILED;
6468             }
6469           exim_sha_update(&h, sub, Ustrlen(sub));
6470           exim_sha_finish(&h, &b);
6471           while (b.len-- > 0)
6472             {
6473             sprintf(st, "%02X", *b.data++);
6474             yield = string_catn(yield, &size, &ptr, US st, 2);
6475             }
6476           }
6477 #else
6478           expand_string_message = US"sha256 only supported with TLS";
6479 #endif
6480         continue;
6481
6482       case EOP_SHA3:
6483 #ifdef EXIM_HAVE_SHA3
6484         {
6485         hctx h;
6486         blob b;
6487         char st[3];
6488         hashmethod m = !arg ? HASH_SHA3_256
6489           : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224
6490           : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256
6491           : Ustrcmp(arg, "384") == 0 ? HASH_SHA3_384
6492           : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
6493           : HASH_BADTYPE;
6494
6495         if (m == HASH_BADTYPE || !exim_sha_init(&h, m))
6496           {
6497           expand_string_message = US"unrecognised sha3 variant";
6498           goto EXPAND_FAILED;
6499           }
6500
6501         exim_sha_update(&h, sub, Ustrlen(sub));
6502         exim_sha_finish(&h, &b);
6503         while (b.len-- > 0)
6504           {
6505           sprintf(st, "%02X", *b.data++);
6506           yield = string_catn(yield, &size, &ptr, US st, 2);
6507           }
6508         }
6509         continue;
6510 #else
6511         expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 +";
6512         goto EXPAND_FAILED;
6513 #endif
6514
6515       /* Convert hex encoding to base64 encoding */
6516
6517       case EOP_HEX2B64:
6518         {
6519         int c = 0;
6520         int b = -1;
6521         uschar *in = sub;
6522         uschar *out = sub;
6523         uschar *enc;
6524
6525         for (enc = sub; *enc != 0; enc++)
6526           {
6527           if (!isxdigit(*enc))
6528             {
6529             expand_string_message = string_sprintf("\"%s\" is not a hex "
6530               "string", sub);
6531             goto EXPAND_FAILED;
6532             }
6533           c++;
6534           }
6535
6536         if ((c & 1) != 0)
6537           {
6538           expand_string_message = string_sprintf("\"%s\" contains an odd "
6539             "number of characters", sub);
6540           goto EXPAND_FAILED;
6541           }
6542
6543         while ((c = *in++) != 0)
6544           {
6545           if (isdigit(c)) c -= '0';
6546           else c = toupper(c) - 'A' + 10;
6547           if (b == -1)
6548             {
6549             b = c << 4;
6550             }
6551           else
6552             {
6553             *out++ = b | c;
6554             b = -1;
6555             }
6556           }
6557
6558         enc = b64encode(sub, out - sub);
6559         yield = string_cat(yield, &size, &ptr, enc);
6560         continue;
6561         }
6562
6563       /* Convert octets outside 0x21..0x7E to \xXX form */
6564
6565       case EOP_HEXQUOTE:
6566         {
6567         uschar *t = sub - 1;
6568         while (*(++t) != 0)
6569           {
6570           if (*t < 0x21 || 0x7E < *t)
6571             yield = string_catn(yield, &size, &ptr,
6572               string_sprintf("\\x%02x", *t), 4);
6573           else
6574             yield = string_catn(yield, &size, &ptr, t, 1);
6575           }
6576         continue;
6577         }
6578
6579       /* count the number of list elements */
6580
6581       case EOP_LISTCOUNT:
6582         {
6583         int cnt = 0;
6584         int sep = 0;
6585         uschar * cp;
6586         uschar buffer[256];
6587
6588         while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
6589         cp = string_sprintf("%d", cnt);
6590         yield = string_cat(yield, &size, &ptr, cp);
6591         continue;
6592         }
6593
6594       /* expand a named list given the name */
6595       /* handles nested named lists; requotes as colon-sep list */
6596
6597       case EOP_LISTNAMED:
6598         {
6599         tree_node *t = NULL;
6600         const uschar * list;
6601         int sep = 0;
6602         uschar * item;
6603         uschar * suffix = US"";
6604         BOOL needsep = FALSE;
6605         uschar buffer[256];
6606
6607         if (*sub == '+') sub++;
6608         if (arg == NULL)        /* no-argument version */
6609           {
6610           if (!(t = tree_search(addresslist_anchor, sub)) &&
6611               !(t = tree_search(domainlist_anchor,  sub)) &&
6612               !(t = tree_search(hostlist_anchor,    sub)))
6613             t = tree_search(localpartlist_anchor, sub);
6614           }
6615         else switch(*arg)       /* specific list-type version */
6616           {
6617           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
6618           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
6619           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
6620           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
6621           default:
6622             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
6623             goto EXPAND_FAILED;
6624           }
6625
6626         if(!t)
6627           {
6628           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
6629             sub, !arg?""
6630               : *arg=='a'?"address "
6631               : *arg=='d'?"domain "
6632               : *arg=='h'?"host "
6633               : *arg=='l'?"localpart "
6634               : 0);
6635           goto EXPAND_FAILED;
6636           }
6637
6638         list = ((namedlist_block *)(t->data.ptr))->string;
6639
6640         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) != NULL)
6641           {
6642           uschar * buf = US" : ";
6643           if (needsep)
6644             yield = string_catn(yield, &size, &ptr, buf, 3);
6645           else
6646             needsep = TRUE;
6647
6648           if (*item == '+')     /* list item is itself a named list */
6649             {
6650             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
6651             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
6652             }
6653           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
6654             {
6655             char * cp;
6656             char tok[3];
6657             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
6658             while ((cp= strpbrk((const char *)item, tok)))
6659               {
6660               yield = string_catn(yield, &size, &ptr, item, cp-(char *)item);
6661               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
6662                 {
6663                 yield = string_catn(yield, &size, &ptr, US"::", 2);
6664                 item = (uschar *)cp;
6665                 }
6666               else              /* sep in item; should already be doubled; emit once */
6667                 {
6668                 yield = string_catn(yield, &size, &ptr, (uschar *)tok, 1);
6669                 if (*cp == sep) cp++;
6670                 item = (uschar *)cp;
6671                 }
6672               }
6673             }
6674           yield = string_cat(yield, &size, &ptr, item);
6675           }
6676         continue;
6677         }
6678
6679       /* mask applies a mask to an IP address; for example the result of
6680       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
6681
6682       case EOP_MASK:
6683         {
6684         int count;
6685         uschar *endptr;
6686         int binary[4];
6687         int mask, maskoffset;
6688         int type = string_is_ip_address(sub, &maskoffset);
6689         uschar buffer[64];
6690
6691         if (type == 0)
6692           {
6693           expand_string_message = string_sprintf("\"%s\" is not an IP address",
6694            sub);
6695           goto EXPAND_FAILED;
6696           }
6697
6698         if (maskoffset == 0)
6699           {
6700           expand_string_message = string_sprintf("missing mask value in \"%s\"",
6701             sub);
6702           goto EXPAND_FAILED;
6703           }
6704
6705         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
6706
6707         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
6708           {
6709           expand_string_message = string_sprintf("mask value too big in \"%s\"",
6710             sub);
6711           goto EXPAND_FAILED;
6712           }
6713
6714         /* Convert the address to binary integer(s) and apply the mask */
6715
6716         sub[maskoffset] = 0;
6717         count = host_aton(sub, binary);
6718         host_mask(count, binary, mask);
6719
6720         /* Convert to masked textual format and add to output. */
6721
6722         yield = string_catn(yield, &size, &ptr, buffer,
6723           host_nmtoa(count, binary, mask, buffer, '.'));
6724         continue;
6725         }
6726
6727       case EOP_IPV6NORM:
6728       case EOP_IPV6DENORM:
6729         {
6730         int type = string_is_ip_address(sub, NULL);
6731         int binary[4];
6732         uschar buffer[44];
6733
6734         switch (type)
6735           {
6736           case 6:
6737             (void) host_aton(sub, binary);
6738             break;
6739
6740           case 4:       /* convert to IPv4-mapped IPv6 */
6741             binary[0] = binary[1] = 0;
6742             binary[2] = 0x0000ffff;
6743             (void) host_aton(sub, binary+3);
6744             break;
6745
6746           case 0:
6747             expand_string_message =
6748               string_sprintf("\"%s\" is not an IP address", sub);
6749             goto EXPAND_FAILED;
6750           }
6751
6752         yield = string_catn(yield, &size, &ptr, buffer,
6753                   c == EOP_IPV6NORM
6754                     ? ipv6_nmtoa(binary, buffer)
6755                     : host_nmtoa(4, binary, -1, buffer, ':')
6756                   );
6757         continue;
6758         }
6759
6760       case EOP_ADDRESS:
6761       case EOP_LOCAL_PART:
6762       case EOP_DOMAIN:
6763         {
6764         uschar * error;
6765         int start, end, domain;
6766         uschar * t = parse_extract_address(sub, &error, &start, &end, &domain,
6767           FALSE);
6768         if (t)
6769           if (c != EOP_DOMAIN)
6770             {
6771             if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1;
6772             yield = string_catn(yield, &size, &ptr, sub+start, end-start);
6773             }
6774           else if (domain != 0)
6775             {
6776             domain += start;
6777             yield = string_catn(yield, &size, &ptr, sub+domain, end-domain);
6778             }
6779         continue;
6780         }
6781
6782       case EOP_ADDRESSES:
6783         {
6784         uschar outsep[2] = { ':', '\0' };
6785         uschar *address, *error;
6786         int save_ptr = ptr;
6787         int start, end, domain;  /* Not really used */
6788
6789         while (isspace(*sub)) sub++;
6790         if (*sub == '>') { *outsep = *++sub; ++sub; }
6791         parse_allow_group = TRUE;
6792
6793         for (;;)
6794           {
6795           uschar *p = parse_find_address_end(sub, FALSE);
6796           uschar saveend = *p;
6797           *p = '\0';
6798           address = parse_extract_address(sub, &error, &start, &end, &domain,
6799             FALSE);
6800           *p = saveend;
6801
6802           /* Add the address to the output list that we are building. This is
6803           done in chunks by searching for the separator character. At the
6804           start, unless we are dealing with the first address of the output
6805           list, add in a space if the new address begins with the separator
6806           character, or is an empty string. */
6807
6808           if (address != NULL)
6809             {
6810             if (ptr != save_ptr && address[0] == *outsep)
6811               yield = string_catn(yield, &size, &ptr, US" ", 1);
6812
6813             for (;;)
6814               {
6815               size_t seglen = Ustrcspn(address, outsep);
6816               yield = string_catn(yield, &size, &ptr, address, seglen + 1);
6817
6818               /* If we got to the end of the string we output one character
6819               too many. */
6820
6821               if (address[seglen] == '\0') { ptr--; break; }
6822               yield = string_catn(yield, &size, &ptr, outsep, 1);
6823               address += seglen + 1;
6824               }
6825
6826             /* Output a separator after the string: we will remove the
6827             redundant final one at the end. */
6828
6829             yield = string_catn(yield, &size, &ptr, outsep, 1);
6830             }
6831
6832           if (saveend == '\0') break;
6833           sub = p + 1;
6834           }
6835
6836         /* If we have generated anything, remove the redundant final
6837         separator. */
6838
6839         if (ptr != save_ptr) ptr--;
6840         parse_allow_group = FALSE;
6841         continue;
6842         }
6843
6844
6845       /* quote puts a string in quotes if it is empty or contains anything
6846       other than alphamerics, underscore, dot, or hyphen.
6847
6848       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
6849       be quoted in order to be a valid local part.
6850
6851       In both cases, newlines and carriage returns are converted into \n and \r
6852       respectively */
6853
6854       case EOP_QUOTE:
6855       case EOP_QUOTE_LOCAL_PART:
6856       if (arg == NULL)
6857         {
6858         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
6859         uschar *t = sub - 1;
6860
6861         if (c == EOP_QUOTE)
6862           {
6863           while (!needs_quote && *(++t) != 0)
6864             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
6865           }
6866         else  /* EOP_QUOTE_LOCAL_PART */
6867           {
6868           while (!needs_quote && *(++t) != 0)
6869             needs_quote = !isalnum(*t) &&
6870               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
6871               (*t != '.' || t == sub || t[1] == 0);
6872           }
6873
6874         if (needs_quote)
6875           {
6876           yield = string_catn(yield, &size, &ptr, US"\"", 1);
6877           t = sub - 1;
6878           while (*(++t) != 0)
6879             {
6880             if (*t == '\n')
6881               yield = string_catn(yield, &size, &ptr, US"\\n", 2);
6882             else if (*t == '\r')
6883               yield = string_catn(yield, &size, &ptr, US"\\r", 2);
6884             else
6885               {
6886               if (*t == '\\' || *t == '"')
6887                 yield = string_catn(yield, &size, &ptr, US"\\", 1);
6888               yield = string_catn(yield, &size, &ptr, t, 1);
6889               }
6890             }
6891           yield = string_catn(yield, &size, &ptr, US"\"", 1);
6892           }
6893         else yield = string_cat(yield, &size, &ptr, sub);
6894         continue;
6895         }
6896
6897       /* quote_lookuptype does lookup-specific quoting */
6898
6899       else
6900         {
6901         int n;
6902         uschar *opt = Ustrchr(arg, '_');
6903
6904         if (opt != NULL) *opt++ = 0;
6905
6906         n = search_findtype(arg, Ustrlen(arg));
6907         if (n < 0)
6908           {
6909           expand_string_message = search_error_message;
6910           goto EXPAND_FAILED;
6911           }
6912
6913         if (lookup_list[n]->quote != NULL)
6914           sub = (lookup_list[n]->quote)(sub, opt);
6915         else if (opt != NULL) sub = NULL;
6916
6917         if (sub == NULL)
6918           {
6919           expand_string_message = string_sprintf(
6920             "\"%s\" unrecognized after \"${quote_%s\"",
6921             opt, arg);
6922           goto EXPAND_FAILED;
6923           }
6924
6925         yield = string_cat(yield, &size, &ptr, sub);
6926         continue;
6927         }
6928
6929       /* rx quote sticks in \ before any non-alphameric character so that
6930       the insertion works in a regular expression. */
6931
6932       case EOP_RXQUOTE:
6933         {
6934         uschar *t = sub - 1;
6935         while (*(++t) != 0)
6936           {
6937           if (!isalnum(*t))
6938             yield = string_catn(yield, &size, &ptr, US"\\", 1);
6939           yield = string_catn(yield, &size, &ptr, t, 1);
6940           }
6941         continue;
6942         }
6943
6944       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
6945       prescribed by the RFC, if there are characters that need to be encoded */
6946
6947       case EOP_RFC2047:
6948         {
6949         uschar buffer[2048];
6950         const uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset,
6951           buffer, sizeof(buffer), FALSE);
6952         yield = string_cat(yield, &size, &ptr, string);
6953         continue;
6954         }
6955
6956       /* RFC 2047 decode */
6957
6958       case EOP_RFC2047D:
6959         {
6960         int len;
6961         uschar *error;
6962         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
6963           headers_charset, '?', &len, &error);
6964         if (error != NULL)
6965           {
6966           expand_string_message = error;
6967           goto EXPAND_FAILED;
6968           }
6969         yield = string_catn(yield, &size, &ptr, decoded, len);
6970         continue;
6971         }
6972
6973       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
6974       underscores */
6975
6976       case EOP_FROM_UTF8:
6977         {
6978         while (*sub != 0)
6979           {
6980           int c;
6981           uschar buff[4];
6982           GETUTF8INC(c, sub);
6983           if (c > 255) c = '_';
6984           buff[0] = c;
6985           yield = string_catn(yield, &size, &ptr, buff, 1);
6986           }
6987         continue;
6988         }
6989
6990           /* replace illegal UTF-8 sequences by replacement character  */
6991
6992       #define UTF8_REPLACEMENT_CHAR US"?"
6993
6994       case EOP_UTF8CLEAN:
6995         {
6996         int seq_len = 0, index = 0;
6997         int bytes_left = 0;
6998         long codepoint = -1;
6999         uschar seq_buff[4];                     /* accumulate utf-8 here */
7000
7001         while (*sub != 0)
7002           {
7003           int complete = 0;
7004           uschar c = *sub++;
7005
7006           if (bytes_left)
7007             {
7008             if ((c & 0xc0) != 0x80)
7009                     /* wrong continuation byte; invalidate all bytes */
7010               complete = 1; /* error */
7011             else
7012               {
7013               codepoint = (codepoint << 6) | (c & 0x3f);
7014               seq_buff[index++] = c;
7015               if (--bytes_left == 0)            /* codepoint complete */
7016                 if(codepoint > 0x10FFFF)        /* is it too large? */
7017                   complete = -1;        /* error (RFC3629 limit) */
7018                 else
7019                   {             /* finished; output utf-8 sequence */
7020                   yield = string_catn(yield, &size, &ptr, seq_buff, seq_len);
7021                   index = 0;
7022                   }
7023               }
7024             }
7025           else  /* no bytes left: new sequence */
7026             {
7027             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
7028               {
7029               yield = string_catn(yield, &size, &ptr, &c, 1);
7030               continue;
7031               }
7032             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
7033               {
7034               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
7035                 complete = -1;
7036               else
7037                 {
7038                   bytes_left = 1;
7039                   codepoint = c & 0x1f;
7040                 }
7041               }
7042             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
7043               {
7044               bytes_left = 2;
7045               codepoint = c & 0x0f;
7046               }
7047             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
7048               {
7049               bytes_left = 3;
7050               codepoint = c & 0x07;
7051               }
7052             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
7053               complete = -1;
7054
7055             seq_buff[index++] = c;
7056             seq_len = bytes_left + 1;
7057             }           /* if(bytes_left) */
7058
7059           if (complete != 0)
7060             {
7061             bytes_left = index = 0;
7062             yield = string_catn(yield, &size, &ptr, UTF8_REPLACEMENT_CHAR, 1);
7063             }
7064           if ((complete == 1) && ((c & 0x80) == 0))
7065                         /* ASCII character follows incomplete sequence */
7066               yield = string_catn(yield, &size, &ptr, &c, 1);
7067           }
7068         continue;
7069         }
7070
7071 #ifdef SUPPORT_I18N
7072       case EOP_UTF8_DOMAIN_TO_ALABEL:
7073         {
7074         uschar * error = NULL;
7075         uschar * s = string_domain_utf8_to_alabel(sub, &error);
7076         if (error)
7077           {
7078           expand_string_message = string_sprintf(
7079             "error converting utf8 (%s) to alabel: %s",
7080             string_printing(sub), error);
7081           goto EXPAND_FAILED;
7082           }
7083         yield = string_cat(yield, &size, &ptr, s);
7084         continue;
7085         }
7086
7087       case EOP_UTF8_DOMAIN_FROM_ALABEL:
7088         {
7089         uschar * error = NULL;
7090         uschar * s = string_domain_alabel_to_utf8(sub, &error);
7091         if (error)
7092           {
7093           expand_string_message = string_sprintf(
7094             "error converting alabel (%s) to utf8: %s",
7095             string_printing(sub), error);
7096           goto EXPAND_FAILED;
7097           }
7098         yield = string_cat(yield, &size, &ptr, s);
7099         continue;
7100         }
7101
7102       case EOP_UTF8_LOCALPART_TO_ALABEL:
7103         {
7104         uschar * error = NULL;
7105         uschar * s = string_localpart_utf8_to_alabel(sub, &error);
7106         if (error)
7107           {
7108           expand_string_message = string_sprintf(
7109             "error converting utf8 (%s) to alabel: %s",
7110             string_printing(sub), error);
7111           goto EXPAND_FAILED;
7112           }
7113         yield = string_cat(yield, &size, &ptr, s);
7114         DEBUG(D_expand) debug_printf_indent("yield: '%s'\n", yield);
7115         continue;
7116         }
7117
7118       case EOP_UTF8_LOCALPART_FROM_ALABEL:
7119         {
7120         uschar * error = NULL;
7121         uschar * s = string_localpart_alabel_to_utf8(sub, &error);
7122         if (error)
7123           {
7124           expand_string_message = string_sprintf(
7125             "error converting alabel (%s) to utf8: %s",
7126             string_printing(sub), error);
7127           goto EXPAND_FAILED;
7128           }
7129         yield = string_cat(yield, &size, &ptr, s);
7130         continue;
7131         }
7132 #endif  /* EXPERIMENTAL_INTERNATIONAL */
7133
7134       /* escape turns all non-printing characters into escape sequences. */
7135
7136       case EOP_ESCAPE:
7137         {
7138         const uschar * t = string_printing(sub);
7139         yield = string_cat(yield, &size, &ptr, t);
7140         continue;
7141         }
7142
7143       case EOP_ESCAPE8BIT:
7144         {
7145         const uschar * s = sub;
7146         uschar c;
7147
7148         for (s = sub; (c = *s); s++)
7149           yield = c < 127 && c != '\\'
7150             ? string_catn(yield, &size, &ptr, s, 1)
7151             : string_catn(yield, &size, &ptr, string_sprintf("\\%03o", c), 4);
7152         continue;
7153         }
7154
7155       /* Handle numeric expression evaluation */
7156
7157       case EOP_EVAL:
7158       case EOP_EVAL10:
7159         {
7160         uschar *save_sub = sub;
7161         uschar *error = NULL;
7162         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
7163         if (error != NULL)
7164           {
7165           expand_string_message = string_sprintf("error in expression "
7166             "evaluation: %s (after processing \"%.*s\")", error, sub-save_sub,
7167               save_sub);
7168           goto EXPAND_FAILED;
7169           }
7170         sprintf(CS var_buffer, PR_EXIM_ARITH, n);
7171         yield = string_cat(yield, &size, &ptr, var_buffer);
7172         continue;
7173         }
7174
7175       /* Handle time period formating */
7176
7177       case EOP_TIME_EVAL:
7178         {
7179         int n = readconf_readtime(sub, 0, FALSE);
7180         if (n < 0)
7181           {
7182           expand_string_message = string_sprintf("string \"%s\" is not an "
7183             "Exim time interval in \"%s\" operator", sub, name);
7184           goto EXPAND_FAILED;
7185           }
7186         sprintf(CS var_buffer, "%d", n);
7187         yield = string_cat(yield, &size, &ptr, var_buffer);
7188         continue;
7189         }
7190
7191       case EOP_TIME_INTERVAL:
7192         {
7193         int n;
7194         uschar *t = read_number(&n, sub);
7195         if (*t != 0) /* Not A Number*/
7196           {
7197           expand_string_message = string_sprintf("string \"%s\" is not a "
7198             "positive number in \"%s\" operator", sub, name);
7199           goto EXPAND_FAILED;
7200           }
7201         t = readconf_printtime(n);
7202         yield = string_cat(yield, &size, &ptr, t);
7203         continue;
7204         }
7205
7206       /* Convert string to base64 encoding */
7207
7208       case EOP_STR2B64:
7209       case EOP_BASE64:
7210         {
7211 #ifdef SUPPORT_TLS
7212         uschar * s = vp && *(void **)vp->value
7213           ? tls_cert_der_b64(*(void **)vp->value)
7214           : b64encode(sub, Ustrlen(sub));
7215 #else
7216         uschar * s = b64encode(sub, Ustrlen(sub));
7217 #endif
7218         yield = string_cat(yield, &size, &ptr, s);
7219         continue;
7220         }
7221
7222       case EOP_BASE64D:
7223         {
7224         uschar * s;
7225         int len = b64decode(sub, &s);
7226         if (len < 0)
7227           {
7228           expand_string_message = string_sprintf("string \"%s\" is not "
7229             "well-formed for \"%s\" operator", sub, name);
7230           goto EXPAND_FAILED;
7231           }
7232         yield = string_cat(yield, &size, &ptr, s);
7233         continue;
7234         }
7235
7236       /* strlen returns the length of the string */
7237
7238       case EOP_STRLEN:
7239         {
7240         uschar buff[24];
7241         (void)sprintf(CS buff, "%d", Ustrlen(sub));
7242         yield = string_cat(yield, &size, &ptr, buff);
7243         continue;
7244         }
7245
7246       /* length_n or l_n takes just the first n characters or the whole string,
7247       whichever is the shorter;
7248
7249       substr_m_n, and s_m_n take n characters from offset m; negative m take
7250       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
7251       takes the rest, either to the right or to the left.
7252
7253       hash_n or h_n makes a hash of length n from the string, yielding n
7254       characters from the set a-z; hash_n_m makes a hash of length n, but
7255       uses m characters from the set a-zA-Z0-9.
7256
7257       nhash_n returns a single number between 0 and n-1 (in text form), while
7258       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
7259       between 0 and n-1 and the second between 0 and m-1. */
7260
7261       case EOP_LENGTH:
7262       case EOP_L:
7263       case EOP_SUBSTR:
7264       case EOP_S:
7265       case EOP_HASH:
7266       case EOP_H:
7267       case EOP_NHASH:
7268       case EOP_NH:
7269         {
7270         int sign = 1;
7271         int value1 = 0;
7272         int value2 = -1;
7273         int *pn;
7274         int len;
7275         uschar *ret;
7276
7277         if (arg == NULL)
7278           {
7279           expand_string_message = string_sprintf("missing values after %s",
7280             name);
7281           goto EXPAND_FAILED;
7282           }
7283
7284         /* "length" has only one argument, effectively being synonymous with
7285         substr_0_n. */
7286
7287         if (c == EOP_LENGTH || c == EOP_L)
7288           {
7289           pn = &value2;
7290           value2 = 0;
7291           }
7292
7293         /* The others have one or two arguments; for "substr" the first may be
7294         negative. The second being negative means "not supplied". */
7295
7296         else
7297           {
7298           pn = &value1;
7299           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
7300           }
7301
7302         /* Read up to two numbers, separated by underscores */
7303
7304         ret = arg;
7305         while (*arg != 0)
7306           {
7307           if (arg != ret && *arg == '_' && pn == &value1)
7308             {
7309             pn = &value2;
7310             value2 = 0;
7311             if (arg[1] != 0) arg++;
7312             }
7313           else if (!isdigit(*arg))
7314             {
7315             expand_string_message =
7316               string_sprintf("non-digit after underscore in \"%s\"", name);
7317             goto EXPAND_FAILED;
7318             }
7319           else *pn = (*pn)*10 + *arg++ - '0';
7320           }
7321         value1 *= sign;
7322
7323         /* Perform the required operation */
7324
7325         ret =
7326           (c == EOP_HASH || c == EOP_H)?
7327              compute_hash(sub, value1, value2, &len) :
7328           (c == EOP_NHASH || c == EOP_NH)?
7329              compute_nhash(sub, value1, value2, &len) :
7330              extract_substr(sub, value1, value2, &len);
7331
7332         if (ret == NULL) goto EXPAND_FAILED;
7333         yield = string_catn(yield, &size, &ptr, ret, len);
7334         continue;
7335         }
7336
7337       /* Stat a path */
7338
7339       case EOP_STAT:
7340         {
7341         uschar *s;
7342         uschar smode[12];
7343         uschar **modetable[3];
7344         int i;
7345         mode_t mode;
7346         struct stat st;
7347
7348         if ((expand_forbid & RDO_EXISTS) != 0)
7349           {
7350           expand_string_message = US"Use of the stat() expansion is not permitted";
7351           goto EXPAND_FAILED;
7352           }
7353
7354         if (stat(CS sub, &st) < 0)
7355           {
7356           expand_string_message = string_sprintf("stat(%s) failed: %s",
7357             sub, strerror(errno));
7358           goto EXPAND_FAILED;
7359           }
7360         mode = st.st_mode;
7361         switch (mode & S_IFMT)
7362           {
7363           case S_IFIFO: smode[0] = 'p'; break;
7364           case S_IFCHR: smode[0] = 'c'; break;
7365           case S_IFDIR: smode[0] = 'd'; break;
7366           case S_IFBLK: smode[0] = 'b'; break;
7367           case S_IFREG: smode[0] = '-'; break;
7368           default: smode[0] = '?'; break;
7369           }
7370
7371         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
7372         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
7373         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
7374
7375         for (i = 0; i < 3; i++)
7376           {
7377           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
7378           mode >>= 3;
7379           }
7380
7381         smode[10] = 0;
7382         s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
7383           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
7384           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
7385           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
7386           (long)st.st_gid, st.st_size, (long)st.st_atime,
7387           (long)st.st_mtime, (long)st.st_ctime);
7388         yield = string_cat(yield, &size, &ptr, s);
7389         continue;
7390         }
7391
7392       /* vaguely random number less than N */
7393
7394       case EOP_RANDINT:
7395         {
7396         int_eximarith_t max;
7397         uschar *s;
7398
7399         max = expanded_string_integer(sub, TRUE);
7400         if (expand_string_message != NULL)
7401           goto EXPAND_FAILED;
7402         s = string_sprintf("%d", vaguely_random_number((int)max));
7403         yield = string_cat(yield, &size, &ptr, s);
7404         continue;
7405         }
7406
7407       /* Reverse IP, including IPv6 to dotted-nibble */
7408
7409       case EOP_REVERSE_IP:
7410         {
7411         int family, maskptr;
7412         uschar reversed[128];
7413
7414         family = string_is_ip_address(sub, &maskptr);
7415         if (family == 0)
7416           {
7417           expand_string_message = string_sprintf(
7418               "reverse_ip() not given an IP address [%s]", sub);
7419           goto EXPAND_FAILED;
7420           }
7421         invert_address(reversed, sub);
7422         yield = string_cat(yield, &size, &ptr, reversed);
7423         continue;
7424         }
7425
7426       /* Unknown operator */
7427
7428       default:
7429       expand_string_message =
7430         string_sprintf("unknown expansion operator \"%s\"", name);
7431       goto EXPAND_FAILED;
7432       }
7433     }
7434
7435   /* Handle a plain name. If this is the first thing in the expansion, release
7436   the pre-allocated buffer. If the result data is known to be in a new buffer,
7437   newsize will be set to the size of that buffer, and we can just point at that
7438   store instead of copying. Many expansion strings contain just one reference,
7439   so this is a useful optimization, especially for humungous headers
7440   ($message_headers). */
7441                                                 /*{*/
7442   if (*s++ == '}')
7443     {
7444     int len;
7445     int newsize = 0;
7446     if (ptr == 0)
7447       {
7448       if (resetok) store_reset(yield);
7449       yield = NULL;
7450       size = 0;
7451       }
7452     if (!(value = find_variable(name, FALSE, skipping, &newsize)))
7453       {
7454       expand_string_message =
7455         string_sprintf("unknown variable in \"${%s}\"", name);
7456       check_variable_error_message(name);
7457       goto EXPAND_FAILED;
7458       }
7459     len = Ustrlen(value);
7460     if (!yield && newsize)
7461       {
7462       yield = value;
7463       size = newsize;
7464       ptr = len;
7465       }
7466     else
7467       yield = string_catn(yield, &size, &ptr, value, len);
7468     continue;
7469     }
7470
7471   /* Else there's something wrong */
7472
7473   expand_string_message =
7474     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
7475     "in a variable reference)", name);
7476   goto EXPAND_FAILED;
7477   }
7478
7479 /* If we hit the end of the string when ket_ends is set, there is a missing
7480 terminating brace. */
7481
7482 if (ket_ends && *s == 0)
7483   {
7484   expand_string_message = malformed_header?
7485     US"missing } at end of string - could be header name not terminated by colon"
7486     :
7487     US"missing } at end of string";
7488   goto EXPAND_FAILED;
7489   }
7490
7491 /* Expansion succeeded; yield may still be NULL here if nothing was actually
7492 added to the string. If so, set up an empty string. Add a terminating zero. If
7493 left != NULL, return a pointer to the terminator. */
7494
7495 if (yield == NULL) yield = store_get(1);
7496 yield[ptr] = 0;
7497 if (left != NULL) *left = s;
7498
7499 /* Any stacking store that was used above the final string is no longer needed.
7500 In many cases the final string will be the first one that was got and so there
7501 will be optimal store usage. */
7502
7503 if (resetok) store_reset(yield + ptr + 1);
7504 else if (resetok_p) *resetok_p = FALSE;
7505
7506 DEBUG(D_expand)
7507   {
7508   debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
7509     "expanding: %.*s\n",
7510     (int)(s - string), string);
7511   debug_printf_indent("%s"
7512     UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7513     "result: %s\n",
7514     skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7515     yield);
7516   if (skipping)
7517     debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7518       "skipping: result is not used\n");
7519   }
7520 expand_level--;
7521 return yield;
7522
7523 /* This is the failure exit: easiest to program with a goto. We still need
7524 to update the pointer to the terminator, for cases of nested calls with "fail".
7525 */
7526
7527 EXPAND_FAILED_CURLY:
7528 if (malformed_header)
7529   expand_string_message =
7530     US"missing or misplaced { or } - could be header name not terminated by colon";
7531
7532 else if (!expand_string_message || !*expand_string_message)
7533   expand_string_message = US"missing or misplaced { or }";
7534
7535 /* At one point, Exim reset the store to yield (if yield was not NULL), but
7536 that is a bad idea, because expand_string_message is in dynamic store. */
7537
7538 EXPAND_FAILED:
7539 if (left != NULL) *left = s;
7540 DEBUG(D_expand)
7541   {
7542   debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n",
7543     string);
7544   debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7545     "error message: %s\n",
7546     expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7547     expand_string_message);
7548   if (expand_string_forcedfail)
7549     debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n");
7550   }
7551 if (resetok_p) *resetok_p = resetok;
7552 expand_level--;
7553 return NULL;
7554 }
7555
7556
7557 /* This is the external function call. Do a quick check for any expansion
7558 metacharacters, and if there are none, just return the input string.
7559
7560 Argument: the string to be expanded
7561 Returns:  the expanded string, or NULL if expansion failed; if failure was
7562           due to a lookup deferring, search_find_defer will be TRUE
7563 */
7564
7565 uschar *
7566 expand_string(uschar *string)
7567 {
7568 search_find_defer = FALSE;
7569 malformed_header = FALSE;
7570 return (Ustrpbrk(string, "$\\") == NULL)? string :
7571   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7572 }
7573
7574
7575
7576 const uschar *
7577 expand_cstring(const uschar *string)
7578 {
7579 search_find_defer = FALSE;
7580 malformed_header = FALSE;
7581 return (Ustrpbrk(string, "$\\") == NULL)? string :
7582   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7583 }
7584
7585
7586
7587 /*************************************************
7588 *              Expand and copy                   *
7589 *************************************************/
7590
7591 /* Now and again we want to expand a string and be sure that the result is in a
7592 new bit of store. This function does that.
7593 Since we know it has been copied, the de-const cast is safe.
7594
7595 Argument: the string to be expanded
7596 Returns:  the expanded string, always in a new bit of store, or NULL
7597 */
7598
7599 uschar *
7600 expand_string_copy(const uschar *string)
7601 {
7602 const uschar *yield = expand_cstring(string);
7603 if (yield == string) yield = string_copy(string);
7604 return US yield;
7605 }
7606
7607
7608
7609 /*************************************************
7610 *        Expand and interpret as an integer      *
7611 *************************************************/
7612
7613 /* Expand a string, and convert the result into an integer.
7614
7615 Arguments:
7616   string  the string to be expanded
7617   isplus  TRUE if a non-negative number is expected
7618
7619 Returns:  the integer value, or
7620           -1 for an expansion error               ) in both cases, message in
7621           -2 for an integer interpretation error  ) expand_string_message
7622           expand_string_message is set NULL for an OK integer
7623 */
7624
7625 int_eximarith_t
7626 expand_string_integer(uschar *string, BOOL isplus)
7627 {
7628 return expanded_string_integer(expand_string(string), isplus);
7629 }
7630
7631
7632 /*************************************************
7633  *         Interpret string as an integer        *
7634  *************************************************/
7635
7636 /* Convert a string (that has already been expanded) into an integer.
7637
7638 This function is used inside the expansion code.
7639
7640 Arguments:
7641   s       the string to be expanded
7642   isplus  TRUE if a non-negative number is expected
7643
7644 Returns:  the integer value, or
7645           -1 if string is NULL (which implies an expansion error)
7646           -2 for an integer interpretation error
7647           expand_string_message is set NULL for an OK integer
7648 */
7649
7650 static int_eximarith_t
7651 expanded_string_integer(const uschar *s, BOOL isplus)
7652 {
7653 int_eximarith_t value;
7654 uschar *msg = US"invalid integer \"%s\"";
7655 uschar *endptr;
7656
7657 /* If expansion failed, expand_string_message will be set. */
7658
7659 if (s == NULL) return -1;
7660
7661 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
7662 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
7663 systems, so we set it zero ourselves. */
7664
7665 errno = 0;
7666 expand_string_message = NULL;               /* Indicates no error */
7667
7668 /* Before Exim 4.64, strings consisting entirely of whitespace compared
7669 equal to 0.  Unfortunately, people actually relied upon that, so preserve
7670 the behaviour explicitly.  Stripping leading whitespace is a harmless
7671 noop change since strtol skips it anyway (provided that there is a number
7672 to find at all). */
7673 if (isspace(*s))
7674   {
7675   while (isspace(*s)) ++s;
7676   if (*s == '\0')
7677     {
7678       DEBUG(D_expand)
7679        debug_printf_indent("treating blank string as number 0\n");
7680       return 0;
7681     }
7682   }
7683
7684 value = strtoll(CS s, CSS &endptr, 10);
7685
7686 if (endptr == s)
7687   {
7688   msg = US"integer expected but \"%s\" found";
7689   }
7690 else if (value < 0 && isplus)
7691   {
7692   msg = US"non-negative integer expected but \"%s\" found";
7693   }
7694 else
7695   {
7696   switch (tolower(*endptr))
7697     {
7698     default:
7699       break;
7700     case 'k':
7701       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
7702       else value *= 1024;
7703       endptr++;
7704       break;
7705     case 'm':
7706       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
7707       else value *= 1024*1024;
7708       endptr++;
7709       break;
7710     case 'g':
7711       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
7712       else value *= 1024*1024*1024;
7713       endptr++;
7714       break;
7715     }
7716   if (errno == ERANGE)
7717     msg = US"absolute value of integer \"%s\" is too large (overflow)";
7718   else
7719     {
7720     while (isspace(*endptr)) endptr++;
7721     if (*endptr == 0) return value;
7722     }
7723   }
7724
7725 expand_string_message = string_sprintf(CS msg, s);
7726 return -2;
7727 }
7728
7729
7730 /* These values are usually fixed boolean values, but they are permitted to be
7731 expanded strings.
7732
7733 Arguments:
7734   addr       address being routed
7735   mtype      the module type
7736   mname      the module name
7737   dbg_opt    debug selectors
7738   oname      the option name
7739   bvalue     the router's boolean value
7740   svalue     the router's string value
7741   rvalue     where to put the returned value
7742
7743 Returns:     OK     value placed in rvalue
7744              DEFER  expansion failed
7745 */
7746
7747 int
7748 exp_bool(address_item *addr,
7749   uschar *mtype, uschar *mname, unsigned dbg_opt,
7750   uschar *oname, BOOL bvalue,
7751   uschar *svalue, BOOL *rvalue)
7752 {
7753 uschar *expanded;
7754 if (svalue == NULL) { *rvalue = bvalue; return OK; }
7755
7756 expanded = expand_string(svalue);
7757 if (expanded == NULL)
7758   {
7759   if (expand_string_forcedfail)
7760     {
7761     DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname);
7762     *rvalue = bvalue;
7763     return OK;
7764     }
7765   addr->message = string_sprintf("failed to expand \"%s\" in %s %s: %s",
7766       oname, mname, mtype, expand_string_message);
7767   DEBUG(dbg_opt) debug_printf("%s\n", addr->message);
7768   return DEFER;
7769   }
7770
7771 DEBUG(dbg_opt) debug_printf("expansion of \"%s\" yields \"%s\"\n", oname,
7772   expanded);
7773
7774 if (strcmpic(expanded, US"true") == 0 || strcmpic(expanded, US"yes") == 0)
7775   *rvalue = TRUE;
7776 else if (strcmpic(expanded, US"false") == 0 || strcmpic(expanded, US"no") == 0)
7777   *rvalue = FALSE;
7778 else
7779   {
7780   addr->message = string_sprintf("\"%s\" is not a valid value for the "
7781     "\"%s\" option in the %s %s", expanded, oname, mname, mtype);
7782   return DEFER;
7783   }
7784
7785 return OK;
7786 }
7787
7788
7789
7790 /* Avoid potentially exposing a password in a string about to be logged */
7791
7792 uschar *
7793 expand_hide_passwords(uschar * s)
7794 {
7795 return (  (  Ustrstr(s, "failed to expand") != NULL
7796           || Ustrstr(s, "expansion of ")    != NULL
7797           )
7798        && (  Ustrstr(s, "mysql")   != NULL
7799           || Ustrstr(s, "pgsql")   != NULL
7800           || Ustrstr(s, "redis")   != NULL
7801           || Ustrstr(s, "sqlite")  != NULL
7802           || Ustrstr(s, "ldap:")   != NULL
7803           || Ustrstr(s, "ldaps:")  != NULL
7804           || Ustrstr(s, "ldapi:")  != NULL
7805           || Ustrstr(s, "ldapdn:") != NULL
7806           || Ustrstr(s, "ldapm:")  != NULL
7807        )  )
7808   ? US"Temporary internal error" : s;
7809 }
7810
7811
7812
7813 /*************************************************
7814 * Error-checking for testsuite                   *
7815 *************************************************/
7816 typedef struct {
7817   const char *  filename;
7818   int           linenumber;
7819   uschar *      region_start;
7820   uschar *      region_end;
7821   const uschar *var_name;
7822   const uschar *var_data;
7823 } err_ctx;
7824
7825 static void
7826 assert_variable_notin(uschar * var_name, uschar * var_data, void * ctx)
7827 {
7828 err_ctx * e = ctx;
7829 if (var_data >= e->region_start  &&  var_data < e->region_end)
7830   {
7831   e->var_name = CUS var_name;
7832   e->var_data = CUS var_data;
7833   }
7834 }
7835
7836 void
7837 assert_no_variables(void * ptr, int len, const char * filename, int linenumber)
7838 {
7839 err_ctx e = {filename, linenumber, ptr, US ptr + len, NULL };
7840 int i;
7841 var_entry * v;
7842
7843 /* check acl_ variables */
7844 tree_walk(acl_var_c, assert_variable_notin, &e);
7845 tree_walk(acl_var_m, assert_variable_notin, &e);
7846
7847 /* check auth<n> variables */
7848 for (i = 0; i < AUTH_VARS; i++) if (auth_vars[i])
7849   assert_variable_notin(US"auth<n>", auth_vars[i], &e);
7850
7851 /* check regex<n> variables */
7852 for (i = 0; i < REGEX_VARS; i++) if (regex_vars[i])
7853   assert_variable_notin(US"regex<n>", regex_vars[i], &e);
7854
7855 /* check known-name variables */
7856 for (v = var_table; v < var_table + var_table_size; v++)
7857   if (v->type == vtype_stringptr)
7858     assert_variable_notin(US v->name, *(USS v->value), &e);
7859
7860 if (e.var_name)
7861   log_write(0, LOG_MAIN|LOG_PANIC_DIE,
7862     "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'",
7863     e.var_name, e.filename, e.linenumber, e.var_data);
7864 }
7865
7866
7867
7868 /*************************************************
7869 **************************************************
7870 *             Stand-alone test program           *
7871 **************************************************
7872 *************************************************/
7873
7874 #ifdef STAND_ALONE
7875
7876
7877 BOOL
7878 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
7879 {
7880 int ovector[3*(EXPAND_MAXN+1)];
7881 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
7882   ovector, nelem(ovector));
7883 BOOL yield = n >= 0;
7884 if (n == 0) n = EXPAND_MAXN + 1;
7885 if (yield)
7886   {
7887   int nn;
7888   expand_nmax = (setup < 0)? 0 : setup + 1;
7889   for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2)
7890     {
7891     expand_nstring[expand_nmax] = subject + ovector[nn];
7892     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
7893     }
7894   expand_nmax--;
7895   }
7896 return yield;
7897 }
7898
7899
7900 int main(int argc, uschar **argv)
7901 {
7902 int i;
7903 uschar buffer[1024];
7904
7905 debug_selector = D_v;
7906 debug_file = stderr;
7907 debug_fd = fileno(debug_file);
7908 big_buffer = malloc(big_buffer_size);
7909
7910 for (i = 1; i < argc; i++)
7911   {
7912   if (argv[i][0] == '+')
7913     {
7914     debug_trace_memory = 2;
7915     argv[i]++;
7916     }
7917   if (isdigit(argv[i][0]))
7918     debug_selector = Ustrtol(argv[i], NULL, 0);
7919   else
7920     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
7921         Ustrlen(argv[i]))
7922       {
7923 #ifdef LOOKUP_LDAP
7924       eldap_default_servers = argv[i];
7925 #endif
7926 #ifdef LOOKUP_MYSQL
7927       mysql_servers = argv[i];
7928 #endif
7929 #ifdef LOOKUP_PGSQL
7930       pgsql_servers = argv[i];
7931 #endif
7932 #ifdef LOOKUP_REDIS
7933       redis_servers = argv[i];
7934 #endif
7935       }
7936 #ifdef EXIM_PERL
7937   else opt_perl_startup = argv[i];
7938 #endif
7939   }
7940
7941 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
7942
7943 expand_nstring[1] = US"string 1....";
7944 expand_nlength[1] = 8;
7945 expand_nmax = 1;
7946
7947 #ifdef EXIM_PERL
7948 if (opt_perl_startup != NULL)
7949   {
7950   uschar *errstr;
7951   printf("Starting Perl interpreter\n");
7952   errstr = init_perl(opt_perl_startup);
7953   if (errstr != NULL)
7954     {
7955     printf("** error in perl_startup code: %s\n", errstr);
7956     return EXIT_FAILURE;
7957     }
7958   }
7959 #endif /* EXIM_PERL */
7960
7961 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
7962   {
7963   void *reset_point = store_get(0);
7964   uschar *yield = expand_string(buffer);
7965   if (yield != NULL)
7966     {
7967     printf("%s\n", yield);
7968     store_reset(reset_point);
7969     }
7970   else
7971     {
7972     if (search_find_defer) printf("search_find deferred\n");
7973     printf("Failed: %s\n", expand_string_message);
7974     if (expand_string_forcedfail) printf("Forced failure\n");
7975     printf("\n");
7976     }
7977   }
7978
7979 search_tidyup();
7980
7981 return 0;
7982 }
7983
7984 #endif
7985
7986 /* vi: aw ai sw=2
7987 */
7988 /* End of expand.c */