src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2018 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOOL, BOOL, BOOL *);
  17 static int_eximarith_t expanded_string_integer(const uschar *, BOOL);
  18
  19 #ifdef STAND_ALONE
  20 # ifndef SUPPORT_CRYPTEQ
  21 #  define SUPPORT_CRYPTEQ
  22 # endif
  23 #endif
  24
  25 #ifdef LOOKUP_LDAP
  26 # include "lookups/ldap.h"
  27 #endif
  28
  29 #ifdef SUPPORT_CRYPTEQ
  30 # ifdef CRYPT_H
  31 #  include <crypt.h>
  32 # endif
  33 # ifndef HAVE_CRYPT16
  34 extern char* crypt16(char*, char*);
  35 # endif
  36 #endif
  37
  38 /* The handling of crypt16() is a mess. I will record below the analysis of the
  39 mess that was sent to me. We decided, however, to make changing this very low
  40 priority, because in practice people are moving away from the crypt()
  41 algorithms nowadays, so it doesn't seem worth it.
  42
  43 <quote>
  44 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  45 the first 8 characters of the password using a 20-round version of crypt
  46 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  47 or an empty block if the password is less than 9 characters, using a
  48 20-round version of crypt and the same salt as was used for the first
  49 block.  Characters after the first 16 are ignored.  It always generates
  50 a 16-byte hash, which is expressed together with the salt as a string
  51 of 24 base 64 digits.  Here are some links to peruse:
  52
  53         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  54         http://seclists.org/bugtraq/1999/Mar/0076.html
  55
  56 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  57 and OSF/1.  This is the same as the standard crypt if given a password
  58 of 8 characters or less.  If given more, it first does the same as crypt
  59 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  60 using as salt the first two base 64 digits from the first hash block.
  61 If the password is more than 16 characters then it crypts the 17th to 24th
  62 characters using as salt the first two base 64 digits from the second hash
  63 block.  And so on: I've seen references to it cutting off the password at
  64 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  65
  66         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  67         http://seclists.org/bugtraq/1999/Mar/0109.html
  68         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  69              TET1_html/sec.c222.html#no_id_208
  70
  71 Exim has something it calls "crypt16".  It will either use a native
  72 crypt16 or its own implementation.  A native crypt16 will presumably
  73 be the one that I called "crypt16" above.  The internal "crypt16"
  74 function, however, is a two-block-maximum implementation of what I called
  75 "bigcrypt".  The documentation matches the internal code.
  76
  77 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  78 that crypt16 and bigcrypt were different things.
  79
  80 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  81 to whatever it is using under that name.  This unfortunately sets a
  82 precedent for using "{crypt16}" to identify two incompatible algorithms
  83 whose output can't be distinguished.  With "{crypt16}" thus rendered
  84 ambiguous, I suggest you deprecate it and invent two new identifiers
  85 for the two algorithms.
  86
  87 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  88 of the password separately means they can be cracked separately, so
  89 the double-length hash only doubles the cracking effort instead of
  90 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  91 bcrypt ({CRYPT}$2a$).
  92 </quote>
  93 */
  94
  95
  96
  97 /*************************************************
  98 *            Local statics and tables            *
  99 *************************************************/
 100
 101 /* Table of item names, and corresponding switch numbers. The names must be in
 102 alphabetical order. */
 103
 104 static uschar *item_table[] = {
 105   US"acl",
 106   US"authresults",
 107   US"certextract",
 108   US"dlfunc",
 109   US"env",
 110   US"extract",
 111   US"filter",
 112   US"hash",
 113   US"hmac",
 114   US"if",
 115 #ifdef SUPPORT_I18N
 116   US"imapfolder",
 117 #endif
 118   US"length",
 119   US"listextract",
 120   US"lookup",
 121   US"map",
 122   US"nhash",
 123   US"perl",
 124   US"prvs",
 125   US"prvscheck",
 126   US"readfile",
 127   US"readsocket",
 128   US"reduce",
 129   US"run",
 130   US"sg",
 131   US"sort",
 132   US"substr",
 133   US"tr" };
 134
 135 enum {
 136   EITEM_ACL,
 137   EITEM_AUTHRESULTS,
 138   EITEM_CERTEXTRACT,
 139   EITEM_DLFUNC,
 140   EITEM_ENV,
 141   EITEM_EXTRACT,
 142   EITEM_FILTER,
 143   EITEM_HASH,
 144   EITEM_HMAC,
 145   EITEM_IF,
 146 #ifdef SUPPORT_I18N
 147   EITEM_IMAPFOLDER,
 148 #endif
 149   EITEM_LENGTH,
 150   EITEM_LISTEXTRACT,
 151   EITEM_LOOKUP,
 152   EITEM_MAP,
 153   EITEM_NHASH,
 154   EITEM_PERL,
 155   EITEM_PRVS,
 156   EITEM_PRVSCHECK,
 157   EITEM_READFILE,
 158   EITEM_READSOCK,
 159   EITEM_REDUCE,
 160   EITEM_RUN,
 161   EITEM_SG,
 162   EITEM_SORT,
 163   EITEM_SUBSTR,
 164   EITEM_TR };
 165
 166 /* Tables of operator names, and corresponding switch numbers. The names must be
 167 in alphabetical order. There are two tables, because underscore is used in some
 168 cases to introduce arguments, whereas for other it is part of the name. This is
 169 an historical mis-design. */
 170
 171 static uschar *op_table_underscore[] = {
 172   US"from_utf8",
 173   US"local_part",
 174   US"quote_local_part",
 175   US"reverse_ip",
 176   US"time_eval",
 177   US"time_interval"
 178 #ifdef SUPPORT_I18N
 179  ,US"utf8_domain_from_alabel",
 180   US"utf8_domain_to_alabel",
 181   US"utf8_localpart_from_alabel",
 182   US"utf8_localpart_to_alabel"
 183 #endif
 184   };
 185
 186 enum {
 187   EOP_FROM_UTF8,
 188   EOP_LOCAL_PART,
 189   EOP_QUOTE_LOCAL_PART,
 190   EOP_REVERSE_IP,
 191   EOP_TIME_EVAL,
 192   EOP_TIME_INTERVAL
 193 #ifdef SUPPORT_I18N
 194  ,EOP_UTF8_DOMAIN_FROM_ALABEL,
 195   EOP_UTF8_DOMAIN_TO_ALABEL,
 196   EOP_UTF8_LOCALPART_FROM_ALABEL,
 197   EOP_UTF8_LOCALPART_TO_ALABEL
 198 #endif
 199   };
 200
 201 static uschar *op_table_main[] = {
 202   US"address",
 203   US"addresses",
 204   US"base32",
 205   US"base32d",
 206   US"base62",
 207   US"base62d",
 208   US"base64",
 209   US"base64d",
 210   US"domain",
 211   US"escape",
 212   US"escape8bit",
 213   US"eval",
 214   US"eval10",
 215   US"expand",
 216   US"h",
 217   US"hash",
 218   US"hex2b64",
 219   US"hexquote",
 220   US"ipv6denorm",
 221   US"ipv6norm",
 222   US"l",
 223   US"lc",
 224   US"length",
 225   US"listcount",
 226   US"listnamed",
 227   US"mask",
 228   US"md5",
 229   US"nh",
 230   US"nhash",
 231   US"quote",
 232   US"randint",
 233   US"rfc2047",
 234   US"rfc2047d",
 235   US"rxquote",
 236   US"s",
 237   US"sha1",
 238   US"sha256",
 239   US"sha3",
 240   US"stat",
 241   US"str2b64",
 242   US"strlen",
 243   US"substr",
 244   US"uc",
 245   US"utf8clean" };
 246
 247 enum {
 248   EOP_ADDRESS =  nelem(op_table_underscore),
 249   EOP_ADDRESSES,
 250   EOP_BASE32,
 251   EOP_BASE32D,
 252   EOP_BASE62,
 253   EOP_BASE62D,
 254   EOP_BASE64,
 255   EOP_BASE64D,
 256   EOP_DOMAIN,
 257   EOP_ESCAPE,
 258   EOP_ESCAPE8BIT,
 259   EOP_EVAL,
 260   EOP_EVAL10,
 261   EOP_EXPAND,
 262   EOP_H,
 263   EOP_HASH,
 264   EOP_HEX2B64,
 265   EOP_HEXQUOTE,
 266   EOP_IPV6DENORM,
 267   EOP_IPV6NORM,
 268   EOP_L,
 269   EOP_LC,
 270   EOP_LENGTH,
 271   EOP_LISTCOUNT,
 272   EOP_LISTNAMED,
 273   EOP_MASK,
 274   EOP_MD5,
 275   EOP_NH,
 276   EOP_NHASH,
 277   EOP_QUOTE,
 278   EOP_RANDINT,
 279   EOP_RFC2047,
 280   EOP_RFC2047D,
 281   EOP_RXQUOTE,
 282   EOP_S,
 283   EOP_SHA1,
 284   EOP_SHA256,
 285   EOP_SHA3,
 286   EOP_STAT,
 287   EOP_STR2B64,
 288   EOP_STRLEN,
 289   EOP_SUBSTR,
 290   EOP_UC,
 291   EOP_UTF8CLEAN };
 292
 293
 294 /* Table of condition names, and corresponding switch numbers. The names must
 295 be in alphabetical order. */
 296
 297 static uschar *cond_table[] = {
 298   US"<",
 299   US"<=",
 300   US"=",
 301   US"==",     /* Backward compatibility */
 302   US">",
 303   US">=",
 304   US"acl",
 305   US"and",
 306   US"bool",
 307   US"bool_lax",
 308   US"crypteq",
 309   US"def",
 310   US"eq",
 311   US"eqi",
 312   US"exists",
 313   US"first_delivery",
 314   US"forall",
 315   US"forany",
 316   US"ge",
 317   US"gei",
 318   US"gt",
 319   US"gti",
 320   US"inlist",
 321   US"inlisti",
 322   US"isip",
 323   US"isip4",
 324   US"isip6",
 325   US"ldapauth",
 326   US"le",
 327   US"lei",
 328   US"lt",
 329   US"lti",
 330   US"match",
 331   US"match_address",
 332   US"match_domain",
 333   US"match_ip",
 334   US"match_local_part",
 335   US"or",
 336   US"pam",
 337   US"pwcheck",
 338   US"queue_running",
 339   US"radius",
 340   US"saslauthd"
 341 };
 342
 343 enum {
 344   ECOND_NUM_L,
 345   ECOND_NUM_LE,
 346   ECOND_NUM_E,
 347   ECOND_NUM_EE,
 348   ECOND_NUM_G,
 349   ECOND_NUM_GE,
 350   ECOND_ACL,
 351   ECOND_AND,
 352   ECOND_BOOL,
 353   ECOND_BOOL_LAX,
 354   ECOND_CRYPTEQ,
 355   ECOND_DEF,
 356   ECOND_STR_EQ,
 357   ECOND_STR_EQI,
 358   ECOND_EXISTS,
 359   ECOND_FIRST_DELIVERY,
 360   ECOND_FORALL,
 361   ECOND_FORANY,
 362   ECOND_STR_GE,
 363   ECOND_STR_GEI,
 364   ECOND_STR_GT,
 365   ECOND_STR_GTI,
 366   ECOND_INLIST,
 367   ECOND_INLISTI,
 368   ECOND_ISIP,
 369   ECOND_ISIP4,
 370   ECOND_ISIP6,
 371   ECOND_LDAPAUTH,
 372   ECOND_STR_LE,
 373   ECOND_STR_LEI,
 374   ECOND_STR_LT,
 375   ECOND_STR_LTI,
 376   ECOND_MATCH,
 377   ECOND_MATCH_ADDRESS,
 378   ECOND_MATCH_DOMAIN,
 379   ECOND_MATCH_IP,
 380   ECOND_MATCH_LOCAL_PART,
 381   ECOND_OR,
 382   ECOND_PAM,
 383   ECOND_PWCHECK,
 384   ECOND_QUEUE_RUNNING,
 385   ECOND_RADIUS,
 386   ECOND_SASLAUTHD
 387 };
 388
 389
 390 /* Types of table entry */
 391
 392 enum vtypes {
 393   vtype_int,            /* value is address of int */
 394   vtype_filter_int,     /* ditto, but recognized only when filtering */
 395   vtype_ino,            /* value is address of ino_t (not always an int) */
 396   vtype_uid,            /* value is address of uid_t (not always an int) */
 397   vtype_gid,            /* value is address of gid_t (not always an int) */
 398   vtype_bool,           /* value is address of bool */
 399   vtype_stringptr,      /* value is address of pointer to string */
 400   vtype_msgbody,        /* as stringptr, but read when first required */
 401   vtype_msgbody_end,    /* ditto, the end of the message */
 402   vtype_msgheaders,     /* the message's headers, processed */
 403   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 404   vtype_localpart,      /* extract local part from string */
 405   vtype_domain,         /* extract domain from string */
 406   vtype_string_func,    /* value is string returned by given function */
 407   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 408   vtype_tode,           /* value not used; generate tod in epoch format */
 409   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 410   vtype_todf,           /* value not used; generate full tod */
 411   vtype_todl,           /* value not used; generate log tod */
 412   vtype_todlf,          /* value not used; generate log file datestamp tod */
 413   vtype_todzone,        /* value not used; generate time zone only */
 414   vtype_todzulu,        /* value not used; generate zulu tod */
 415   vtype_reply,          /* value not used; get reply from headers */
 416   vtype_pid,            /* value not used; result is pid */
 417   vtype_host_lookup,    /* value not used; get host name */
 418   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 419   vtype_pspace,         /* partition space; value is T/F for spool/log */
 420   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 421   vtype_cert            /* SSL certificate */
 422   #ifndef DISABLE_DKIM
 423   ,vtype_dkim           /* Lookup of value in DKIM signature */
 424   #endif
 425 };
 426
 427 /* Type for main variable table */
 428
 429 typedef struct {
 430   const char *name;
 431   enum vtypes type;
 432   void       *value;
 433 } var_entry;
 434
 435 /* Type for entries pointing to address/length pairs. Not currently
 436 in use. */
 437
 438 typedef struct {
 439   uschar **address;
 440   int  *length;
 441 } alblock;
 442
 443 static uschar * fn_recipients(void);
 444
 445 /* This table must be kept in alphabetical order. */
 446
 447 static var_entry var_table[] = {
 448   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 449      they will be confused with user-creatable ACL variables. */
 450   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 451   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 452   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 453   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 454   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 455   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 456   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 457   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 458   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 459   { "acl_narg",            vtype_int,         &acl_narg },
 460   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 461   { "address_data",        vtype_stringptr,   &deliver_address_data },
 462   { "address_file",        vtype_stringptr,   &address_file },
 463   { "address_pipe",        vtype_stringptr,   &address_pipe },
 464 #ifdef EXPERIMENTAL_ARC
 465   { "arc_state",           vtype_stringptr,   &arc_state },
 466   { "arc_state_reason",    vtype_stringptr,   &arc_state_reason },
 467 #endif
 468   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 469   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 470   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 471   { "authentication_failed",vtype_int,        &authentication_failed },
 472 #ifdef WITH_CONTENT_SCAN
 473   { "av_failed",           vtype_int,         &av_failed },
 474 #endif
 475 #ifdef EXPERIMENTAL_BRIGHTMAIL
 476   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 477   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 478   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 479   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 480 #endif
 481   { "body_linecount",      vtype_int,         &body_linecount },
 482   { "body_zerocount",      vtype_int,         &body_zerocount },
 483   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 484   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 485   { "caller_gid",          vtype_gid,         &real_gid },
 486   { "caller_uid",          vtype_uid,         &real_uid },
 487   { "callout_address",     vtype_stringptr,   &callout_address },
 488   { "compile_date",        vtype_stringptr,   &version_date },
 489   { "compile_number",      vtype_stringptr,   &version_cnumber },
 490   { "config_dir",          vtype_stringptr,   &config_main_directory },
 491   { "config_file",         vtype_stringptr,   &config_main_filename },
 492   { "csa_status",          vtype_stringptr,   &csa_status },
 493 #ifdef EXPERIMENTAL_DCC
 494   { "dcc_header",          vtype_stringptr,   &dcc_header },
 495   { "dcc_result",          vtype_stringptr,   &dcc_result },
 496 #endif
 497 #ifndef DISABLE_DKIM
 498   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 499   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 500   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 501   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 502   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 503   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 504   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 505   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 506   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 507   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 508   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 509   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 510   { "dkim_key_length",     vtype_int,         &dkim_key_length },
 511   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 512   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 513   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 514   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 515   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 516   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 517   { "dkim_verify_reason",  vtype_stringptr,   &dkim_verify_reason },
 518   { "dkim_verify_status",  vtype_stringptr,   &dkim_verify_status },
 519 #endif
 520 #ifdef EXPERIMENTAL_DMARC
 521   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 522   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 523   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 524   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 525 #endif
 526   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 527   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 528   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 529   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 530   { "domain",              vtype_stringptr,   &deliver_domain },
 531   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 532 #ifndef DISABLE_EVENT
 533   { "event_data",          vtype_stringptr,   &event_data },
 534
 535   /*XXX want to use generic vars for as many of these as possible*/
 536   { "event_defer_errno",   vtype_int,         &event_defer_errno },
 537
 538   { "event_name",          vtype_stringptr,   &event_name },
 539 #endif
 540   { "exim_gid",            vtype_gid,         &exim_gid },
 541   { "exim_path",           vtype_stringptr,   &exim_path },
 542   { "exim_uid",            vtype_uid,         &exim_uid },
 543   { "exim_version",        vtype_stringptr,   &version_string },
 544   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 545   { "home",                vtype_stringptr,   &deliver_home },
 546   { "host",                vtype_stringptr,   &deliver_host },
 547   { "host_address",        vtype_stringptr,   &deliver_host_address },
 548   { "host_data",           vtype_stringptr,   &host_data },
 549   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 550   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 551   { "host_port",           vtype_int,         &deliver_host_port },
 552   { "initial_cwd",         vtype_stringptr,   &initial_cwd },
 553   { "inode",               vtype_ino,         &deliver_inode },
 554   { "interface_address",   vtype_stringptr,   &interface_address },
 555   { "interface_port",      vtype_int,         &interface_port },
 556   { "item",                vtype_stringptr,   &iterate_item },
 557   #ifdef LOOKUP_LDAP
 558   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 559   #endif
 560   { "load_average",        vtype_load_avg,    NULL },
 561   { "local_part",          vtype_stringptr,   &deliver_localpart },
 562   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 563   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 564   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 565   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 566   { "local_user_gid",      vtype_gid,         &local_user_gid },
 567   { "local_user_uid",      vtype_uid,         &local_user_uid },
 568   { "localhost_number",    vtype_int,         &host_number },
 569   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 570   { "log_space",           vtype_pspace,      (void *)FALSE },
 571   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 572   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 573 #ifdef WITH_CONTENT_SCAN
 574   { "malware_name",        vtype_stringptr,   &malware_name },
 575 #endif
 576   { "max_received_linelength", vtype_int,     &max_received_linelength },
 577   { "message_age",         vtype_int,         &message_age },
 578   { "message_body",        vtype_msgbody,     &message_body },
 579   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 580   { "message_body_size",   vtype_int,         &message_body_size },
 581   { "message_exim_id",     vtype_stringptr,   &message_id },
 582   { "message_headers",     vtype_msgheaders,  NULL },
 583   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 584   { "message_id",          vtype_stringptr,   &message_id },
 585   { "message_linecount",   vtype_int,         &message_linecount },
 586   { "message_size",        vtype_int,         &message_size },
 587 #ifdef SUPPORT_I18N
 588   { "message_smtputf8",    vtype_bool,        &message_smtputf8 },
 589 #endif
 590 #ifdef WITH_CONTENT_SCAN
 591   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 592   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 593   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 594   { "mime_charset",        vtype_stringptr,   &mime_charset },
 595   { "mime_content_description", vtype_stringptr, &mime_content_description },
 596   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 597   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 598   { "mime_content_size",   vtype_int,         &mime_content_size },
 599   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 600   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 601   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 602   { "mime_filename",       vtype_stringptr,   &mime_filename },
 603   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 604   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 605   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 606   { "mime_part_count",     vtype_int,         &mime_part_count },
 607 #endif
 608   { "n0",                  vtype_filter_int,  &filter_n[0] },
 609   { "n1",                  vtype_filter_int,  &filter_n[1] },
 610   { "n2",                  vtype_filter_int,  &filter_n[2] },
 611   { "n3",                  vtype_filter_int,  &filter_n[3] },
 612   { "n4",                  vtype_filter_int,  &filter_n[4] },
 613   { "n5",                  vtype_filter_int,  &filter_n[5] },
 614   { "n6",                  vtype_filter_int,  &filter_n[6] },
 615   { "n7",                  vtype_filter_int,  &filter_n[7] },
 616   { "n8",                  vtype_filter_int,  &filter_n[8] },
 617   { "n9",                  vtype_filter_int,  &filter_n[9] },
 618   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 619   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 620   { "originator_gid",      vtype_gid,         &originator_gid },
 621   { "originator_uid",      vtype_uid,         &originator_uid },
 622   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 623   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 624   { "pid",                 vtype_pid,         NULL },
 625 #ifndef DISABLE_PRDR
 626   { "prdr_requested",      vtype_bool,        &prdr_requested },
 627 #endif
 628   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 629 #if defined(SUPPORT_PROXY) || defined(SUPPORT_SOCKS)
 630   { "proxy_external_address",vtype_stringptr, &proxy_external_address },
 631   { "proxy_external_port", vtype_int,         &proxy_external_port },
 632   { "proxy_local_address", vtype_stringptr,   &proxy_local_address },
 633   { "proxy_local_port",    vtype_int,         &proxy_local_port },
 634   { "proxy_session",       vtype_bool,        &proxy_session },
 635 #endif
 636   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 637   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 638   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 639   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 640   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 641   { "queue_name",          vtype_stringptr,   &queue_name },
 642   { "rcpt_count",          vtype_int,         &rcpt_count },
 643   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 644   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 645   { "received_count",      vtype_int,         &received_count },
 646   { "received_for",        vtype_stringptr,   &received_for },
 647   { "received_ip_address", vtype_stringptr,   &interface_address },
 648   { "received_port",       vtype_int,         &interface_port },
 649   { "received_protocol",   vtype_stringptr,   &received_protocol },
 650   { "received_time",       vtype_int,         &received_time.tv_sec },
 651   { "recipient_data",      vtype_stringptr,   &recipient_data },
 652   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 653   { "recipients",          vtype_string_func, &fn_recipients },
 654   { "recipients_count",    vtype_int,         &recipients_count },
 655 #ifdef WITH_CONTENT_SCAN
 656   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 657 #endif
 658   { "reply_address",       vtype_reply,       NULL },
 659   { "return_path",         vtype_stringptr,   &return_path },
 660   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 661   { "router_name",         vtype_stringptr,   &router_name },
 662   { "runrc",               vtype_int,         &runrc },
 663   { "self_hostname",       vtype_stringptr,   &self_hostname },
 664   { "sender_address",      vtype_stringptr,   &sender_address },
 665   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 666   { "sender_address_domain", vtype_domain,    &sender_address },
 667   { "sender_address_local_part", vtype_localpart, &sender_address },
 668   { "sender_data",         vtype_stringptr,   &sender_data },
 669   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 670   { "sender_helo_dnssec",  vtype_bool,        &sender_helo_dnssec },
 671   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 672   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 673   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 674   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 675   { "sender_host_name",    vtype_host_lookup, NULL },
 676   { "sender_host_port",    vtype_int,         &sender_host_port },
 677   { "sender_ident",        vtype_stringptr,   &sender_ident },
 678   { "sender_rate",         vtype_stringptr,   &sender_rate },
 679   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 680   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 681   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 682   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 683   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 684   { "sending_port",        vtype_int,         &sending_port },
 685   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 686   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 687   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 688   { "smtp_command_history", vtype_string_func, &smtp_cmd_hist },
 689   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 690   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 691   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 692   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 693   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 694   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 695   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 696   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 697   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 698   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 699   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 700   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 701 #ifdef WITH_CONTENT_SCAN
 702   { "spam_action",         vtype_stringptr,   &spam_action },
 703   { "spam_bar",            vtype_stringptr,   &spam_bar },
 704   { "spam_report",         vtype_stringptr,   &spam_report },
 705   { "spam_score",          vtype_stringptr,   &spam_score },
 706   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 707 #endif
 708 #ifdef SUPPORT_SPF
 709   { "spf_guess",           vtype_stringptr,   &spf_guess },
 710   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 711   { "spf_received",        vtype_stringptr,   &spf_received },
 712   { "spf_result",          vtype_stringptr,   &spf_result },
 713   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 714 #endif
 715   { "spool_directory",     vtype_stringptr,   &spool_directory },
 716   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 717   { "spool_space",         vtype_pspace,      (void *)TRUE },
 718 #ifdef EXPERIMENTAL_SRS
 719   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 720   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 721   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 722   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 723   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 724   { "srs_status",          vtype_stringptr,   &srs_status },
 725 #endif
 726   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 727
 728   /* The non-(in,out) variables are now deprecated */
 729   { "tls_bits",            vtype_int,         &tls_in.bits },
 730   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 731   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 732
 733   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 734   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 735   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 736   { "tls_in_ocsp",         vtype_int,         &tls_in.ocsp },
 737   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 738   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 739   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 740 #if defined(SUPPORT_TLS)
 741   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 742 #endif
 743   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 744   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 745   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 746 #ifdef SUPPORT_DANE
 747   { "tls_out_dane",        vtype_bool,        &tls_out.dane_verified },
 748 #endif
 749   { "tls_out_ocsp",        vtype_int,         &tls_out.ocsp },
 750   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 751   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 752   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 753 #if defined(SUPPORT_TLS)
 754   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 755 #endif
 756 #ifdef SUPPORT_DANE
 757   { "tls_out_tlsa_usage",  vtype_int,         &tls_out.tlsa_usage },
 758 #endif
 759
 760   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 761 #if defined(SUPPORT_TLS)
 762   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 763 #endif
 764
 765   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 766   { "tod_epoch",           vtype_tode,        NULL },
 767   { "tod_epoch_l",         vtype_todel,       NULL },
 768   { "tod_full",            vtype_todf,        NULL },
 769   { "tod_log",             vtype_todl,        NULL },
 770   { "tod_logfile",         vtype_todlf,       NULL },
 771   { "tod_zone",            vtype_todzone,     NULL },
 772   { "tod_zulu",            vtype_todzulu,     NULL },
 773   { "transport_name",      vtype_stringptr,   &transport_name },
 774   { "value",               vtype_stringptr,   &lookup_value },
 775   { "verify_mode",         vtype_stringptr,   &verify_mode },
 776   { "version_number",      vtype_stringptr,   &version_string },
 777   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 778   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 779   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 780   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 781   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 782   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 783 };
 784
 785 static int var_table_size = nelem(var_table);
 786 static uschar var_buffer[256];
 787 static BOOL malformed_header;
 788
 789 /* For textual hashes */
 790
 791 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 792                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 793                                "0123456789";
 794
 795 enum { HMAC_MD5, HMAC_SHA1 };
 796
 797 /* For numeric hashes */
 798
 799 static unsigned int prime[] = {
 800   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 801  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 802  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 803
 804 /* For printing modes in symbolic form */
 805
 806 static uschar *mtable_normal[] =
 807   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 808
 809 static uschar *mtable_setid[] =
 810   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 811
 812 static uschar *mtable_sticky[] =
 813   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 814
 815
 816
 817 /*************************************************
 818 *           Tables for UTF-8 support             *
 819 *************************************************/
 820
 821 /* Table of the number of extra characters, indexed by the first character
 822 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 823 0x3d. */
 824
 825 static uschar utf8_table1[] = {
 826   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 827   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 828   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 829   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 830
 831 /* These are the masks for the data bits in the first byte of a character,
 832 indexed by the number of additional bytes. */
 833
 834 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 835
 836 /* Get the next UTF-8 character, advancing the pointer. */
 837
 838 #define GETUTF8INC(c, ptr) \
 839   c = *ptr++; \
 840   if ((c & 0xc0) == 0xc0) \
 841     { \
 842     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 843     int s = 6*a; \
 844     c = (c & utf8_table2[a]) << s; \
 845     while (a-- > 0) \
 846       { \
 847       s -= 6; \
 848       c |= (*ptr++ & 0x3f) << s; \
 849       } \
 850     }
 851
 852
 853
 854 static uschar * base32_chars = US"abcdefghijklmnopqrstuvwxyz234567";
 855
 856 /*************************************************
 857 *           Binary chop search on a table        *
 858 *************************************************/
 859
 860 /* This is used for matching expansion items and operators.
 861
 862 Arguments:
 863   name        the name that is being sought
 864   table       the table to search
 865   table_size  the number of items in the table
 866
 867 Returns:      the offset in the table, or -1
 868 */
 869
 870 static int
 871 chop_match(uschar *name, uschar **table, int table_size)
 872 {
 873 uschar **bot = table;
 874 uschar **top = table + table_size;
 875
 876 while (top > bot)
 877   {
 878   uschar **mid = bot + (top - bot)/2;
 879   int c = Ustrcmp(name, *mid);
 880   if (c == 0) return mid - table;
 881   if (c > 0) bot = mid + 1; else top = mid;
 882   }
 883
 884 return -1;
 885 }
 886
 887
 888
 889 /*************************************************
 890 *          Check a condition string              *
 891 *************************************************/
 892
 893 /* This function is called to expand a string, and test the result for a "true"
 894 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 895 forced fail or lookup defer.
 896
 897 We used to release all store used, but this is not not safe due
 898 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 899 is reasonably careful to release what it can.
 900
 901 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 902
 903 Arguments:
 904   condition     the condition string
 905   m1            text to be incorporated in panic error
 906   m2            ditto
 907
 908 Returns:        TRUE if condition is met, FALSE if not
 909 */
 910
 911 BOOL
 912 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 913 {
 914 int rc;
 915 uschar *ss = expand_string(condition);
 916 if (ss == NULL)
 917   {
 918   if (!expand_string_forcedfail && !search_find_defer)
 919     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 920       "for %s %s: %s", condition, m1, m2, expand_string_message);
 921   return FALSE;
 922   }
 923 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 924   strcmpic(ss, US"false") != 0;
 925 return rc;
 926 }
 927
 928
 929
 930
 931 /*************************************************
 932 *        Pseudo-random number generation         *
 933 *************************************************/
 934
 935 /* Pseudo-random number generation.  The result is not "expected" to be
 936 cryptographically strong but not so weak that someone will shoot themselves
 937 in the foot using it as a nonce in some email header scheme or whatever
 938 weirdness they'll twist this into.  The result should ideally handle fork().
 939
 940 However, if we're stuck unable to provide this, then we'll fall back to
 941 appallingly bad randomness.
 942
 943 If SUPPORT_TLS is defined then this will not be used except as an emergency
 944 fallback.
 945
 946 Arguments:
 947   max       range maximum
 948 Returns     a random number in range [0, max-1]
 949 */
 950
 951 #ifdef SUPPORT_TLS
 952 # define vaguely_random_number vaguely_random_number_fallback
 953 #endif
 954 int
 955 vaguely_random_number(int max)
 956 {
 957 #ifdef SUPPORT_TLS
 958 # undef vaguely_random_number
 959 #endif
 960   static pid_t pid = 0;
 961   pid_t p2;
 962 #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
 963   struct timeval tv;
 964 #endif
 965
 966   p2 = getpid();
 967   if (p2 != pid)
 968     {
 969     if (pid != 0)
 970       {
 971
 972 #ifdef HAVE_ARC4RANDOM
 973       /* cryptographically strong randomness, common on *BSD platforms, not
 974       so much elsewhere.  Alas. */
 975 #ifndef NOT_HAVE_ARC4RANDOM_STIR
 976       arc4random_stir();
 977 #endif
 978 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 979 #ifdef HAVE_SRANDOMDEV
 980       /* uses random(4) for seeding */
 981       srandomdev();
 982 #else
 983       gettimeofday(&tv, NULL);
 984       srandom(tv.tv_sec | tv.tv_usec | getpid());
 985 #endif
 986 #else
 987       /* Poor randomness and no seeding here */
 988 #endif
 989
 990       }
 991     pid = p2;
 992     }
 993
 994 #ifdef HAVE_ARC4RANDOM
 995   return arc4random() % max;
 996 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 997   return random() % max;
 998 #else
 999   /* This one returns a 16-bit number, definitely not crypto-strong */
1000   return random_number(max);
1001 #endif
1002 }
1003
1004
1005
1006
1007 /*************************************************
1008 *             Pick out a name from a string      *
1009 *************************************************/
1010
1011 /* If the name is too long, it is silently truncated.
1012
1013 Arguments:
1014   name      points to a buffer into which to put the name
1015   max       is the length of the buffer
1016   s         points to the first alphabetic character of the name
1017   extras    chars other than alphanumerics to permit
1018
1019 Returns:    pointer to the first character after the name
1020
1021 Note: The test for *s != 0 in the while loop is necessary because
1022 Ustrchr() yields non-NULL if the character is zero (which is not something
1023 I expected). */
1024
1025 static const uschar *
1026 read_name(uschar *name, int max, const uschar *s, uschar *extras)
1027 {
1028 int ptr = 0;
1029 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
1030   {
1031   if (ptr < max-1) name[ptr++] = *s;
1032   s++;
1033   }
1034 name[ptr] = 0;
1035 return s;
1036 }
1037
1038
1039
1040 /*************************************************
1041 *     Pick out the rest of a header name         *
1042 *************************************************/
1043
1044 /* A variable name starting $header_ (or just $h_ for those who like
1045 abbreviations) might not be the complete header name because headers can
1046 contain any printing characters in their names, except ':'. This function is
1047 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
1048 on the end, if the name was terminated by white space.
1049
1050 Arguments:
1051   name      points to a buffer in which the name read so far exists
1052   max       is the length of the buffer
1053   s         points to the first character after the name so far, i.e. the
1054             first non-alphameric character after $header_xxxxx
1055
1056 Returns:    a pointer to the first character after the header name
1057 */
1058
1059 static const uschar *
1060 read_header_name(uschar *name, int max, const uschar *s)
1061 {
1062 int prelen = Ustrchr(name, '_') - name + 1;
1063 int ptr = Ustrlen(name) - prelen;
1064 if (ptr > 0) memmove(name, name+prelen, ptr);
1065 while (mac_isgraph(*s) && *s != ':')
1066   {
1067   if (ptr < max-1) name[ptr++] = *s;
1068   s++;
1069   }
1070 if (*s == ':') s++;
1071 name[ptr++] = ':';
1072 name[ptr] = 0;
1073 return s;
1074 }
1075
1076
1077
1078 /*************************************************
1079 *           Pick out a number from a string      *
1080 *************************************************/
1081
1082 /* Arguments:
1083   n     points to an integer into which to put the number
1084   s     points to the first digit of the number
1085
1086 Returns:  a pointer to the character after the last digit
1087 */
1088 /*XXX consider expanding to int_eximarith_t.  But the test for
1089 "overbig numbers" in 0002 still needs to overflow it. */
1090
1091 static uschar *
1092 read_number(int *n, uschar *s)
1093 {
1094 *n = 0;
1095 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1096 return s;
1097 }
1098
1099 static const uschar *
1100 read_cnumber(int *n, const uschar *s)
1101 {
1102 *n = 0;
1103 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1104 return s;
1105 }
1106
1107
1108
1109 /*************************************************
1110 *        Extract keyed subfield from a string    *
1111 *************************************************/
1112
1113 /* The yield is in dynamic store; NULL means that the key was not found.
1114
1115 Arguments:
1116   key       points to the name of the key
1117   s         points to the string from which to extract the subfield
1118
1119 Returns:    NULL if the subfield was not found, or
1120             a pointer to the subfield's data
1121 */
1122
1123 static uschar *
1124 expand_getkeyed(uschar *key, const uschar *s)
1125 {
1126 int length = Ustrlen(key);
1127 while (isspace(*s)) s++;
1128
1129 /* Loop to search for the key */
1130
1131 while (*s != 0)
1132   {
1133   int dkeylength;
1134   uschar *data;
1135   const uschar *dkey = s;
1136
1137   while (*s != 0 && *s != '=' && !isspace(*s)) s++;
1138   dkeylength = s - dkey;
1139   while (isspace(*s)) s++;
1140   if (*s == '=') while (isspace((*(++s))));
1141
1142   data = string_dequote(&s);
1143   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1144     return data;
1145
1146   while (isspace(*s)) s++;
1147   }
1148
1149 return NULL;
1150 }
1151
1152
1153
1154 static var_entry *
1155 find_var_ent(uschar * name)
1156 {
1157 int first = 0;
1158 int last = var_table_size;
1159
1160 while (last > first)
1161   {
1162   int middle = (first + last)/2;
1163   int c = Ustrcmp(name, var_table[middle].name);
1164
1165   if (c > 0) { first = middle + 1; continue; }
1166   if (c < 0) { last = middle; continue; }
1167   return &var_table[middle];
1168   }
1169 return NULL;
1170 }
1171
1172 /*************************************************
1173 *   Extract numbered subfield from string        *
1174 *************************************************/
1175
1176 /* Extracts a numbered field from a string that is divided by tokens - for
1177 example a line from /etc/passwd is divided by colon characters.  First field is
1178 numbered one.  Negative arguments count from the right. Zero returns the whole
1179 string. Returns NULL if there are insufficient tokens in the string
1180
1181 ***WARNING***
1182 Modifies final argument - this is a dynamically generated string, so that's OK.
1183
1184 Arguments:
1185   field       number of field to be extracted,
1186                 first field = 1, whole string = 0, last field = -1
1187   separators  characters that are used to break string into tokens
1188   s           points to the string from which to extract the subfield
1189
1190 Returns:      NULL if the field was not found,
1191               a pointer to the field's data inside s (modified to add 0)
1192 */
1193
1194 static uschar *
1195 expand_gettokened (int field, uschar *separators, uschar *s)
1196 {
1197 int sep = 1;
1198 int count;
1199 uschar *ss = s;
1200 uschar *fieldtext = NULL;
1201
1202 if (field == 0) return s;
1203
1204 /* Break the line up into fields in place; for field > 0 we stop when we have
1205 done the number of fields we want. For field < 0 we continue till the end of
1206 the string, counting the number of fields. */
1207
1208 count = (field > 0)? field : INT_MAX;
1209
1210 while (count-- > 0)
1211   {
1212   size_t len;
1213
1214   /* Previous field was the last one in the string. For a positive field
1215   number, this means there are not enough fields. For a negative field number,
1216   check that there are enough, and scan back to find the one that is wanted. */
1217
1218   if (sep == 0)
1219     {
1220     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1221     if ((-field) == (INT_MAX - count - 1)) return s;
1222     while (field++ < 0)
1223       {
1224       ss--;
1225       while (ss[-1] != 0) ss--;
1226       }
1227     fieldtext = ss;
1228     break;
1229     }
1230
1231   /* Previous field was not last in the string; save its start and put a
1232   zero at its end. */
1233
1234   fieldtext = ss;
1235   len = Ustrcspn(ss, separators);
1236   sep = ss[len];
1237   ss[len] = 0;
1238   ss += len + 1;
1239   }
1240
1241 return fieldtext;
1242 }
1243
1244
1245 static uschar *
1246 expand_getlistele(int field, const uschar * list)
1247 {
1248 const uschar * tlist= list;
1249 int sep= 0;
1250 uschar dummy;
1251
1252 if(field<0)
1253   {
1254   for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1255   sep= 0;
1256   }
1257 if(field==0) return NULL;
1258 while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1259 return string_nextinlist(&list, &sep, NULL, 0);
1260 }
1261
1262
1263 /* Certificate fields, by name.  Worry about by-OID later */
1264 /* Names are chosen to not have common prefixes */
1265
1266 #ifdef SUPPORT_TLS
1267 typedef struct
1268 {
1269 uschar * name;
1270 int      namelen;
1271 uschar * (*getfn)(void * cert, uschar * mod);
1272 } certfield;
1273 static certfield certfields[] =
1274 {                       /* linear search; no special order */
1275   { US"version",         7,  &tls_cert_version },
1276   { US"serial_number",   13, &tls_cert_serial_number },
1277   { US"subject",         7,  &tls_cert_subject },
1278   { US"notbefore",       9,  &tls_cert_not_before },
1279   { US"notafter",        8,  &tls_cert_not_after },
1280   { US"issuer",          6,  &tls_cert_issuer },
1281   { US"signature",       9,  &tls_cert_signature },
1282   { US"sig_algorithm",   13, &tls_cert_signature_algorithm },
1283   { US"subj_altname",    12, &tls_cert_subject_altname },
1284   { US"ocsp_uri",        8,  &tls_cert_ocsp_uri },
1285   { US"crl_uri",         7,  &tls_cert_crl_uri },
1286 };
1287
1288 static uschar *
1289 expand_getcertele(uschar * field, uschar * certvar)
1290 {
1291 var_entry * vp;
1292 certfield * cp;
1293
1294 if (!(vp = find_var_ent(certvar)))
1295   {
1296   expand_string_message =
1297     string_sprintf("no variable named \"%s\"", certvar);
1298   return NULL;          /* Unknown variable name */
1299   }
1300 /* NB this stops us passing certs around in variable.  Might
1301 want to do that in future */
1302 if (vp->type != vtype_cert)
1303   {
1304   expand_string_message =
1305     string_sprintf("\"%s\" is not a certificate", certvar);
1306   return NULL;          /* Unknown variable name */
1307   }
1308 if (!*(void **)vp->value)
1309   return NULL;
1310
1311 if (*field >= '0' && *field <= '9')
1312   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1313
1314 for(cp = certfields;
1315     cp < certfields + nelem(certfields);
1316     cp++)
1317   if (Ustrncmp(cp->name, field, cp->namelen) == 0)
1318     {
1319     uschar * modifier = *(field += cp->namelen) == ','
1320       ? ++field : NULL;
1321     return (*cp->getfn)( *(void **)vp->value, modifier );
1322     }
1323
1324 expand_string_message =
1325   string_sprintf("bad field selector \"%s\" for certextract", field);
1326 return NULL;
1327 }
1328 #endif  /*SUPPORT_TLS*/
1329
1330 /*************************************************
1331 *        Extract a substring from a string       *
1332 *************************************************/
1333
1334 /* Perform the ${substr or ${length expansion operations.
1335
1336 Arguments:
1337   subject     the input string
1338   value1      the offset from the start of the input string to the start of
1339                 the output string; if negative, count from the right.
1340   value2      the length of the output string, or negative (-1) for unset
1341                 if value1 is positive, unset means "all after"
1342                 if value1 is negative, unset means "all before"
1343   len         set to the length of the returned string
1344
1345 Returns:      pointer to the output string, or NULL if there is an error
1346 */
1347
1348 static uschar *
1349 extract_substr(uschar *subject, int value1, int value2, int *len)
1350 {
1351 int sublen = Ustrlen(subject);
1352
1353 if (value1 < 0)    /* count from right */
1354   {
1355   value1 += sublen;
1356
1357   /* If the position is before the start, skip to the start, and adjust the
1358   length. If the length ends up negative, the substring is null because nothing
1359   can precede. This falls out naturally when the length is unset, meaning "all
1360   to the left". */
1361
1362   if (value1 < 0)
1363     {
1364     value2 += value1;
1365     if (value2 < 0) value2 = 0;
1366     value1 = 0;
1367     }
1368
1369   /* Otherwise an unset length => characters before value1 */
1370
1371   else if (value2 < 0)
1372     {
1373     value2 = value1;
1374     value1 = 0;
1375     }
1376   }
1377
1378 /* For a non-negative offset, if the starting position is past the end of the
1379 string, the result will be the null string. Otherwise, an unset length means
1380 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1381
1382 else
1383   {
1384   if (value1 > sublen)
1385     {
1386     value1 = sublen;
1387     value2 = 0;
1388     }
1389   else if (value2 < 0) value2 = sublen;
1390   }
1391
1392 /* Cut the length down to the maximum possible for the offset value, and get
1393 the required characters. */
1394
1395 if (value1 + value2 > sublen) value2 = sublen - value1;
1396 *len = value2;
1397 return subject + value1;
1398 }
1399
1400
1401
1402
1403 /*************************************************
1404 *            Old-style hash of a string          *
1405 *************************************************/
1406
1407 /* Perform the ${hash expansion operation.
1408
1409 Arguments:
1410   subject     the input string (an expanded substring)
1411   value1      the length of the output string; if greater or equal to the
1412                 length of the input string, the input string is returned
1413   value2      the number of hash characters to use, or 26 if negative
1414   len         set to the length of the returned string
1415
1416 Returns:      pointer to the output string, or NULL if there is an error
1417 */
1418
1419 static uschar *
1420 compute_hash(uschar *subject, int value1, int value2, int *len)
1421 {
1422 int sublen = Ustrlen(subject);
1423
1424 if (value2 < 0) value2 = 26;
1425 else if (value2 > Ustrlen(hashcodes))
1426   {
1427   expand_string_message =
1428     string_sprintf("hash count \"%d\" too big", value2);
1429   return NULL;
1430   }
1431
1432 /* Calculate the hash text. We know it is shorter than the original string, so
1433 can safely place it in subject[] (we know that subject is always itself an
1434 expanded substring). */
1435
1436 if (value1 < sublen)
1437   {
1438   int c;
1439   int i = 0;
1440   int j = value1;
1441   while ((c = (subject[j])) != 0)
1442     {
1443     int shift = (c + j++) & 7;
1444     subject[i] ^= (c << shift) | (c >> (8-shift));
1445     if (++i >= value1) i = 0;
1446     }
1447   for (i = 0; i < value1; i++)
1448     subject[i] = hashcodes[(subject[i]) % value2];
1449   }
1450 else value1 = sublen;
1451
1452 *len = value1;
1453 return subject;
1454 }
1455
1456
1457
1458
1459 /*************************************************
1460 *             Numeric hash of a string           *
1461 *************************************************/
1462
1463 /* Perform the ${nhash expansion operation. The first characters of the
1464 string are treated as most important, and get the highest prime numbers.
1465
1466 Arguments:
1467   subject     the input string
1468   value1      the maximum value of the first part of the result
1469   value2      the maximum value of the second part of the result,
1470                 or negative to produce only a one-part result
1471   len         set to the length of the returned string
1472
1473 Returns:  pointer to the output string, or NULL if there is an error.
1474 */
1475
1476 static uschar *
1477 compute_nhash (uschar *subject, int value1, int value2, int *len)
1478 {
1479 uschar *s = subject;
1480 int i = 0;
1481 unsigned long int total = 0; /* no overflow */
1482
1483 while (*s != 0)
1484   {
1485   if (i == 0) i = nelem(prime) - 1;
1486   total += prime[i--] * (unsigned int)(*s++);
1487   }
1488
1489 /* If value2 is unset, just compute one number */
1490
1491 if (value2 < 0)
1492   s = string_sprintf("%lu", total % value1);
1493
1494 /* Otherwise do a div/mod hash */
1495
1496 else
1497   {
1498   total = total % (value1 * value2);
1499   s = string_sprintf("%lu/%lu", total/value2, total % value2);
1500   }
1501
1502 *len = Ustrlen(s);
1503 return s;
1504 }
1505
1506
1507
1508
1509
1510 /*************************************************
1511 *     Find the value of a header or headers      *
1512 *************************************************/
1513
1514 /* Multiple instances of the same header get concatenated, and this function
1515 can also return a concatenation of all the header lines. When concatenating
1516 specific headers that contain lists of addresses, a comma is inserted between
1517 them. Otherwise we use a straight concatenation. Because some messages can have
1518 pathologically large number of lines, there is a limit on the length that is
1519 returned. Also, to avoid massive store use which would result from using
1520 string_cat() as it copies and extends strings, we do a preliminary pass to find
1521 out exactly how much store will be needed. On "normal" messages this will be
1522 pretty trivial.
1523
1524 Arguments:
1525   name          the name of the header, without the leading $header_ or $h_,
1526                 or NULL if a concatenation of all headers is required
1527   exists_only   TRUE if called from a def: test; don't need to build a string;
1528                 just return a string that is not "" and not "0" if the header
1529                 exists
1530   newsize       return the size of memory block that was obtained; may be NULL
1531                 if exists_only is TRUE
1532   want_raw      TRUE if called for $rh_ or $rheader_ variables; no processing,
1533                 other than concatenating, will be done on the header. Also used
1534                 for $message_headers_raw.
1535   charset       name of charset to translate MIME words to; used only if
1536                 want_raw is false; if NULL, no translation is done (this is
1537                 used for $bh_ and $bheader_)
1538
1539 Returns:        NULL if the header does not exist, else a pointer to a new
1540                 store block
1541 */
1542
1543 static uschar *
1544 find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw,
1545   uschar *charset)
1546 {
1547 BOOL found = name == NULL;
1548 int comma = 0;
1549 int len = found? 0 : Ustrlen(name);
1550 int i;
1551 uschar *yield = NULL;
1552 uschar *ptr = NULL;
1553
1554 /* Loop for two passes - saves code repetition */
1555
1556 for (i = 0; i < 2; i++)
1557   {
1558   int size = 0;
1559   header_line *h;
1560
1561   for (h = header_list; size < header_insert_maxlen && h; h = h->next)
1562     if (h->type != htype_old && h->text)  /* NULL => Received: placeholder */
1563       if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1564         {
1565         int ilen;
1566         uschar *t;
1567
1568         if (exists_only) return US"1";      /* don't need actual string */
1569         found = TRUE;
1570         t = h->text + len;                  /* text to insert */
1571         if (!want_raw)                      /* unless wanted raw, */
1572           while (isspace(*t)) t++;          /* remove leading white space */
1573         ilen = h->slen - (t - h->text);     /* length to insert */
1574
1575         /* Unless wanted raw, remove trailing whitespace, including the
1576         newline. */
1577
1578         if (!want_raw)
1579           while (ilen > 0 && isspace(t[ilen-1])) ilen--;
1580
1581         /* Set comma = 1 if handling a single header and it's one of those
1582         that contains an address list, except when asked for raw headers. Only
1583         need to do this once. */
1584
1585         if (!want_raw && name && comma == 0 &&
1586             Ustrchr("BCFRST", h->type) != NULL)
1587           comma = 1;
1588
1589         /* First pass - compute total store needed; second pass - compute
1590         total store used, including this header. */
1591
1592         size += ilen + comma + 1;  /* +1 for the newline */
1593
1594         /* Second pass - concatenate the data, up to a maximum. Note that
1595         the loop stops when size hits the limit. */
1596
1597         if (i != 0)
1598           {
1599           if (size > header_insert_maxlen)
1600             {
1601             ilen -= size - header_insert_maxlen - 1;
1602             comma = 0;
1603             }
1604           Ustrncpy(ptr, t, ilen);
1605           ptr += ilen;
1606
1607           /* For a non-raw header, put in the comma if needed, then add
1608           back the newline we removed above, provided there was some text in
1609           the header. */
1610
1611           if (!want_raw && ilen > 0)
1612             {
1613             if (comma != 0) *ptr++ = ',';
1614             *ptr++ = '\n';
1615             }
1616           }
1617         }
1618
1619   /* At end of first pass, return NULL if no header found. Then truncate size
1620   if necessary, and get the buffer to hold the data, returning the buffer size.
1621   */
1622
1623   if (i == 0)
1624     {
1625     if (!found) return NULL;
1626     if (size > header_insert_maxlen) size = header_insert_maxlen;
1627     *newsize = size + 1;
1628     ptr = yield = store_get(*newsize);
1629     }
1630   }
1631
1632 /* That's all we do for raw header expansion. */
1633
1634 if (want_raw)
1635   *ptr = 0;
1636
1637 /* Otherwise, remove a final newline and a redundant added comma. Then we do
1638 RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2()
1639 function can return an error with decoded data if the charset translation
1640 fails. If decoding fails, it returns NULL. */
1641
1642 else
1643   {
1644   uschar *decoded, *error;
1645   if (ptr > yield && ptr[-1] == '\n') ptr--;
1646   if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--;
1647   *ptr = 0;
1648   decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL,
1649     newsize, &error);
1650   if (error != NULL)
1651     {
1652     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1653       "    input was: %s\n", error, yield);
1654     }
1655   if (decoded != NULL) yield = decoded;
1656   }
1657
1658 return yield;
1659 }
1660
1661
1662
1663
1664 /* Append an "iprev" element to an Autherntication-Results: header
1665 if we have attempted to get the calling host's name.
1666 */
1667
1668 static gstring *
1669 authres_iprev(gstring * g)
1670 {
1671 if (sender_host_name)
1672   return string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")");
1673 if (host_lookup_deferred)
1674   return string_catn(g, US";\n\tiprev=temperror", 19);
1675 if (host_lookup_failed)
1676   return string_catn(g, US";\n\tiprev=fail", 13);
1677 return g;
1678 }
1679
1680
1681
1682 /*************************************************
1683 *               Return list of recipients        *
1684 *************************************************/
1685 /* A recipients list is available only during system message filtering,
1686 during ACL processing after DATA, and while expanding pipe commands
1687 generated from a system filter, but not elsewhere. */
1688
1689 static uschar *
1690 fn_recipients(void)
1691 {
1692 gstring * g = NULL;
1693 int i;
1694
1695 if (!enable_dollar_recipients) return NULL;
1696
1697 for (i = 0; i < recipients_count; i++)
1698   {
1699   /*XXX variant of list_appendele? */
1700   if (i != 0) g = string_catn(g, US", ", 2);
1701   g = string_cat(g, recipients_list[i].address);
1702   }
1703 return string_from_gstring(g);
1704 }
1705
1706
1707 /*************************************************
1708 *               Find value of a variable         *
1709 *************************************************/
1710
1711 /* The table of variables is kept in alphabetic order, so we can search it
1712 using a binary chop. The "choplen" variable is nothing to do with the binary
1713 chop.
1714
1715 Arguments:
1716   name          the name of the variable being sought
1717   exists_only   TRUE if this is a def: test; passed on to find_header()
1718   skipping      TRUE => skip any processing evaluation; this is not the same as
1719                   exists_only because def: may test for values that are first
1720                   evaluated here
1721   newsize       pointer to an int which is initially zero; if the answer is in
1722                 a new memory buffer, *newsize is set to its size
1723
1724 Returns:        NULL if the variable does not exist, or
1725                 a pointer to the variable's contents, or
1726                 something non-NULL if exists_only is TRUE
1727 */
1728
1729 static uschar *
1730 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1731 {
1732 var_entry * vp;
1733 uschar *s, *domain;
1734 uschar **ss;
1735 void * val;
1736
1737 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1738 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1739 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1740 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1741 (this gave backwards compatibility at the changeover). There may be built-in
1742 variables whose names start acl_ but they should never start in this way. This
1743 slightly messy specification is a consequence of the history, needless to say.
1744
1745 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1746 set, in which case give an error. */
1747
1748 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1749      !isalpha(name[5]))
1750   {
1751   tree_node *node =
1752     tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4);
1753   return node ? node->data.ptr : strict_acl_vars ? NULL : US"";
1754   }
1755
1756 /* Handle $auth<n> variables. */
1757
1758 if (Ustrncmp(name, "auth", 4) == 0)
1759   {
1760   uschar *endptr;
1761   int n = Ustrtoul(name + 4, &endptr, 10);
1762   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1763     return !auth_vars[n-1] ? US"" : auth_vars[n-1];
1764   }
1765 else if (Ustrncmp(name, "regex", 5) == 0)
1766   {
1767   uschar *endptr;
1768   int n = Ustrtoul(name + 5, &endptr, 10);
1769   if (*endptr == 0 && n != 0 && n <= REGEX_VARS)
1770     return !regex_vars[n-1] ? US"" : regex_vars[n-1];
1771   }
1772
1773 /* For all other variables, search the table */
1774
1775 if (!(vp = find_var_ent(name)))
1776   return NULL;          /* Unknown variable name */
1777
1778 /* Found an existing variable. If in skipping state, the value isn't needed,
1779 and we want to avoid processing (such as looking up the host name). */
1780
1781 if (skipping)
1782   return US"";
1783
1784 val = vp->value;
1785 switch (vp->type)
1786   {
1787   case vtype_filter_int:
1788     if (!filter_running) return NULL;
1789     /* Fall through */
1790     /* VVVVVVVVVVVV */
1791   case vtype_int:
1792     sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1793     return var_buffer;
1794
1795   case vtype_ino:
1796     sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1797     return var_buffer;
1798
1799   case vtype_gid:
1800     sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1801     return var_buffer;
1802
1803   case vtype_uid:
1804     sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1805     return var_buffer;
1806
1807   case vtype_bool:
1808     sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1809     return var_buffer;
1810
1811   case vtype_stringptr:                      /* Pointer to string */
1812     return (s = *((uschar **)(val))) ? s : US"";
1813
1814   case vtype_pid:
1815     sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1816     return var_buffer;
1817
1818   case vtype_load_avg:
1819     sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1820     return var_buffer;
1821
1822   case vtype_host_lookup:                    /* Lookup if not done so */
1823     if (  !sender_host_name && sender_host_address
1824        && !host_lookup_failed && host_name_lookup() == OK)
1825       host_build_sender_fullhost();
1826     return sender_host_name ? sender_host_name : US"";
1827
1828   case vtype_localpart:                      /* Get local part from address */
1829     s = *((uschar **)(val));
1830     if (s == NULL) return US"";
1831     domain = Ustrrchr(s, '@');
1832     if (domain == NULL) return s;
1833     if (domain - s > sizeof(var_buffer) - 1)
1834       log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1835           " in string expansion", sizeof(var_buffer));
1836     Ustrncpy(var_buffer, s, domain - s);
1837     var_buffer[domain - s] = 0;
1838     return var_buffer;
1839
1840   case vtype_domain:                         /* Get domain from address */
1841     s = *((uschar **)(val));
1842     if (s == NULL) return US"";
1843     domain = Ustrrchr(s, '@');
1844     return (domain == NULL)? US"" : domain + 1;
1845
1846   case vtype_msgheaders:
1847     return find_header(NULL, exists_only, newsize, FALSE, NULL);
1848
1849   case vtype_msgheaders_raw:
1850     return find_header(NULL, exists_only, newsize, TRUE, NULL);
1851
1852   case vtype_msgbody:                        /* Pointer to msgbody string */
1853   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1854     ss = (uschar **)(val);
1855     if (!*ss && deliver_datafile >= 0)  /* Read body when needed */
1856       {
1857       uschar *body;
1858       off_t start_offset = SPOOL_DATA_START_OFFSET;
1859       int len = message_body_visible;
1860       if (len > message_size) len = message_size;
1861       *ss = body = store_malloc(len+1);
1862       body[0] = 0;
1863       if (vp->type == vtype_msgbody_end)
1864         {
1865         struct stat statbuf;
1866         if (fstat(deliver_datafile, &statbuf) == 0)
1867           {
1868           start_offset = statbuf.st_size - len;
1869           if (start_offset < SPOOL_DATA_START_OFFSET)
1870             start_offset = SPOOL_DATA_START_OFFSET;
1871           }
1872         }
1873       if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0)
1874         log_write(0, LOG_MAIN|LOG_PANIC_DIE, "deliver_datafile lseek: %s",
1875           strerror(errno));
1876       len = read(deliver_datafile, body, len);
1877       if (len > 0)
1878         {
1879         body[len] = 0;
1880         if (message_body_newlines)   /* Separate loops for efficiency */
1881           while (len > 0)
1882             { if (body[--len] == 0) body[len] = ' '; }
1883         else
1884           while (len > 0)
1885             { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1886         }
1887       }
1888     return *ss ? *ss : US"";
1889
1890   case vtype_todbsdin:                       /* BSD inbox time of day */
1891     return tod_stamp(tod_bsdin);
1892
1893   case vtype_tode:                           /* Unix epoch time of day */
1894     return tod_stamp(tod_epoch);
1895
1896   case vtype_todel:                          /* Unix epoch/usec time of day */
1897     return tod_stamp(tod_epoch_l);
1898
1899   case vtype_todf:                           /* Full time of day */
1900     return tod_stamp(tod_full);
1901
1902   case vtype_todl:                           /* Log format time of day */
1903     return tod_stamp(tod_log_bare);            /* (without timezone) */
1904
1905   case vtype_todzone:                        /* Time zone offset only */
1906     return tod_stamp(tod_zone);
1907
1908   case vtype_todzulu:                        /* Zulu time */
1909     return tod_stamp(tod_zulu);
1910
1911   case vtype_todlf:                          /* Log file datestamp tod */
1912     return tod_stamp(tod_log_datestamp_daily);
1913
1914   case vtype_reply:                          /* Get reply address */
1915     s = find_header(US"reply-to:", exists_only, newsize, TRUE,
1916       headers_charset);
1917     if (s != NULL) while (isspace(*s)) s++;
1918     if (s == NULL || *s == 0)
1919       {
1920       *newsize = 0;                            /* For the *s==0 case */
1921       s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset);
1922       }
1923     if (s != NULL)
1924       {
1925       uschar *t;
1926       while (isspace(*s)) s++;
1927       for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1928       while (t > s && isspace(t[-1])) t--;
1929       *t = 0;
1930       }
1931     return (s == NULL)? US"" : s;
1932
1933   case vtype_string_func:
1934     {
1935     uschar * (*fn)() = val;
1936     return fn();
1937     }
1938
1939   case vtype_pspace:
1940     {
1941     int inodes;
1942     sprintf(CS var_buffer, "%d",
1943       receive_statvfs(val == (void *)TRUE, &inodes));
1944     }
1945   return var_buffer;
1946
1947   case vtype_pinodes:
1948     {
1949     int inodes;
1950     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1951     sprintf(CS var_buffer, "%d", inodes);
1952     }
1953   return var_buffer;
1954
1955   case vtype_cert:
1956     return *(void **)val ? US"<cert>" : US"";
1957
1958 #ifndef DISABLE_DKIM
1959   case vtype_dkim:
1960     return dkim_exim_expand_query((int)(long)val);
1961 #endif
1962
1963   }
1964
1965 return NULL;  /* Unknown variable. Silences static checkers. */
1966 }
1967
1968
1969
1970
1971 void
1972 modify_variable(uschar *name, void * value)
1973 {
1974 var_entry * vp;
1975 if ((vp = find_var_ent(name))) vp->value = value;
1976 return;          /* Unknown variable name, fail silently */
1977 }
1978
1979
1980
1981
1982
1983
1984 /*************************************************
1985 *           Read and expand substrings           *
1986 *************************************************/
1987
1988 /* This function is called to read and expand argument substrings for various
1989 expansion items. Some have a minimum requirement that is less than the maximum;
1990 in these cases, the first non-present one is set to NULL.
1991
1992 Arguments:
1993   sub        points to vector of pointers to set
1994   n          maximum number of substrings
1995   m          minimum required
1996   sptr       points to current string pointer
1997   skipping   the skipping flag
1998   check_end  if TRUE, check for final '}'
1999   name       name of item, for error message
2000   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
2001              the store.
2002
2003 Returns:     0 OK; string pointer updated
2004              1 curly bracketing error (too few arguments)
2005              2 too many arguments (only if check_end is set); message set
2006              3 other error (expansion failure)
2007 */
2008
2009 static int
2010 read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping,
2011   BOOL check_end, uschar *name, BOOL *resetok)
2012 {
2013 int i;
2014 const uschar *s = *sptr;
2015
2016 while (isspace(*s)) s++;
2017 for (i = 0; i < n; i++)
2018   {
2019   if (*s != '{')
2020     {
2021     if (i < m)
2022       {
2023       expand_string_message = string_sprintf("Not enough arguments for '%s' "
2024         "(min is %d)", name, m);
2025       return 1;
2026       }
2027     sub[i] = NULL;
2028     break;
2029     }
2030   if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok)))
2031     return 3;
2032   if (*s++ != '}') return 1;
2033   while (isspace(*s)) s++;
2034   }
2035 if (check_end && *s++ != '}')
2036   {
2037   if (s[-1] == '{')
2038     {
2039     expand_string_message = string_sprintf("Too many arguments for '%s' "
2040       "(max is %d)", name, n);
2041     return 2;
2042     }
2043   expand_string_message = string_sprintf("missing '}' after '%s'", name);
2044   return 1;
2045   }
2046
2047 *sptr = s;
2048 return 0;
2049 }
2050
2051
2052
2053
2054 /*************************************************
2055 *     Elaborate message for bad variable         *
2056 *************************************************/
2057
2058 /* For the "unknown variable" message, take a look at the variable's name, and
2059 give additional information about possible ACL variables. The extra information
2060 is added on to expand_string_message.
2061
2062 Argument:   the name of the variable
2063 Returns:    nothing
2064 */
2065
2066 static void
2067 check_variable_error_message(uschar *name)
2068 {
2069 if (Ustrncmp(name, "acl_", 4) == 0)
2070   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
2071     (name[4] == 'c' || name[4] == 'm')?
2072       (isalpha(name[5])?
2073         US"6th character of a user-defined ACL variable must be a digit or underscore" :
2074         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
2075       ) :
2076       US"user-defined ACL variables must start acl_c or acl_m");
2077 }
2078
2079
2080
2081 /*
2082 Load args from sub array to globals, and call acl_check().
2083 Sub array will be corrupted on return.
2084
2085 Returns:       OK         access is granted by an ACCEPT verb
2086                DISCARD    access is (apparently) granted by a DISCARD verb
2087                FAIL       access is denied
2088                FAIL_DROP  access is denied; drop the connection
2089                DEFER      can't tell at the moment
2090                ERROR      disaster
2091 */
2092 static int
2093 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
2094 {
2095 int i;
2096 int sav_narg = acl_narg;
2097 int ret;
2098 uschar * dummy_logmsg;
2099 extern int acl_where;
2100
2101 if(--nsub > nelem(acl_arg)) nsub = nelem(acl_arg);
2102 for (i = 0; i < nsub && sub[i+1]; i++)
2103   {
2104   uschar * tmp = acl_arg[i];
2105   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2106   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2107   }
2108 acl_narg = i;
2109 while (i < nsub)
2110   {
2111   sub[i+1] = acl_arg[i];
2112   acl_arg[i++] = NULL;
2113   }
2114
2115 DEBUG(D_expand)
2116   debug_printf_indent("expanding: acl: %s  arg: %s%s\n",
2117     sub[0],
2118     acl_narg>0 ? acl_arg[0] : US"<none>",
2119     acl_narg>1 ? " +more"   : "");
2120
2121 ret = acl_eval(acl_where, sub[0], user_msgp, &dummy_logmsg);
2122
2123 for (i = 0; i < nsub; i++)
2124   acl_arg[i] = sub[i+1];        /* restore old args */
2125 acl_narg = sav_narg;
2126
2127 return ret;
2128 }
2129
2130
2131
2132
2133 /*************************************************
2134 *        Read and evaluate a condition           *
2135 *************************************************/
2136
2137 /*
2138 Arguments:
2139   s        points to the start of the condition text
2140   resetok  points to a BOOL which is written false if it is unsafe to
2141            free memory. Certain condition types (acl) may have side-effect
2142            allocation which must be preserved.
2143   yield    points to a BOOL to hold the result of the condition test;
2144            if NULL, we are just reading through a condition that is
2145            part of an "or" combination to check syntax, or in a state
2146            where the answer isn't required
2147
2148 Returns:   a pointer to the first character after the condition, or
2149            NULL after an error
2150 */
2151
2152 static const uschar *
2153 eval_condition(const uschar *s, BOOL *resetok, BOOL *yield)
2154 {
2155 BOOL testfor = TRUE;
2156 BOOL tempcond, combined_cond;
2157 BOOL *subcondptr;
2158 BOOL sub2_honour_dollar = TRUE;
2159 int i, rc, cond_type, roffset;
2160 int_eximarith_t num[2];
2161 struct stat statbuf;
2162 uschar name[256];
2163 const uschar *sub[10];
2164
2165 const pcre *re;
2166 const uschar *rerror;
2167
2168 for (;;)
2169   {
2170   while (isspace(*s)) s++;
2171   if (*s == '!') { testfor = !testfor; s++; } else break;
2172   }
2173
2174 /* Numeric comparisons are symbolic */
2175
2176 if (*s == '=' || *s == '>' || *s == '<')
2177   {
2178   int p = 0;
2179   name[p++] = *s++;
2180   if (*s == '=')
2181     {
2182     name[p++] = '=';
2183     s++;
2184     }
2185   name[p] = 0;
2186   }
2187
2188 /* All other conditions are named */
2189
2190 else s = read_name(name, 256, s, US"_");
2191
2192 /* If we haven't read a name, it means some non-alpha character is first. */
2193
2194 if (name[0] == 0)
2195   {
2196   expand_string_message = string_sprintf("condition name expected, "
2197     "but found \"%.16s\"", s);
2198   return NULL;
2199   }
2200
2201 /* Find which condition we are dealing with, and switch on it */
2202
2203 cond_type = chop_match(name, cond_table, nelem(cond_table));
2204 switch(cond_type)
2205   {
2206   /* def: tests for a non-empty variable, or for the existence of a header. If
2207   yield == NULL we are in a skipping state, and don't care about the answer. */
2208
2209   case ECOND_DEF:
2210   if (*s != ':')
2211     {
2212     expand_string_message = US"\":\" expected after \"def\"";
2213     return NULL;
2214     }
2215
2216   s = read_name(name, 256, s+1, US"_");
2217
2218   /* Test for a header's existence. If the name contains a closing brace
2219   character, this may be a user error where the terminating colon has been
2220   omitted. Set a flag to adjust a subsequent error message in this case. */
2221
2222   if (Ustrncmp(name, "h_", 2) == 0 ||
2223       Ustrncmp(name, "rh_", 3) == 0 ||
2224       Ustrncmp(name, "bh_", 3) == 0 ||
2225       Ustrncmp(name, "header_", 7) == 0 ||
2226       Ustrncmp(name, "rheader_", 8) == 0 ||
2227       Ustrncmp(name, "bheader_", 8) == 0)
2228     {
2229     s = read_header_name(name, 256, s);
2230     /* {-for-text-editors */
2231     if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2232     if (yield != NULL) *yield =
2233       (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor;
2234     }
2235
2236   /* Test for a variable's having a non-empty value. A non-existent variable
2237   causes an expansion failure. */
2238
2239   else
2240     {
2241     uschar *value = find_variable(name, TRUE, yield == NULL, NULL);
2242     if (value == NULL)
2243       {
2244       expand_string_message = (name[0] == 0)?
2245         string_sprintf("variable name omitted after \"def:\"") :
2246         string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2247       check_variable_error_message(name);
2248       return NULL;
2249       }
2250     if (yield != NULL) *yield = (value[0] != 0) == testfor;
2251     }
2252
2253   return s;
2254
2255
2256   /* first_delivery tests for first delivery attempt */
2257
2258   case ECOND_FIRST_DELIVERY:
2259   if (yield != NULL) *yield = deliver_firsttime == testfor;
2260   return s;
2261
2262
2263   /* queue_running tests for any process started by a queue runner */
2264
2265   case ECOND_QUEUE_RUNNING:
2266   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2267   return s;
2268
2269
2270   /* exists:  tests for file existence
2271        isip:  tests for any IP address
2272       isip4:  tests for an IPv4 address
2273       isip6:  tests for an IPv6 address
2274         pam:  does PAM authentication
2275      radius:  does RADIUS authentication
2276    ldapauth:  does LDAP authentication
2277     pwcheck:  does Cyrus SASL pwcheck authentication
2278   */
2279
2280   case ECOND_EXISTS:
2281   case ECOND_ISIP:
2282   case ECOND_ISIP4:
2283   case ECOND_ISIP6:
2284   case ECOND_PAM:
2285   case ECOND_RADIUS:
2286   case ECOND_LDAPAUTH:
2287   case ECOND_PWCHECK:
2288
2289   while (isspace(*s)) s++;
2290   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2291
2292   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2293   if (sub[0] == NULL) return NULL;
2294   /* {-for-text-editors */
2295   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2296
2297   if (yield == NULL) return s;   /* No need to run the test if skipping */
2298
2299   switch(cond_type)
2300     {
2301     case ECOND_EXISTS:
2302     if ((expand_forbid & RDO_EXISTS) != 0)
2303       {
2304       expand_string_message = US"File existence tests are not permitted";
2305       return NULL;
2306       }
2307     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2308     break;
2309
2310     case ECOND_ISIP:
2311     case ECOND_ISIP4:
2312     case ECOND_ISIP6:
2313     rc = string_is_ip_address(sub[0], NULL);
2314     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2315              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2316     break;
2317
2318     /* Various authentication tests - all optionally compiled */
2319
2320     case ECOND_PAM:
2321     #ifdef SUPPORT_PAM
2322     rc = auth_call_pam(sub[0], &expand_string_message);
2323     goto END_AUTH;
2324     #else
2325     goto COND_FAILED_NOT_COMPILED;
2326     #endif  /* SUPPORT_PAM */
2327
2328     case ECOND_RADIUS:
2329     #ifdef RADIUS_CONFIG_FILE
2330     rc = auth_call_radius(sub[0], &expand_string_message);
2331     goto END_AUTH;
2332     #else
2333     goto COND_FAILED_NOT_COMPILED;
2334     #endif  /* RADIUS_CONFIG_FILE */
2335
2336     case ECOND_LDAPAUTH:
2337     #ifdef LOOKUP_LDAP
2338       {
2339       /* Just to keep the interface the same */
2340       BOOL do_cache;
2341       int old_pool = store_pool;
2342       store_pool = POOL_SEARCH;
2343       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2344         &expand_string_message, &do_cache);
2345       store_pool = old_pool;
2346       }
2347     goto END_AUTH;
2348     #else
2349     goto COND_FAILED_NOT_COMPILED;
2350     #endif  /* LOOKUP_LDAP */
2351
2352     case ECOND_PWCHECK:
2353     #ifdef CYRUS_PWCHECK_SOCKET
2354     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2355     goto END_AUTH;
2356     #else
2357     goto COND_FAILED_NOT_COMPILED;
2358     #endif  /* CYRUS_PWCHECK_SOCKET */
2359
2360     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2361         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2362     END_AUTH:
2363     if (rc == ERROR || rc == DEFER) return NULL;
2364     *yield = (rc == OK) == testfor;
2365     #endif
2366     }
2367   return s;
2368
2369
2370   /* call ACL (in a conditional context).  Accept true, deny false.
2371   Defer is a forced-fail.  Anything set by message= goes to $value.
2372   Up to ten parameters are used; we use the braces round the name+args
2373   like the saslauthd condition does, to permit a variable number of args.
2374   See also the expansion-item version EITEM_ACL and the traditional
2375   acl modifier ACLC_ACL.
2376   Since the ACL may allocate new global variables, tell our caller to not
2377   reclaim memory.
2378   */
2379
2380   case ECOND_ACL:
2381     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2382     {
2383     uschar *sub[10];
2384     uschar *user_msg;
2385     BOOL cond = FALSE;
2386
2387     while (isspace(*s)) s++;
2388     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2389
2390     switch(read_subs(sub, nelem(sub), 1,
2391       &s, yield == NULL, TRUE, US"acl", resetok))
2392       {
2393       case 1: expand_string_message = US"too few arguments or bracketing "
2394         "error for acl";
2395       case 2:
2396       case 3: return NULL;
2397       }
2398
2399     if (yield != NULL)
2400       {
2401       *resetok = FALSE; /* eval_acl() might allocate; do not reclaim */
2402       switch(eval_acl(sub, nelem(sub), &user_msg))
2403         {
2404         case OK:
2405           cond = TRUE;
2406         case FAIL:
2407           lookup_value = NULL;
2408           if (user_msg)
2409             lookup_value = string_copy(user_msg);
2410           *yield = cond == testfor;
2411           break;
2412
2413         case DEFER:
2414           expand_string_forcedfail = TRUE;
2415           /*FALLTHROUGH*/
2416         default:
2417           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
2418           return NULL;
2419         }
2420       }
2421     return s;
2422     }
2423
2424
2425   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2426
2427      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2428
2429   However, the last two are optional. That is why the whole set is enclosed
2430   in their own set of braces. */
2431
2432   case ECOND_SASLAUTHD:
2433 #ifndef CYRUS_SASLAUTHD_SOCKET
2434     goto COND_FAILED_NOT_COMPILED;
2435 #else
2436     {
2437     uschar *sub[4];
2438     while (isspace(*s)) s++;
2439     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2440     switch(read_subs(sub, nelem(sub), 2, &s, yield == NULL, TRUE, US"saslauthd",
2441                     resetok))
2442       {
2443       case 1: expand_string_message = US"too few arguments or bracketing "
2444         "error for saslauthd";
2445       case 2:
2446       case 3: return NULL;
2447       }
2448     if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2449     if (yield != NULL)
2450       {
2451       int rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2452         &expand_string_message);
2453       if (rc == ERROR || rc == DEFER) return NULL;
2454       *yield = (rc == OK) == testfor;
2455       }
2456     return s;
2457     }
2458 #endif /* CYRUS_SASLAUTHD_SOCKET */
2459
2460
2461   /* symbolic operators for numeric and string comparison, and a number of
2462   other operators, all requiring two arguments.
2463
2464   crypteq:           encrypts plaintext and compares against an encrypted text,
2465                        using crypt(), crypt16(), MD5 or SHA-1
2466   inlist/inlisti:    checks if first argument is in the list of the second
2467   match:             does a regular expression match and sets up the numerical
2468                        variables if it succeeds
2469   match_address:     matches in an address list
2470   match_domain:      matches in a domain list
2471   match_ip:          matches a host list that is restricted to IP addresses
2472   match_local_part:  matches in a local part list
2473   */
2474
2475   case ECOND_MATCH_ADDRESS:
2476   case ECOND_MATCH_DOMAIN:
2477   case ECOND_MATCH_IP:
2478   case ECOND_MATCH_LOCAL_PART:
2479 #ifndef EXPAND_LISTMATCH_RHS
2480     sub2_honour_dollar = FALSE;
2481 #endif
2482     /* FALLTHROUGH */
2483
2484   case ECOND_CRYPTEQ:
2485   case ECOND_INLIST:
2486   case ECOND_INLISTI:
2487   case ECOND_MATCH:
2488
2489   case ECOND_NUM_L:     /* Numerical comparisons */
2490   case ECOND_NUM_LE:
2491   case ECOND_NUM_E:
2492   case ECOND_NUM_EE:
2493   case ECOND_NUM_G:
2494   case ECOND_NUM_GE:
2495
2496   case ECOND_STR_LT:    /* String comparisons */
2497   case ECOND_STR_LTI:
2498   case ECOND_STR_LE:
2499   case ECOND_STR_LEI:
2500   case ECOND_STR_EQ:
2501   case ECOND_STR_EQI:
2502   case ECOND_STR_GT:
2503   case ECOND_STR_GTI:
2504   case ECOND_STR_GE:
2505   case ECOND_STR_GEI:
2506
2507   for (i = 0; i < 2; i++)
2508     {
2509     /* Sometimes, we don't expand substrings; too many insecure configurations
2510     created using match_address{}{} and friends, where the second param
2511     includes information from untrustworthy sources. */
2512     BOOL honour_dollar = TRUE;
2513     if ((i > 0) && !sub2_honour_dollar)
2514       honour_dollar = FALSE;
2515
2516     while (isspace(*s)) s++;
2517     if (*s != '{')
2518       {
2519       if (i == 0) goto COND_FAILED_CURLY_START;
2520       expand_string_message = string_sprintf("missing 2nd string in {} "
2521         "after \"%s\"", name);
2522       return NULL;
2523       }
2524     if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2525         honour_dollar, resetok)))
2526       return NULL;
2527     DEBUG(D_expand) if (i == 1 && !sub2_honour_dollar && Ustrchr(sub[1], '$'))
2528       debug_printf_indent("WARNING: the second arg is NOT expanded,"
2529                         " for security reasons\n");
2530     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2531
2532     /* Convert to numerical if required; we know that the names of all the
2533     conditions that compare numbers do not start with a letter. This just saves
2534     checking for them individually. */
2535
2536     if (!isalpha(name[0]) && yield != NULL)
2537       if (sub[i][0] == 0)
2538         {
2539         num[i] = 0;
2540         DEBUG(D_expand)
2541           debug_printf_indent("empty string cast to zero for numerical comparison\n");
2542         }
2543       else
2544         {
2545         num[i] = expanded_string_integer(sub[i], FALSE);
2546         if (expand_string_message != NULL) return NULL;
2547         }
2548     }
2549
2550   /* Result not required */
2551
2552   if (yield == NULL) return s;
2553
2554   /* Do an appropriate comparison */
2555
2556   switch(cond_type)
2557     {
2558     case ECOND_NUM_E:
2559     case ECOND_NUM_EE:
2560     tempcond = (num[0] == num[1]);
2561     break;
2562
2563     case ECOND_NUM_G:
2564     tempcond = (num[0] > num[1]);
2565     break;
2566
2567     case ECOND_NUM_GE:
2568     tempcond = (num[0] >= num[1]);
2569     break;
2570
2571     case ECOND_NUM_L:
2572     tempcond = (num[0] < num[1]);
2573     break;
2574
2575     case ECOND_NUM_LE:
2576     tempcond = (num[0] <= num[1]);
2577     break;
2578
2579     case ECOND_STR_LT:
2580     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2581     break;
2582
2583     case ECOND_STR_LTI:
2584     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2585     break;
2586
2587     case ECOND_STR_LE:
2588     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2589     break;
2590
2591     case ECOND_STR_LEI:
2592     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2593     break;
2594
2595     case ECOND_STR_EQ:
2596     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2597     break;
2598
2599     case ECOND_STR_EQI:
2600     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2601     break;
2602
2603     case ECOND_STR_GT:
2604     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2605     break;
2606
2607     case ECOND_STR_GTI:
2608     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2609     break;
2610
2611     case ECOND_STR_GE:
2612     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2613     break;
2614
2615     case ECOND_STR_GEI:
2616     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2617     break;
2618
2619     case ECOND_MATCH:   /* Regular expression match */
2620     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2621       NULL);
2622     if (re == NULL)
2623       {
2624       expand_string_message = string_sprintf("regular expression error in "
2625         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2626       return NULL;
2627       }
2628     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2629     break;
2630
2631     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2632     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2633     goto MATCHED_SOMETHING;
2634
2635     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2636     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2637       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2638     goto MATCHED_SOMETHING;
2639
2640     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2641     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2642       {
2643       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2644         sub[0]);
2645       return NULL;
2646       }
2647     else
2648       {
2649       unsigned int *nullcache = NULL;
2650       check_host_block cb;
2651
2652       cb.host_name = US"";
2653       cb.host_address = sub[0];
2654
2655       /* If the host address starts off ::ffff: it is an IPv6 address in
2656       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2657       addresses. */
2658
2659       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2660         cb.host_address + 7 : cb.host_address;
2661
2662       rc = match_check_list(
2663              &sub[1],                   /* the list */
2664              0,                         /* separator character */
2665              &hostlist_anchor,          /* anchor pointer */
2666              &nullcache,                /* cache pointer */
2667              check_host,                /* function for testing */
2668              &cb,                       /* argument for function */
2669              MCL_HOST,                  /* type of check */
2670              sub[0],                    /* text for debugging */
2671              NULL);                     /* where to pass back data */
2672       }
2673     goto MATCHED_SOMETHING;
2674
2675     case ECOND_MATCH_LOCAL_PART:
2676     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2677       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2678     /* Fall through */
2679     /* VVVVVVVVVVVV */
2680     MATCHED_SOMETHING:
2681     switch(rc)
2682       {
2683       case OK:
2684       tempcond = TRUE;
2685       break;
2686
2687       case FAIL:
2688       tempcond = FALSE;
2689       break;
2690
2691       case DEFER:
2692       expand_string_message = string_sprintf("unable to complete match "
2693         "against \"%s\": %s", sub[1], search_error_message);
2694       return NULL;
2695       }
2696
2697     break;
2698
2699     /* Various "encrypted" comparisons. If the second string starts with
2700     "{" then an encryption type is given. Default to crypt() or crypt16()
2701     (build-time choice). */
2702     /* }-for-text-editors */
2703
2704     case ECOND_CRYPTEQ:
2705     #ifndef SUPPORT_CRYPTEQ
2706     goto COND_FAILED_NOT_COMPILED;
2707     #else
2708     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2709       {
2710       int sublen = Ustrlen(sub[1]+5);
2711       md5 base;
2712       uschar digest[16];
2713
2714       md5_start(&base);
2715       md5_end(&base, sub[0], Ustrlen(sub[0]), digest);
2716
2717       /* If the length that we are comparing against is 24, the MD5 digest
2718       is expressed as a base64 string. This is the way LDAP does it. However,
2719       some other software uses a straightforward hex representation. We assume
2720       this if the length is 32. Other lengths fail. */
2721
2722       if (sublen == 24)
2723         {
2724         uschar *coded = b64encode(digest, 16);
2725         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2726           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2727         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2728         }
2729       else if (sublen == 32)
2730         {
2731         int i;
2732         uschar coded[36];
2733         for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2734         coded[32] = 0;
2735         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2736           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2737         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2738         }
2739       else
2740         {
2741         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2742           "fail\n  crypted=%s\n", sub[1]+5);
2743         tempcond = FALSE;
2744         }
2745       }
2746
2747     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2748       {
2749       int sublen = Ustrlen(sub[1]+6);
2750       hctx h;
2751       uschar digest[20];
2752
2753       sha1_start(&h);
2754       sha1_end(&h, sub[0], Ustrlen(sub[0]), digest);
2755
2756       /* If the length that we are comparing against is 28, assume the SHA1
2757       digest is expressed as a base64 string. If the length is 40, assume a
2758       straightforward hex representation. Other lengths fail. */
2759
2760       if (sublen == 28)
2761         {
2762         uschar *coded = b64encode(digest, 20);
2763         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2764           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2765         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2766         }
2767       else if (sublen == 40)
2768         {
2769         int i;
2770         uschar coded[44];
2771         for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2772         coded[40] = 0;
2773         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2774           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2775         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2776         }
2777       else
2778         {
2779         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2780           "fail\n  crypted=%s\n", sub[1]+6);
2781         tempcond = FALSE;
2782         }
2783       }
2784
2785     else   /* {crypt} or {crypt16} and non-{ at start */
2786            /* }-for-text-editors */
2787       {
2788       int which = 0;
2789       uschar *coded;
2790
2791       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2792         {
2793         sub[1] += 7;
2794         which = 1;
2795         }
2796       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2797         {
2798         sub[1] += 9;
2799         which = 2;
2800         }
2801       else if (sub[1][0] == '{')                /* }-for-text-editors */
2802         {
2803         expand_string_message = string_sprintf("unknown encryption mechanism "
2804           "in \"%s\"", sub[1]);
2805         return NULL;
2806         }
2807
2808       switch(which)
2809         {
2810         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2811         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2812         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2813         }
2814
2815       #define STR(s) # s
2816       #define XSTR(s) STR(s)
2817       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2818         "  subject=%s\n  crypted=%s\n",
2819         which == 0 ? XSTR(DEFAULT_CRYPT) : which == 1 ? "crypt" : "crypt16",
2820         coded, sub[1]);
2821       #undef STR
2822       #undef XSTR
2823
2824       /* If the encrypted string contains fewer than two characters (for the
2825       salt), force failure. Otherwise we get false positives: with an empty
2826       string the yield of crypt() is an empty string! */
2827
2828       if (coded)
2829         tempcond = Ustrlen(sub[1]) < 2 ? FALSE : Ustrcmp(coded, sub[1]) == 0;
2830       else if (errno == EINVAL)
2831         tempcond = FALSE;
2832       else
2833         {
2834         expand_string_message = string_sprintf("crypt error: %s\n",
2835           US strerror(errno));
2836         return NULL;
2837         }
2838       }
2839     break;
2840     #endif  /* SUPPORT_CRYPTEQ */
2841
2842     case ECOND_INLIST:
2843     case ECOND_INLISTI:
2844       {
2845       const uschar * list = sub[1];
2846       int sep = 0;
2847       uschar *save_iterate_item = iterate_item;
2848       int (*compare)(const uschar *, const uschar *);
2849
2850       DEBUG(D_expand) debug_printf_indent("condition: %s  item: %s\n", name, sub[0]);
2851
2852       tempcond = FALSE;
2853       compare = cond_type == ECOND_INLISTI
2854         ? strcmpic : (int (*)(const uschar *, const uschar *)) strcmp;
2855
2856       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
2857         {
2858         DEBUG(D_expand) debug_printf_indent(" compare %s\n", iterate_item);
2859         if (compare(sub[0], iterate_item) == 0)
2860           {
2861           tempcond = TRUE;
2862           break;
2863           }
2864         }
2865       iterate_item = save_iterate_item;
2866       }
2867
2868     }   /* Switch for comparison conditions */
2869
2870   *yield = tempcond == testfor;
2871   return s;    /* End of comparison conditions */
2872
2873
2874   /* and/or: computes logical and/or of several conditions */
2875
2876   case ECOND_AND:
2877   case ECOND_OR:
2878   subcondptr = (yield == NULL)? NULL : &tempcond;
2879   combined_cond = (cond_type == ECOND_AND);
2880
2881   while (isspace(*s)) s++;
2882   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2883
2884   for (;;)
2885     {
2886     while (isspace(*s)) s++;
2887     /* {-for-text-editors */
2888     if (*s == '}') break;
2889     if (*s != '{')                                      /* }-for-text-editors */
2890       {
2891       expand_string_message = string_sprintf("each subcondition "
2892         "inside an \"%s{...}\" condition must be in its own {}", name);
2893       return NULL;
2894       }
2895
2896     if (!(s = eval_condition(s+1, resetok, subcondptr)))
2897       {
2898       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
2899         expand_string_message, name);
2900       return NULL;
2901       }
2902     while (isspace(*s)) s++;
2903
2904     /* {-for-text-editors */
2905     if (*s++ != '}')
2906       {
2907       /* {-for-text-editors */
2908       expand_string_message = string_sprintf("missing } at end of condition "
2909         "inside \"%s\" group", name);
2910       return NULL;
2911       }
2912
2913     if (yield != NULL)
2914       {
2915       if (cond_type == ECOND_AND)
2916         {
2917         combined_cond &= tempcond;
2918         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
2919         }                                       /* evaluate any more */
2920       else
2921         {
2922         combined_cond |= tempcond;
2923         if (combined_cond) subcondptr = NULL;   /* once true, don't */
2924         }                                       /* evaluate any more */
2925       }
2926     }
2927
2928   if (yield != NULL) *yield = (combined_cond == testfor);
2929   return ++s;
2930
2931
2932   /* forall/forany: iterates a condition with different values */
2933
2934   case ECOND_FORALL:
2935   case ECOND_FORANY:
2936     {
2937     const uschar * list;
2938     int sep = 0;
2939     uschar *save_iterate_item = iterate_item;
2940
2941     DEBUG(D_expand) debug_printf_indent("condition: %s\n", name);
2942
2943     while (isspace(*s)) s++;
2944     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2945     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
2946     if (sub[0] == NULL) return NULL;
2947     /* {-for-text-editors */
2948     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2949
2950     while (isspace(*s)) s++;
2951     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2952
2953     sub[1] = s;
2954
2955     /* Call eval_condition once, with result discarded (as if scanning a
2956     "false" part). This allows us to find the end of the condition, because if
2957     the list it empty, we won't actually evaluate the condition for real. */
2958
2959     if (!(s = eval_condition(sub[1], resetok, NULL)))
2960       {
2961       expand_string_message = string_sprintf("%s inside \"%s\" condition",
2962         expand_string_message, name);
2963       return NULL;
2964       }
2965     while (isspace(*s)) s++;
2966
2967     /* {-for-text-editors */
2968     if (*s++ != '}')
2969       {
2970       /* {-for-text-editors */
2971       expand_string_message = string_sprintf("missing } at end of condition "
2972         "inside \"%s\"", name);
2973       return NULL;
2974       }
2975
2976     if (yield != NULL) *yield = !testfor;
2977     list = sub[0];
2978     while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
2979       {
2980       DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, iterate_item);
2981       if (!eval_condition(sub[1], resetok, &tempcond))
2982         {
2983         expand_string_message = string_sprintf("%s inside \"%s\" condition",
2984           expand_string_message, name);
2985         iterate_item = save_iterate_item;
2986         return NULL;
2987         }
2988       DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", name,
2989         tempcond? "true":"false");
2990
2991       if (yield != NULL) *yield = (tempcond == testfor);
2992       if (tempcond == (cond_type == ECOND_FORANY)) break;
2993       }
2994
2995     iterate_item = save_iterate_item;
2996     return s;
2997     }
2998
2999
3000   /* The bool{} expansion condition maps a string to boolean.
3001   The values supported should match those supported by the ACL condition
3002   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
3003   of true/false.  Note that Router "condition" rules have a different
3004   interpretation, where general data can be used and only a few values
3005   map to FALSE.
3006   Note that readconf.c boolean matching, for boolean configuration options,
3007   only matches true/yes/false/no.
3008   The bool_lax{} condition matches the Router logic, which is much more
3009   liberal. */
3010   case ECOND_BOOL:
3011   case ECOND_BOOL_LAX:
3012     {
3013     uschar *sub_arg[1];
3014     uschar *t, *t2;
3015     uschar *ourname;
3016     size_t len;
3017     BOOL boolvalue = FALSE;
3018     while (isspace(*s)) s++;
3019     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
3020     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
3021     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
3022       {
3023       case 1: expand_string_message = string_sprintf(
3024                   "too few arguments or bracketing error for %s",
3025                   ourname);
3026       /*FALLTHROUGH*/
3027       case 2:
3028       case 3: return NULL;
3029       }
3030     t = sub_arg[0];
3031     while (isspace(*t)) t++;
3032     len = Ustrlen(t);
3033     if (len)
3034       {
3035       /* trailing whitespace: seems like a good idea to ignore it too */
3036       t2 = t + len - 1;
3037       while (isspace(*t2)) t2--;
3038       if (t2 != (t + len))
3039         {
3040         *++t2 = '\0';
3041         len = t2 - t;
3042         }
3043       }
3044     DEBUG(D_expand)
3045       debug_printf_indent("considering %s: %s\n", ourname, len ? t : US"<empty>");
3046     /* logic for the lax case from expand_check_condition(), which also does
3047     expands, and the logic is both short and stable enough that there should
3048     be no maintenance burden from replicating it. */
3049     if (len == 0)
3050       boolvalue = FALSE;
3051     else if (*t == '-'
3052              ? Ustrspn(t+1, "0123456789") == len-1
3053              : Ustrspn(t,   "0123456789") == len)
3054       {
3055       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
3056       /* expand_check_condition only does a literal string "0" check */
3057       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
3058         boolvalue = TRUE;
3059       }
3060     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
3061       boolvalue = TRUE;
3062     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
3063       boolvalue = FALSE;
3064     else if (cond_type == ECOND_BOOL_LAX)
3065       boolvalue = TRUE;
3066     else
3067       {
3068       expand_string_message = string_sprintf("unrecognised boolean "
3069        "value \"%s\"", t);
3070       return NULL;
3071       }
3072     DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", ourname,
3073         boolvalue? "true":"false");
3074     if (yield != NULL) *yield = (boolvalue == testfor);
3075     return s;
3076     }
3077
3078   /* Unknown condition */
3079
3080   default:
3081   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
3082   return NULL;
3083   }   /* End switch on condition type */
3084
3085 /* Missing braces at start and end of data */
3086
3087 COND_FAILED_CURLY_START:
3088 expand_string_message = string_sprintf("missing { after \"%s\"", name);
3089 return NULL;
3090
3091 COND_FAILED_CURLY_END:
3092 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
3093   name);
3094 return NULL;
3095
3096 /* A condition requires code that is not compiled */
3097
3098 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
3099     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
3100     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
3101 COND_FAILED_NOT_COMPILED:
3102 expand_string_message = string_sprintf("support for \"%s\" not compiled",
3103   name);
3104 return NULL;
3105 #endif
3106 }
3107
3108
3109
3110
3111 /*************************************************
3112 *          Save numerical variables              *
3113 *************************************************/
3114
3115 /* This function is called from items such as "if" that want to preserve and
3116 restore the numbered variables.
3117
3118 Arguments:
3119   save_expand_string    points to an array of pointers to set
3120   save_expand_nlength   points to an array of ints for the lengths
3121
3122 Returns:                the value of expand max to save
3123 */
3124
3125 static int
3126 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3127 {
3128 int i;
3129 for (i = 0; i <= expand_nmax; i++)
3130   {
3131   save_expand_nstring[i] = expand_nstring[i];
3132   save_expand_nlength[i] = expand_nlength[i];
3133   }
3134 return expand_nmax;
3135 }
3136
3137
3138
3139 /*************************************************
3140 *           Restore numerical variables          *
3141 *************************************************/
3142
3143 /* This function restored saved values of numerical strings.
3144
3145 Arguments:
3146   save_expand_nmax      the number of strings to restore
3147   save_expand_string    points to an array of pointers
3148   save_expand_nlength   points to an array of ints
3149
3150 Returns:                nothing
3151 */
3152
3153 static void
3154 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3155   int *save_expand_nlength)
3156 {
3157 int i;
3158 expand_nmax = save_expand_nmax;
3159 for (i = 0; i <= expand_nmax; i++)
3160   {
3161   expand_nstring[i] = save_expand_nstring[i];
3162   expand_nlength[i] = save_expand_nlength[i];
3163   }
3164 }
3165
3166
3167
3168
3169
3170 /*************************************************
3171 *            Handle yes/no substrings            *
3172 *************************************************/
3173
3174 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3175 alternative substrings that depend on whether or not the condition was true,
3176 or the lookup or extraction succeeded. The substrings always have to be
3177 expanded, to check their syntax, but "skipping" is set when the result is not
3178 needed - this avoids unnecessary nested lookups.
3179
3180 Arguments:
3181   skipping       TRUE if we were skipping when this item was reached
3182   yes            TRUE if the first string is to be used, else use the second
3183   save_lookup    a value to put back into lookup_value before the 2nd expansion
3184   sptr           points to the input string pointer
3185   yieldptr       points to the output growable-string pointer
3186   type           "lookup", "if", "extract", "run", "env", "listextract" or
3187                  "certextract" for error message
3188   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3189                 the store.
3190
3191 Returns:         0 OK; lookup_value has been reset to save_lookup
3192                  1 expansion failed
3193                  2 expansion failed because of bracketing error
3194 */
3195
3196 static int
3197 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr,
3198   gstring ** yieldptr, uschar *type, BOOL *resetok)
3199 {
3200 int rc = 0;
3201 const uschar *s = *sptr;    /* Local value */
3202 uschar *sub1, *sub2;
3203 const uschar * errwhere;
3204
3205 /* If there are no following strings, we substitute the contents of $value for
3206 lookups and for extractions in the success case. For the ${if item, the string
3207 "true" is substituted. In the fail case, nothing is substituted for all three
3208 items. */
3209
3210 while (isspace(*s)) s++;
3211 if (*s == '}')
3212   {
3213   if (type[0] == 'i')
3214     {
3215     if (yes && !skipping)
3216       *yieldptr = string_catn(*yieldptr, US"true", 4);
3217     }
3218   else
3219     {
3220     if (yes && lookup_value && !skipping)
3221       *yieldptr = string_cat(*yieldptr, lookup_value);
3222     lookup_value = save_lookup;
3223     }
3224   s++;
3225   goto RETURN;
3226   }
3227
3228 /* The first following string must be braced. */
3229
3230 if (*s++ != '{')
3231   {
3232   errwhere = US"'yes' part did not start with '{'";
3233   goto FAILED_CURLY;
3234   }
3235
3236 /* Expand the first substring. Forced failures are noticed only if we actually
3237 want this string. Set skipping in the call in the fail case (this will always
3238 be the case if we were already skipping). */
3239
3240 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3241 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
3242 expand_string_forcedfail = FALSE;
3243 if (*s++ != '}')
3244   {
3245   errwhere = US"'yes' part did not end with '}'";
3246   goto FAILED_CURLY;
3247   }
3248
3249 /* If we want the first string, add it to the output */
3250
3251 if (yes)
3252   *yieldptr = string_cat(*yieldptr, sub1);
3253
3254 /* If this is called from a lookup/env or a (cert)extract, we want to restore
3255 $value to what it was at the start of the item, so that it has this value
3256 during the second string expansion. For the call from "if" or "run" to this
3257 function, save_lookup is set to lookup_value, so that this statement does
3258 nothing. */
3259
3260 lookup_value = save_lookup;
3261
3262 /* There now follows either another substring, or "fail", or nothing. This
3263 time, forced failures are noticed only if we want the second string. We must
3264 set skipping in the nested call if we don't want this string, or if we were
3265 already skipping. */
3266
3267 while (isspace(*s)) s++;
3268 if (*s == '{')
3269   {
3270   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3271   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
3272   expand_string_forcedfail = FALSE;
3273   if (*s++ != '}')
3274     {
3275     errwhere = US"'no' part did not start with '{'";
3276     goto FAILED_CURLY;
3277     }
3278
3279   /* If we want the second string, add it to the output */
3280
3281   if (!yes)
3282     *yieldptr = string_cat(*yieldptr, sub2);
3283   }
3284
3285 /* If there is no second string, but the word "fail" is present when the use of
3286 the second string is wanted, set a flag indicating it was a forced failure
3287 rather than a syntactic error. Swallow the terminating } in case this is nested
3288 inside another lookup or if or extract. */
3289
3290 else if (*s != '}')
3291   {
3292   uschar name[256];
3293   /* deconst cast ok here as source is s anyway */
3294   s = US read_name(name, sizeof(name), s, US"_");
3295   if (Ustrcmp(name, "fail") == 0)
3296     {
3297     if (!yes && !skipping)
3298       {
3299       while (isspace(*s)) s++;
3300       if (*s++ != '}')
3301         {
3302         errwhere = US"did not close with '}' after forcedfail";
3303         goto FAILED_CURLY;
3304         }
3305       expand_string_message =
3306         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3307       expand_string_forcedfail = TRUE;
3308       goto FAILED;
3309       }
3310     }
3311   else
3312     {
3313     expand_string_message =
3314       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3315     goto FAILED;
3316     }
3317   }
3318
3319 /* All we have to do now is to check on the final closing brace. */
3320
3321 while (isspace(*s)) s++;
3322 if (*s++ != '}')
3323   {
3324   errwhere = US"did not close with '}'";
3325   goto FAILED_CURLY;
3326   }
3327
3328
3329 RETURN:
3330 /* Update the input pointer value before returning */
3331 *sptr = s;
3332 return rc;
3333
3334 FAILED_CURLY:
3335   /* Get here if there is a bracketing failure */
3336   expand_string_message = string_sprintf(
3337     "curly-bracket problem in conditional yes/no parsing: %s\n"
3338     " remaining string is '%s'", errwhere, --s);
3339   rc = 2;
3340   goto RETURN;
3341
3342 FAILED:
3343   /* Get here for other failures */
3344   rc = 1;
3345   goto RETURN;
3346 }
3347
3348
3349
3350
3351 /*************************************************
3352 *    Handle MD5 or SHA-1 computation for HMAC    *
3353 *************************************************/
3354
3355 /* These are some wrapping functions that enable the HMAC code to be a bit
3356 cleaner. A good compiler will spot the tail recursion.
3357
3358 Arguments:
3359   type         HMAC_MD5 or HMAC_SHA1
3360   remaining    are as for the cryptographic hash functions
3361
3362 Returns:       nothing
3363 */
3364
3365 static void
3366 chash_start(int type, void *base)
3367 {
3368 if (type == HMAC_MD5)
3369   md5_start((md5 *)base);
3370 else
3371   sha1_start((hctx *)base);
3372 }
3373
3374 static void
3375 chash_mid(int type, void *base, uschar *string)
3376 {
3377 if (type == HMAC_MD5)
3378   md5_mid((md5 *)base, string);
3379 else
3380   sha1_mid((hctx *)base, string);
3381 }
3382
3383 static void
3384 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3385 {
3386 if (type == HMAC_MD5)
3387   md5_end((md5 *)base, string, length, digest);
3388 else
3389   sha1_end((hctx *)base, string, length, digest);
3390 }
3391
3392
3393
3394
3395
3396 /********************************************************
3397 * prvs: Get last three digits of days since Jan 1, 1970 *
3398 ********************************************************/
3399
3400 /* This is needed to implement the "prvs" BATV reverse
3401    path signing scheme
3402
3403 Argument: integer "days" offset to add or substract to
3404           or from the current number of days.
3405
3406 Returns:  pointer to string containing the last three
3407           digits of the number of days since Jan 1, 1970,
3408           modified by the offset argument, NULL if there
3409           was an error in the conversion.
3410
3411 */
3412
3413 static uschar *
3414 prvs_daystamp(int day_offset)
3415 {
3416 uschar *days = store_get(32);                /* Need at least 24 for cases */
3417 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3418   (time(NULL) + day_offset*86400)/86400);
3419 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3420 }
3421
3422
3423
3424 /********************************************************
3425 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3426 ********************************************************/
3427
3428 /* This is needed to implement the "prvs" BATV reverse
3429    path signing scheme
3430
3431 Arguments:
3432   address RFC2821 Address to use
3433       key The key to use (must be less than 64 characters
3434           in size)
3435   key_num Single-digit key number to use. Defaults to
3436           '0' when NULL.
3437
3438 Returns:  pointer to string containing the first three
3439           bytes of the final hash in hex format, NULL if
3440           there was an error in the process.
3441 */
3442
3443 static uschar *
3444 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3445 {
3446 gstring * hash_source;
3447 uschar * p;
3448 int i;
3449 hctx h;
3450 uschar innerhash[20];
3451 uschar finalhash[20];
3452 uschar innerkey[64];
3453 uschar outerkey[64];
3454 uschar *finalhash_hex = store_get(40);
3455
3456 if (key_num == NULL)
3457   key_num = US"0";
3458
3459 if (Ustrlen(key) > 64)
3460   return NULL;
3461
3462 hash_source = string_catn(NULL, key_num, 1);
3463 hash_source = string_catn(hash_source, daystamp, 3);
3464 hash_source = string_cat(hash_source, address);
3465 (void) string_from_gstring(hash_source);
3466
3467 DEBUG(D_expand)
3468   debug_printf_indent("prvs: hash source is '%s'\n", hash_source->s);
3469
3470 memset(innerkey, 0x36, 64);
3471 memset(outerkey, 0x5c, 64);
3472
3473 for (i = 0; i < Ustrlen(key); i++)
3474   {
3475   innerkey[i] ^= key[i];
3476   outerkey[i] ^= key[i];
3477   }
3478
3479 chash_start(HMAC_SHA1, &h);
3480 chash_mid(HMAC_SHA1, &h, innerkey);
3481 chash_end(HMAC_SHA1, &h, hash_source->s, hash_source->ptr, innerhash);
3482
3483 chash_start(HMAC_SHA1, &h);
3484 chash_mid(HMAC_SHA1, &h, outerkey);
3485 chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash);
3486
3487 p = finalhash_hex;
3488 for (i = 0; i < 3; i++)
3489   {
3490   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3491   *p++ = hex_digits[finalhash[i] & 0x0f];
3492   }
3493 *p = '\0';
3494
3495 return finalhash_hex;
3496 }
3497
3498
3499
3500
3501 /*************************************************
3502 *        Join a file onto the output string      *
3503 *************************************************/
3504
3505 /* This is used for readfile/readsock and after a run expansion.
3506 It joins the contents of a file onto the output string, globally replacing
3507 newlines with a given string (optionally).
3508
3509 Arguments:
3510   f            the FILE
3511   yield        pointer to the expandable string struct
3512   eol          newline replacement string, or NULL
3513
3514 Returns:       new pointer for expandable string, terminated if non-null
3515 */
3516
3517 static gstring *
3518 cat_file(FILE *f, gstring *yield, uschar *eol)
3519 {
3520 uschar buffer[1024];
3521
3522 while (Ufgets(buffer, sizeof(buffer), f))
3523   {
3524   int len = Ustrlen(buffer);
3525   if (eol && buffer[len-1] == '\n') len--;
3526   yield = string_catn(yield, buffer, len);
3527   if (eol && buffer[len])
3528     yield = string_cat(yield, eol);
3529   }
3530
3531 (void) string_from_gstring(yield);
3532 return yield;
3533 }
3534
3535
3536
3537
3538 /*************************************************
3539 *          Evaluate numeric expression           *
3540 *************************************************/
3541
3542 /* This is a set of mutually recursive functions that evaluate an arithmetic
3543 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3544 these functions that is called from elsewhere is eval_expr, whose interface is:
3545
3546 Arguments:
3547   sptr        pointer to the pointer to the string - gets updated
3548   decimal     TRUE if numbers are to be assumed decimal
3549   error       pointer to where to put an error message - must be NULL on input
3550   endket      TRUE if ')' must terminate - FALSE for external call
3551
3552 Returns:      on success: the value of the expression, with *error still NULL
3553               on failure: an undefined value, with *error = a message
3554 */
3555
3556 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3557
3558
3559 static int_eximarith_t
3560 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3561 {
3562 uschar *s = *sptr;
3563 int_eximarith_t x = eval_op_or(&s, decimal, error);
3564 if (*error == NULL)
3565   {
3566   if (endket)
3567     {
3568     if (*s != ')')
3569       *error = US"expecting closing parenthesis";
3570     else
3571       while (isspace(*(++s)));
3572     }
3573   else if (*s != 0) *error = US"expecting operator";
3574   }
3575 *sptr = s;
3576 return x;
3577 }
3578
3579
3580 static int_eximarith_t
3581 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3582 {
3583 register int c;
3584 int_eximarith_t n;
3585 uschar *s = *sptr;
3586 while (isspace(*s)) s++;
3587 c = *s;
3588 if (isdigit(c))
3589   {
3590   int count;
3591   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3592   s += count;
3593   switch (tolower(*s))
3594     {
3595     default: break;
3596     case 'k': n *= 1024; s++; break;
3597     case 'm': n *= 1024*1024; s++; break;
3598     case 'g': n *= 1024*1024*1024; s++; break;
3599     }
3600   while (isspace (*s)) s++;
3601   }
3602 else if (c == '(')
3603   {
3604   s++;
3605   n = eval_expr(&s, decimal, error, 1);
3606   }
3607 else
3608   {
3609   *error = US"expecting number or opening parenthesis";
3610   n = 0;
3611   }
3612 *sptr = s;
3613 return n;
3614 }
3615
3616
3617 static int_eximarith_t
3618 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3619 {
3620 uschar *s = *sptr;
3621 int_eximarith_t x;
3622 while (isspace(*s)) s++;
3623 if (*s == '+' || *s == '-' || *s == '~')
3624   {
3625   int op = *s++;
3626   x = eval_op_unary(&s, decimal, error);
3627   if (op == '-') x = -x;
3628     else if (op == '~') x = ~x;
3629   }
3630 else
3631   {
3632   x = eval_number(&s, decimal, error);
3633   }
3634 *sptr = s;
3635 return x;
3636 }
3637
3638
3639 static int_eximarith_t
3640 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3641 {
3642 uschar *s = *sptr;
3643 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3644 if (*error == NULL)
3645   {
3646   while (*s == '*' || *s == '/' || *s == '%')
3647     {
3648     int op = *s++;
3649     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3650     if (*error != NULL) break;
3651     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3652      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3653      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3654      * -N*M is INT_MIN will yield INT_MIN.
3655      * Since we don't support floating point, this is somewhat simpler.
3656      * Ideally, we'd return an error, but since we overflow for all other
3657      * arithmetic, consistency suggests otherwise, but what's the correct value
3658      * to use?  There is none.
3659      * The C standard guarantees overflow for unsigned arithmetic but signed
3660      * overflow invokes undefined behaviour; in practice, this is overflow
3661      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3662      * that long/longlong larger than int are available, or we could just work
3663      * with larger types.  We should consider whether to guarantee 32bit eval
3664      * and 64-bit working variables, with errors returned.  For now ...
3665      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3666      * can just let the other invalid results occur otherwise, as they have
3667      * until now.  For this one case, we can coerce.
3668      */
3669     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3670       {
3671       DEBUG(D_expand)
3672         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3673             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3674       x = EXIM_ARITH_MAX;
3675       continue;
3676       }
3677     if (op == '*')
3678       x *= y;
3679     else
3680       {
3681       if (y == 0)
3682         {
3683         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3684         x = 0;
3685         break;
3686         }
3687       if (op == '/')
3688         x /= y;
3689       else
3690         x %= y;
3691       }
3692     }
3693   }
3694 *sptr = s;
3695 return x;
3696 }
3697
3698
3699 static int_eximarith_t
3700 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3701 {
3702 uschar *s = *sptr;
3703 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3704 if (!*error)
3705   {
3706   while (*s == '+' || *s == '-')
3707     {
3708     int op = *s++;
3709     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3710     if (*error) break;
3711     if (  (x >=   EXIM_ARITH_MAX/2  && x >=   EXIM_ARITH_MAX/2)
3712        || (x <= -(EXIM_ARITH_MAX/2) && y <= -(EXIM_ARITH_MAX/2)))
3713       {                 /* over-conservative check */
3714       *error = op == '+'
3715         ? US"overflow in sum" : US"overflow in difference";
3716       break;
3717       }
3718     if (op == '+') x += y; else x -= y;
3719     }
3720   }
3721 *sptr = s;
3722 return x;
3723 }
3724
3725
3726 static int_eximarith_t
3727 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3728 {
3729 uschar *s = *sptr;
3730 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3731 if (*error == NULL)
3732   {
3733   while ((*s == '<' || *s == '>') && s[1] == s[0])
3734     {
3735     int_eximarith_t y;
3736     int op = *s++;
3737     s++;
3738     y = eval_op_sum(&s, decimal, error);
3739     if (*error != NULL) break;
3740     if (op == '<') x <<= y; else x >>= y;
3741     }
3742   }
3743 *sptr = s;
3744 return x;
3745 }
3746
3747
3748 static int_eximarith_t
3749 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3750 {
3751 uschar *s = *sptr;
3752 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3753 if (*error == NULL)
3754   {
3755   while (*s == '&')
3756     {
3757     int_eximarith_t y;
3758     s++;
3759     y = eval_op_shift(&s, decimal, error);
3760     if (*error != NULL) break;
3761     x &= y;
3762     }
3763   }
3764 *sptr = s;
3765 return x;
3766 }
3767
3768
3769 static int_eximarith_t
3770 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3771 {
3772 uschar *s = *sptr;
3773 int_eximarith_t x = eval_op_and(&s, decimal, error);
3774 if (*error == NULL)
3775   {
3776   while (*s == '^')
3777     {
3778     int_eximarith_t y;
3779     s++;
3780     y = eval_op_and(&s, decimal, error);
3781     if (*error != NULL) break;
3782     x ^= y;
3783     }
3784   }
3785 *sptr = s;
3786 return x;
3787 }
3788
3789
3790 static int_eximarith_t
3791 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3792 {
3793 uschar *s = *sptr;
3794 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3795 if (*error == NULL)
3796   {
3797   while (*s == '|')
3798     {
3799     int_eximarith_t y;
3800     s++;
3801     y = eval_op_xor(&s, decimal, error);
3802     if (*error != NULL) break;
3803     x |= y;
3804     }
3805   }
3806 *sptr = s;
3807 return x;
3808 }
3809
3810
3811
3812 /*************************************************
3813 *                 Expand string                  *
3814 *************************************************/
3815
3816 /* Returns either an unchanged string, or the expanded string in stacking pool
3817 store. Interpreted sequences are:
3818
3819    \...                    normal escaping rules
3820    $name                   substitutes the variable
3821    ${name}                 ditto
3822    ${op:string}            operates on the expanded string value
3823    ${item{arg1}{arg2}...}  expands the args and then does the business
3824                              some literal args are not enclosed in {}
3825
3826 There are now far too many operators and item types to make it worth listing
3827 them here in detail any more.
3828
3829 We use an internal routine recursively to handle embedded substrings. The
3830 external function follows. The yield is NULL if the expansion failed, and there
3831 are two cases: if something collapsed syntactically, or if "fail" was given
3832 as the action on a lookup failure. These can be distinguished by looking at the
3833 variable expand_string_forcedfail, which is TRUE in the latter case.
3834
3835 The skipping flag is set true when expanding a substring that isn't actually
3836 going to be used (after "if" or "lookup") and it prevents lookups from
3837 happening lower down.
3838
3839 Store usage: At start, a store block of the length of the input plus 64
3840 is obtained. This is expanded as necessary by string_cat(), which might have to
3841 get a new block, or might be able to expand the original. At the end of the
3842 function we can release any store above that portion of the yield block that
3843 was actually used. In many cases this will be optimal.
3844
3845 However: if the first item in the expansion is a variable name or header name,
3846 we reset the store before processing it; if the result is in fresh store, we
3847 use that without copying. This is helpful for expanding strings like
3848 $message_headers which can get very long.
3849
3850 There's a problem if a ${dlfunc item has side-effects that cause allocation,
3851 since resetting the store at the end of the expansion will free store that was
3852 allocated by the plugin code as well as the slop after the expanded string. So
3853 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
3854 and, given the acl condition, ${if }. This is an unfortunate consequence of
3855 string expansion becoming too powerful.
3856
3857 Arguments:
3858   string         the string to be expanded
3859   ket_ends       true if expansion is to stop at }
3860   left           if not NULL, a pointer to the first character after the
3861                  expansion is placed here (typically used with ket_ends)
3862   skipping       TRUE for recursive calls when the value isn't actually going
3863                  to be used (to allow for optimisation)
3864   honour_dollar  TRUE if $ is to be expanded,
3865                  FALSE if it's just another character
3866   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
3867                  the store.
3868
3869 Returns:         NULL if expansion fails:
3870                    expand_string_forcedfail is set TRUE if failure was forced
3871                    expand_string_message contains a textual error message
3872                  a pointer to the expanded string on success
3873 */
3874
3875 static uschar *
3876 expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left,
3877   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
3878 {
3879 gstring * yield = string_get(Ustrlen(string) + 64);
3880 int item_type;
3881 const uschar *s = string;
3882 uschar *save_expand_nstring[EXPAND_MAXN+1];
3883 int save_expand_nlength[EXPAND_MAXN+1];
3884 BOOL resetok = TRUE;
3885
3886 expand_level++;
3887 DEBUG(D_expand)
3888   debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n",
3889     skipping
3890     ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning"
3891     : "considering",
3892     string);
3893
3894 expand_string_forcedfail = FALSE;
3895 expand_string_message = US"";
3896
3897 while (*s != 0)
3898   {
3899   uschar *value;
3900   uschar name[256];
3901
3902   /* \ escapes the next character, which must exist, or else
3903   the expansion fails. There's a special escape, \N, which causes
3904   copying of the subject verbatim up to the next \N. Otherwise,
3905   the escapes are the standard set. */
3906
3907   if (*s == '\\')
3908     {
3909     if (s[1] == 0)
3910       {
3911       expand_string_message = US"\\ at end of string";
3912       goto EXPAND_FAILED;
3913       }
3914
3915     if (s[1] == 'N')
3916       {
3917       const uschar * t = s + 2;
3918       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
3919       yield = string_catn(yield, t, s - t);
3920       if (*s != 0) s += 2;
3921       }
3922
3923     else
3924       {
3925       uschar ch[1];
3926       ch[0] = string_interpret_escape(&s);
3927       s++;
3928       yield = string_catn(yield, ch, 1);
3929       }
3930
3931     continue;
3932     }
3933
3934   /*{*/
3935   /* Anything other than $ is just copied verbatim, unless we are
3936   looking for a terminating } character. */
3937
3938   /*{*/
3939   if (ket_ends && *s == '}') break;
3940
3941   if (*s != '$' || !honour_dollar)
3942     {
3943     yield = string_catn(yield, s++, 1);
3944     continue;
3945     }
3946
3947   /* No { after the $ - must be a plain name or a number for string
3948   match variable. There has to be a fudge for variables that are the
3949   names of header fields preceded by "$header_" because header field
3950   names can contain any printing characters except space and colon.
3951   For those that don't like typing this much, "$h_" is a synonym for
3952   "$header_". A non-existent header yields a NULL value; nothing is
3953   inserted. */  /*}*/
3954
3955   if (isalpha((*(++s))))
3956     {
3957     int len;
3958     int newsize = 0;
3959     gstring * g = NULL;
3960
3961     s = read_name(name, sizeof(name), s, US"_");
3962
3963     /* If this is the first thing to be expanded, release the pre-allocated
3964     buffer. */
3965
3966     if (!yield)
3967       g = store_get(sizeof(gstring));
3968     else if (yield->ptr == 0)
3969       {
3970       if (resetok) store_reset(yield);
3971       yield = NULL;
3972       g = store_get(sizeof(gstring));   /* alloc _before_ calling find_variable() */
3973       }
3974
3975     /* Header */
3976
3977     if (Ustrncmp(name, "h_", 2) == 0 ||
3978         Ustrncmp(name, "rh_", 3) == 0 ||
3979         Ustrncmp(name, "bh_", 3) == 0 ||
3980         Ustrncmp(name, "header_", 7) == 0 ||
3981         Ustrncmp(name, "rheader_", 8) == 0 ||
3982         Ustrncmp(name, "bheader_", 8) == 0)
3983       {
3984       BOOL want_raw = (name[0] == 'r')? TRUE : FALSE;
3985       uschar *charset = (name[0] == 'b')? NULL : headers_charset;
3986       s = read_header_name(name, sizeof(name), s);
3987       value = find_header(name, FALSE, &newsize, want_raw, charset);
3988
3989       /* If we didn't find the header, and the header contains a closing brace
3990       character, this may be a user error where the terminating colon
3991       has been omitted. Set a flag to adjust the error message in this case.
3992       But there is no error here - nothing gets inserted. */
3993
3994       if (!value)
3995         {
3996         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
3997         continue;
3998         }
3999       }
4000
4001     /* Variable */
4002
4003     else if (!(value = find_variable(name, FALSE, skipping, &newsize)))
4004       {
4005       expand_string_message =
4006         string_sprintf("unknown variable name \"%s\"", name);
4007         check_variable_error_message(name);
4008       goto EXPAND_FAILED;
4009       }
4010
4011     /* If the data is known to be in a new buffer, newsize will be set to the
4012     size of that buffer. If this is the first thing in an expansion string,
4013     yield will be NULL; just point it at the new store instead of copying. Many
4014     expansion strings contain just one reference, so this is a useful
4015     optimization, especially for humungous headers.  We need to use a gstring
4016     structure that is not allocated after that new-buffer, else a later store
4017     reset in the middle of the buffer will make it inaccessible. */
4018
4019     len = Ustrlen(value);
4020     if (!yield && newsize != 0)
4021       {
4022       yield = g;
4023       yield->size = newsize;
4024       yield->ptr = len;
4025       yield->s = value;
4026       }
4027     else
4028       yield = string_catn(yield, value, len);
4029
4030     continue;
4031     }
4032
4033   if (isdigit(*s))
4034     {
4035     int n;
4036     s = read_cnumber(&n, s);
4037     if (n >= 0 && n <= expand_nmax)
4038       yield = string_catn(yield, expand_nstring[n], expand_nlength[n]);
4039     continue;
4040     }
4041
4042   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
4043
4044   if (*s != '{')                                                        /*}*/
4045     {
4046     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
4047     goto EXPAND_FAILED;
4048     }
4049
4050   /* After { there can be various things, but they all start with
4051   an initial word, except for a number for a string match variable. */
4052
4053   if (isdigit((*(++s))))
4054     {
4055     int n;
4056     s = read_cnumber(&n, s);            /*{*/
4057     if (*s++ != '}')
4058       {                                 /*{*/
4059       expand_string_message = US"} expected after number";
4060       goto EXPAND_FAILED;
4061       }
4062     if (n >= 0 && n <= expand_nmax)
4063       yield = string_catn(yield, expand_nstring[n], expand_nlength[n]);
4064     continue;
4065     }
4066
4067   if (!isalpha(*s))
4068     {
4069     expand_string_message = US"letter or digit expected after ${";      /*}*/
4070     goto EXPAND_FAILED;
4071     }
4072
4073   /* Allow "-" in names to cater for substrings with negative
4074   arguments. Since we are checking for known names after { this is
4075   OK. */
4076
4077   s = read_name(name, sizeof(name), s, US"_-");
4078   item_type = chop_match(name, item_table, nelem(item_table));
4079
4080   switch(item_type)
4081     {
4082     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
4083     If the ACL returns accept or reject we return content set by "message ="
4084     There is currently no limit on recursion; this would have us call
4085     acl_check_internal() directly and get a current level from somewhere.
4086     See also the acl expansion condition ECOND_ACL and the traditional
4087     acl modifier ACLC_ACL.
4088     Assume that the function has side-effects on the store that must be preserved.
4089     */
4090
4091     case EITEM_ACL:
4092       /* ${acl {name} {arg1}{arg2}...} */
4093       {
4094       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
4095       uschar *user_msg;
4096
4097       switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, US"acl",
4098                       &resetok))
4099         {
4100         case 1: goto EXPAND_FAILED_CURLY;
4101         case 2:
4102         case 3: goto EXPAND_FAILED;
4103         }
4104       if (skipping) continue;
4105
4106       resetok = FALSE;
4107       switch(eval_acl(sub, nelem(sub), &user_msg))
4108         {
4109         case OK:
4110         case FAIL:
4111           DEBUG(D_expand)
4112             debug_printf_indent("acl expansion yield: %s\n", user_msg);
4113           if (user_msg)
4114             yield = string_cat(yield, user_msg);
4115           continue;
4116
4117         case DEFER:
4118           expand_string_forcedfail = TRUE;
4119           /*FALLTHROUGH*/
4120         default:
4121           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
4122           goto EXPAND_FAILED;
4123         }
4124       }
4125
4126     case EITEM_AUTHRESULTS:
4127       /* ${authresults {mysystemname}} */
4128       {
4129       uschar *sub_arg[1];
4130
4131       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4132                       &resetok))
4133         {
4134         case 1: goto EXPAND_FAILED_CURLY;
4135         case 2:
4136         case 3: goto EXPAND_FAILED;
4137         }
4138
4139       yield = string_append(yield, 3,
4140                         US"Authentication-Results: ", sub_arg[0], US"; none");
4141       yield->ptr -= 6;
4142
4143       yield = authres_iprev(yield);
4144       yield = authres_smtpauth(yield);
4145 #ifdef SUPPORT_SPF
4146       yield = authres_spf(yield);
4147 #endif
4148 #ifndef DISABLE_DKIM
4149       yield = authres_dkim(yield);
4150 #endif
4151 #ifdef EXPERIMENTAL_DMARC
4152       yield = authres_dmarc(yield);
4153 #endif
4154 #ifdef EXPERIMENTAL_ARC
4155       yield = authres_arc(yield);
4156 #endif
4157       continue;
4158       }
4159
4160     /* Handle conditionals - preserve the values of the numerical expansion
4161     variables in case they get changed by a regular expression match in the
4162     condition. If not, they retain their external settings. At the end
4163     of this "if" section, they get restored to their previous values. */
4164
4165     case EITEM_IF:
4166       {
4167       BOOL cond = FALSE;
4168       const uschar *next_s;
4169       int save_expand_nmax =
4170         save_expand_strings(save_expand_nstring, save_expand_nlength);
4171
4172       while (isspace(*s)) s++;
4173       next_s = eval_condition(s, &resetok, skipping ? NULL : &cond);
4174       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
4175
4176       DEBUG(D_expand)
4177         {
4178         debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4179           "condition: %.*s\n",
4180           (int)(next_s - s), s);
4181         debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4182           UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
4183           "result: %s\n",
4184           cond ? "true" : "false");
4185         }
4186
4187       s = next_s;
4188
4189       /* The handling of "yes" and "no" result strings is now in a separate
4190       function that is also used by ${lookup} and ${extract} and ${run}. */
4191
4192       switch(process_yesno(
4193                skipping,                     /* were previously skipping */
4194                cond,                         /* success/failure indicator */
4195                lookup_value,                 /* value to reset for string2 */
4196                &s,                           /* input pointer */
4197                &yield,                       /* output pointer */
4198                US"if",                       /* condition type */
4199                &resetok))
4200         {
4201         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4202         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4203         }
4204
4205       /* Restore external setting of expansion variables for continuation
4206       at this level. */
4207
4208       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4209         save_expand_nlength);
4210       continue;
4211       }
4212
4213 #ifdef SUPPORT_I18N
4214     case EITEM_IMAPFOLDER:
4215       {                         /* ${imapfolder {name}{sep]{specials}} */
4216       uschar *sub_arg[3];
4217       uschar *encoded;
4218
4219       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4220                       &resetok))
4221         {
4222         case 1: goto EXPAND_FAILED_CURLY;
4223         case 2:
4224         case 3: goto EXPAND_FAILED;
4225         }
4226
4227       if (sub_arg[1] == NULL)           /* One argument */
4228         {
4229         sub_arg[1] = US"/";             /* default separator */
4230         sub_arg[2] = NULL;
4231         }
4232       else if (Ustrlen(sub_arg[1]) != 1)
4233         {
4234         expand_string_message =
4235           string_sprintf(
4236                 "IMAP folder separator must be one character, found \"%s\"",
4237                 sub_arg[1]);
4238         goto EXPAND_FAILED;
4239         }
4240
4241       if (!skipping)
4242         {
4243         if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
4244                             sub_arg[1][0], sub_arg[2], &expand_string_message)))
4245           goto EXPAND_FAILED;
4246         yield = string_cat(yield, encoded);
4247         }
4248       continue;
4249       }
4250 #endif
4251
4252     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4253     expanding an internal string that isn't actually going to be used. All we
4254     need to do is check the syntax, so don't do a lookup at all. Preserve the
4255     values of the numerical expansion variables in case they get changed by a
4256     partial lookup. If not, they retain their external settings. At the end
4257     of this "lookup" section, they get restored to their previous values. */
4258
4259     case EITEM_LOOKUP:
4260       {
4261       int stype, partial, affixlen, starflags;
4262       int expand_setup = 0;
4263       int nameptr = 0;
4264       uschar *key, *filename;
4265       const uschar *affix;
4266       uschar *save_lookup_value = lookup_value;
4267       int save_expand_nmax =
4268         save_expand_strings(save_expand_nstring, save_expand_nlength);
4269
4270       if ((expand_forbid & RDO_LOOKUP) != 0)
4271         {
4272         expand_string_message = US"lookup expansions are not permitted";
4273         goto EXPAND_FAILED;
4274         }
4275
4276       /* Get the key we are to look up for single-key+file style lookups.
4277       Otherwise set the key NULL pro-tem. */
4278
4279       while (isspace(*s)) s++;
4280       if (*s == '{')                                    /*}*/
4281         {
4282         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4283         if (!key) goto EXPAND_FAILED;                   /*{{*/
4284         if (*s++ != '}')
4285           {
4286           expand_string_message = US"missing '}' after lookup key";
4287           goto EXPAND_FAILED_CURLY;
4288           }
4289         while (isspace(*s)) s++;
4290         }
4291       else key = NULL;
4292
4293       /* Find out the type of database */
4294
4295       if (!isalpha(*s))
4296         {
4297         expand_string_message = US"missing lookup type";
4298         goto EXPAND_FAILED;
4299         }
4300
4301       /* The type is a string that may contain special characters of various
4302       kinds. Allow everything except space or { to appear; the actual content
4303       is checked by search_findtype_partial. */         /*}*/
4304
4305       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4306         {
4307         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4308         s++;
4309         }
4310       name[nameptr] = 0;
4311       while (isspace(*s)) s++;
4312
4313       /* Now check for the individual search type and any partial or default
4314       options. Only those types that are actually in the binary are valid. */
4315
4316       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4317         &starflags);
4318       if (stype < 0)
4319         {
4320         expand_string_message = search_error_message;
4321         goto EXPAND_FAILED;
4322         }
4323
4324       /* Check that a key was provided for those lookup types that need it,
4325       and was not supplied for those that use the query style. */
4326
4327       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4328         {
4329         if (key == NULL)
4330           {
4331           expand_string_message = string_sprintf("missing {key} for single-"
4332             "key \"%s\" lookup", name);
4333           goto EXPAND_FAILED;
4334           }
4335         }
4336       else
4337         {
4338         if (key != NULL)
4339           {
4340           expand_string_message = string_sprintf("a single key was given for "
4341             "lookup type \"%s\", which is not a single-key lookup type", name);
4342           goto EXPAND_FAILED;
4343           }
4344         }
4345
4346       /* Get the next string in brackets and expand it. It is the file name for
4347       single-key+file lookups, and the whole query otherwise. In the case of
4348       queries that also require a file name (e.g. sqlite), the file name comes
4349       first. */
4350
4351       if (*s != '{')
4352         {
4353         expand_string_message = US"missing '{' for lookup file-or-query arg";
4354         goto EXPAND_FAILED_CURLY;
4355         }
4356       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4357       if (filename == NULL) goto EXPAND_FAILED;
4358       if (*s++ != '}')
4359         {
4360         expand_string_message = US"missing '}' closing lookup file-or-query arg";
4361         goto EXPAND_FAILED_CURLY;
4362         }
4363       while (isspace(*s)) s++;
4364
4365       /* If this isn't a single-key+file lookup, re-arrange the variables
4366       to be appropriate for the search_ functions. For query-style lookups,
4367       there is just a "key", and no file name. For the special query-style +
4368       file types, the query (i.e. "key") starts with a file name. */
4369
4370       if (!key)
4371         {
4372         while (isspace(*filename)) filename++;
4373         key = filename;
4374
4375         if (mac_islookup(stype, lookup_querystyle))
4376           filename = NULL;
4377         else
4378           {
4379           if (*filename != '/')
4380             {
4381             expand_string_message = string_sprintf(
4382               "absolute file name expected for \"%s\" lookup", name);
4383             goto EXPAND_FAILED;
4384             }
4385           while (*key != 0 && !isspace(*key)) key++;
4386           if (*key != 0) *key++ = 0;
4387           }
4388         }
4389
4390       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4391       the entry was not found. Note that there is no search_close() function.
4392       Files are left open in case of re-use. At suitable places in higher logic,
4393       search_tidyup() is called to tidy all open files. This can save opening
4394       the same file several times. However, files may also get closed when
4395       others are opened, if too many are open at once. The rule is that a
4396       handle should not be used after a second search_open().
4397
4398       Request that a partial search sets up $1 and maybe $2 by passing
4399       expand_setup containing zero. If its value changes, reset expand_nmax,
4400       since new variables will have been set. Note that at the end of this
4401       "lookup" section, the old numeric variables are restored. */
4402
4403       if (skipping)
4404         lookup_value = NULL;
4405       else
4406         {
4407         void *handle = search_open(filename, stype, 0, NULL, NULL);
4408         if (handle == NULL)
4409           {
4410           expand_string_message = search_error_message;
4411           goto EXPAND_FAILED;
4412           }
4413         lookup_value = search_find(handle, filename, key, partial, affix,
4414           affixlen, starflags, &expand_setup);
4415         if (search_find_defer)
4416           {
4417           expand_string_message =
4418             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4419               string_printing2(key, FALSE), search_error_message);
4420           goto EXPAND_FAILED;
4421           }
4422         if (expand_setup > 0) expand_nmax = expand_setup;
4423         }
4424
4425       /* The handling of "yes" and "no" result strings is now in a separate
4426       function that is also used by ${if} and ${extract}. */
4427
4428       switch(process_yesno(
4429                skipping,                     /* were previously skipping */
4430                lookup_value != NULL,         /* success/failure indicator */
4431                save_lookup_value,            /* value to reset for string2 */
4432                &s,                           /* input pointer */
4433                &yield,                       /* output pointer */
4434                US"lookup",                   /* condition type */
4435                &resetok))
4436         {
4437         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4438         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4439         }
4440
4441       /* Restore external setting of expansion variables for carrying on
4442       at this level, and continue. */
4443
4444       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4445         save_expand_nlength);
4446       continue;
4447       }
4448
4449     /* If Perl support is configured, handle calling embedded perl subroutines,
4450     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4451     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4452     arguments (defined below). */
4453
4454     #define EXIM_PERL_MAX_ARGS 8
4455
4456     case EITEM_PERL:
4457     #ifndef EXIM_PERL
4458     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4459       "is not included in this binary";
4460     goto EXPAND_FAILED;
4461
4462     #else   /* EXIM_PERL */
4463       {
4464       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4465       gstring *new_yield;
4466
4467       if ((expand_forbid & RDO_PERL) != 0)
4468         {
4469         expand_string_message = US"Perl calls are not permitted";
4470         goto EXPAND_FAILED;
4471         }
4472
4473       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4474            US"perl", &resetok))
4475         {
4476         case 1: goto EXPAND_FAILED_CURLY;
4477         case 2:
4478         case 3: goto EXPAND_FAILED;
4479         }
4480
4481       /* If skipping, we don't actually do anything */
4482
4483       if (skipping) continue;
4484
4485       /* Start the interpreter if necessary */
4486
4487       if (!opt_perl_started)
4488         {
4489         uschar *initerror;
4490         if (opt_perl_startup == NULL)
4491           {
4492           expand_string_message = US"A setting of perl_startup is needed when "
4493             "using the Perl interpreter";
4494           goto EXPAND_FAILED;
4495           }
4496         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4497         initerror = init_perl(opt_perl_startup);
4498         if (initerror != NULL)
4499           {
4500           expand_string_message =
4501             string_sprintf("error in perl_startup code: %s\n", initerror);
4502           goto EXPAND_FAILED;
4503           }
4504         opt_perl_started = TRUE;
4505         }
4506
4507       /* Call the function */
4508
4509       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4510       new_yield = call_perl_cat(yield, &expand_string_message,
4511         sub_arg[0], sub_arg + 1);
4512
4513       /* NULL yield indicates failure; if the message pointer has been set to
4514       NULL, the yield was undef, indicating a forced failure. Otherwise the
4515       message will indicate some kind of Perl error. */
4516
4517       if (new_yield == NULL)
4518         {
4519         if (expand_string_message == NULL)
4520           {
4521           expand_string_message =
4522             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4523               "failure", sub_arg[0]);
4524           expand_string_forcedfail = TRUE;
4525           }
4526         goto EXPAND_FAILED;
4527         }
4528
4529       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4530       set during a callback from Perl. */
4531
4532       expand_string_forcedfail = FALSE;
4533       yield = new_yield;
4534       continue;
4535       }
4536     #endif /* EXIM_PERL */
4537
4538     /* Transform email address to "prvs" scheme to use
4539        as BATV-signed return path */
4540
4541     case EITEM_PRVS:
4542       {
4543       uschar *sub_arg[3];
4544       uschar *p,*domain;
4545
4546       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4547         {
4548         case 1: goto EXPAND_FAILED_CURLY;
4549         case 2:
4550         case 3: goto EXPAND_FAILED;
4551         }
4552
4553       /* If skipping, we don't actually do anything */
4554       if (skipping) continue;
4555
4556       /* sub_arg[0] is the address */
4557       if (  !(domain = Ustrrchr(sub_arg[0],'@'))
4558          || domain == sub_arg[0] || Ustrlen(domain) == 1)
4559         {
4560         expand_string_message = US"prvs first argument must be a qualified email address";
4561         goto EXPAND_FAILED;
4562         }
4563
4564       /* Calculate the hash. The third argument must be a single-digit
4565       key number, or unset. */
4566
4567       if (  sub_arg[2]
4568          && (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4569         {
4570         expand_string_message = US"prvs third argument must be a single digit";
4571         goto EXPAND_FAILED;
4572         }
4573
4574       p = prvs_hmac_sha1(sub_arg[0], sub_arg[1], sub_arg[2], prvs_daystamp(7));
4575       if (!p)
4576         {
4577         expand_string_message = US"prvs hmac-sha1 conversion failed";
4578         goto EXPAND_FAILED;
4579         }
4580
4581       /* Now separate the domain from the local part */
4582       *domain++ = '\0';
4583
4584       yield = string_catn(yield, US"prvs=", 5);
4585       yield = string_catn(yield, sub_arg[2] ? sub_arg[2] : US"0", 1);
4586       yield = string_catn(yield, prvs_daystamp(7), 3);
4587       yield = string_catn(yield, p, 6);
4588       yield = string_catn(yield, US"=", 1);
4589       yield = string_cat (yield, sub_arg[0]);
4590       yield = string_catn(yield, US"@", 1);
4591       yield = string_cat (yield, domain);
4592
4593       continue;
4594       }
4595
4596     /* Check a prvs-encoded address for validity */
4597
4598     case EITEM_PRVSCHECK:
4599       {
4600       uschar *sub_arg[3];
4601       gstring * g;
4602       const pcre *re;
4603       uschar *p;
4604
4605       /* TF: Ugliness: We want to expand parameter 1 first, then set
4606          up expansion variables that are used in the expansion of
4607          parameter 2. So we clone the string for the first
4608          expansion, where we only expand parameter 1.
4609
4610          PH: Actually, that isn't necessary. The read_subs() function is
4611          designed to work this way for the ${if and ${lookup expansions. I've
4612          tidied the code.
4613       */
4614
4615       /* Reset expansion variables */
4616       prvscheck_result = NULL;
4617       prvscheck_address = NULL;
4618       prvscheck_keynum = NULL;
4619
4620       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4621         {
4622         case 1: goto EXPAND_FAILED_CURLY;
4623         case 2:
4624         case 3: goto EXPAND_FAILED;
4625         }
4626
4627       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4628                               TRUE,FALSE);
4629
4630       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4631         {
4632         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4633         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4634         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4635         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4636         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4637
4638         DEBUG(D_expand) debug_printf_indent("prvscheck localpart: %s\n", local_part);
4639         DEBUG(D_expand) debug_printf_indent("prvscheck key number: %s\n", key_num);
4640         DEBUG(D_expand) debug_printf_indent("prvscheck daystamp: %s\n", daystamp);
4641         DEBUG(D_expand) debug_printf_indent("prvscheck hash: %s\n", hash);
4642         DEBUG(D_expand) debug_printf_indent("prvscheck domain: %s\n", domain);
4643
4644         /* Set up expansion variables */
4645         g = string_cat (NULL, local_part);
4646         g = string_catn(g, US"@", 1);
4647         g = string_cat (g, domain);
4648         prvscheck_address = string_from_gstring(g);
4649         prvscheck_keynum = string_copy(key_num);
4650
4651         /* Now expand the second argument */
4652         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4653           {
4654           case 1: goto EXPAND_FAILED_CURLY;
4655           case 2:
4656           case 3: goto EXPAND_FAILED;
4657           }
4658
4659         /* Now we have the key and can check the address. */
4660
4661         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4662           daystamp);
4663
4664         if (!p)
4665           {
4666           expand_string_message = US"hmac-sha1 conversion failed";
4667           goto EXPAND_FAILED;
4668           }
4669
4670         DEBUG(D_expand) debug_printf_indent("prvscheck: received hash is %s\n", hash);
4671         DEBUG(D_expand) debug_printf_indent("prvscheck:      own hash is %s\n", p);
4672
4673         if (Ustrcmp(p,hash) == 0)
4674           {
4675           /* Success, valid BATV address. Now check the expiry date. */
4676           uschar *now = prvs_daystamp(0);
4677           unsigned int inow = 0,iexpire = 1;
4678
4679           (void)sscanf(CS now,"%u",&inow);
4680           (void)sscanf(CS daystamp,"%u",&iexpire);
4681
4682           /* When "iexpire" is < 7, a "flip" has occured.
4683              Adjust "inow" accordingly. */
4684           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4685
4686           if (iexpire >= inow)
4687             {
4688             prvscheck_result = US"1";
4689             DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n");
4690             }
4691           else
4692             {
4693             prvscheck_result = NULL;
4694             DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n");
4695             }
4696           }
4697         else
4698           {
4699           prvscheck_result = NULL;
4700           DEBUG(D_expand) debug_printf_indent("prvscheck: hash failure, $pvrs_result unset\n");
4701           }
4702
4703         /* Now expand the final argument. We leave this till now so that
4704         it can include $prvscheck_result. */
4705
4706         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4707           {
4708           case 1: goto EXPAND_FAILED_CURLY;
4709           case 2:
4710           case 3: goto EXPAND_FAILED;
4711           }
4712
4713         yield = string_cat(yield,
4714           !sub_arg[0] || !*sub_arg[0] ? prvscheck_address : sub_arg[0]);
4715
4716         /* Reset the "internal" variables afterwards, because they are in
4717         dynamic store that will be reclaimed if the expansion succeeded. */
4718
4719         prvscheck_address = NULL;
4720         prvscheck_keynum = NULL;
4721         }
4722       else
4723         /* Does not look like a prvs encoded address, return the empty string.
4724            We need to make sure all subs are expanded first, so as to skip over
4725            the entire item. */
4726
4727         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4728           {
4729           case 1: goto EXPAND_FAILED_CURLY;
4730           case 2:
4731           case 3: goto EXPAND_FAILED;
4732           }
4733
4734       continue;
4735       }
4736
4737     /* Handle "readfile" to insert an entire file */
4738
4739     case EITEM_READFILE:
4740       {
4741       FILE *f;
4742       uschar *sub_arg[2];
4743
4744       if ((expand_forbid & RDO_READFILE) != 0)
4745         {
4746         expand_string_message = US"file insertions are not permitted";
4747         goto EXPAND_FAILED;
4748         }
4749
4750       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4751         {
4752         case 1: goto EXPAND_FAILED_CURLY;
4753         case 2:
4754         case 3: goto EXPAND_FAILED;
4755         }
4756
4757       /* If skipping, we don't actually do anything */
4758
4759       if (skipping) continue;
4760
4761       /* Open the file and read it */
4762
4763       if (!(f = Ufopen(sub_arg[0], "rb")))
4764         {
4765         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4766         goto EXPAND_FAILED;
4767         }
4768
4769       yield = cat_file(f, yield, sub_arg[1]);
4770       (void)fclose(f);
4771       continue;
4772       }
4773
4774     /* Handle "readsocket" to insert data from a socket, either
4775     Inet or Unix domain */
4776
4777     case EITEM_READSOCK:
4778       {
4779       int fd;
4780       int timeout = 5;
4781       int save_ptr = yield->ptr;
4782       FILE *f;
4783       uschar *arg;
4784       uschar *sub_arg[4];
4785       BOOL do_shutdown = TRUE;
4786       blob reqstr;
4787
4788       if (expand_forbid & RDO_READSOCK)
4789         {
4790         expand_string_message = US"socket insertions are not permitted";
4791         goto EXPAND_FAILED;
4792         }
4793
4794       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4795       because there may be a string for expansion on failure. */
4796
4797       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4798         {
4799         case 1: goto EXPAND_FAILED_CURLY;
4800         case 2:                             /* Won't occur: no end check */
4801         case 3: goto EXPAND_FAILED;
4802         }
4803
4804       /* Grab the request string, if any */
4805
4806       reqstr.data = sub_arg[1];
4807       reqstr.len = Ustrlen(sub_arg[1]);
4808
4809       /* Sort out timeout, if given.  The second arg is a list with the first element
4810       being a time value.  Any more are options of form "name=value".  Currently the
4811       only option recognised is "shutdown". */
4812
4813       if (sub_arg[2])
4814         {
4815         const uschar * list = sub_arg[2];
4816         uschar * item;
4817         int sep = 0;
4818
4819         item = string_nextinlist(&list, &sep, NULL, 0);
4820         if ((timeout = readconf_readtime(item, 0, FALSE)) < 0)
4821           {
4822           expand_string_message = string_sprintf("bad time value %s", item);
4823           goto EXPAND_FAILED;
4824           }
4825
4826         while ((item = string_nextinlist(&list, &sep, NULL, 0)))
4827           if (Ustrncmp(item, US"shutdown=", 9) == 0)
4828             if (Ustrcmp(item + 9, US"no") == 0)
4829               do_shutdown = FALSE;
4830         }
4831       else sub_arg[3] = NULL;                     /* No eol if no timeout */
4832
4833       /* If skipping, we don't actually do anything. Otherwise, arrange to
4834       connect to either an IP or a Unix socket. */
4835
4836       if (!skipping)
4837         {
4838         /* Handle an IP (internet) domain */
4839
4840         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
4841           {
4842           int port;
4843           uschar * server_name = sub_arg[0] + 5;
4844           uschar * port_name = Ustrrchr(server_name, ':');
4845
4846           /* Sort out the port */
4847
4848           if (!port_name)
4849             {
4850             expand_string_message =
4851               string_sprintf("missing port for readsocket %s", sub_arg[0]);
4852             goto EXPAND_FAILED;
4853             }
4854           *port_name++ = 0;           /* Terminate server name */
4855
4856           if (isdigit(*port_name))
4857             {
4858             uschar *end;
4859             port = Ustrtol(port_name, &end, 0);
4860             if (end != port_name + Ustrlen(port_name))
4861               {
4862               expand_string_message =
4863                 string_sprintf("invalid port number %s", port_name);
4864               goto EXPAND_FAILED;
4865               }
4866             }
4867           else
4868             {
4869             struct servent *service_info = getservbyname(CS port_name, "tcp");
4870             if (!service_info)
4871               {
4872               expand_string_message = string_sprintf("unknown port \"%s\"",
4873                 port_name);
4874               goto EXPAND_FAILED;
4875               }
4876             port = ntohs(service_info->s_port);
4877             }
4878
4879           fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
4880                   timeout, NULL, &expand_string_message, &reqstr);
4881           callout_address = NULL;
4882           if (fd < 0)
4883               goto SOCK_FAIL;
4884           reqstr.len = 0;
4885           }
4886
4887         /* Handle a Unix domain socket */
4888
4889         else
4890           {
4891           struct sockaddr_un sockun;         /* don't call this "sun" ! */
4892           int rc;
4893
4894           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
4895             {
4896             expand_string_message = string_sprintf("failed to create socket: %s",
4897               strerror(errno));
4898             goto SOCK_FAIL;
4899             }
4900
4901           sockun.sun_family = AF_UNIX;
4902           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
4903             sub_arg[0]);
4904
4905           sigalrm_seen = FALSE;
4906           alarm(timeout);
4907           rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun));
4908           alarm(0);
4909           if (sigalrm_seen)
4910             {
4911             expand_string_message = US "socket connect timed out";
4912             goto SOCK_FAIL;
4913             }
4914           if (rc < 0)
4915             {
4916             expand_string_message = string_sprintf("failed to connect to socket "
4917               "%s: %s", sub_arg[0], strerror(errno));
4918             goto SOCK_FAIL;
4919             }
4920           }
4921
4922         DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]);
4923
4924         /* Allow sequencing of test actions */
4925         if (running_in_test_harness) millisleep(100);
4926
4927         /* Write the request string, if not empty or already done */
4928
4929         if (reqstr.len)
4930           {
4931           DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n",
4932             reqstr.data);
4933           if (write(fd, reqstr.data, reqstr.len) != reqstr.len)
4934             {
4935             expand_string_message = string_sprintf("request write to socket "
4936               "failed: %s", strerror(errno));
4937             goto SOCK_FAIL;
4938             }
4939           }
4940
4941         /* Shut down the sending side of the socket. This helps some servers to
4942         recognise that it is their turn to do some work. Just in case some
4943         system doesn't have this function, make it conditional. */
4944
4945 #ifdef SHUT_WR
4946         if (do_shutdown) shutdown(fd, SHUT_WR);
4947 #endif
4948
4949         if (running_in_test_harness) millisleep(100);
4950
4951         /* Now we need to read from the socket, under a timeout. The function
4952         that reads a file can be used. */
4953
4954         f = fdopen(fd, "rb");
4955         sigalrm_seen = FALSE;
4956         alarm(timeout);
4957         yield = cat_file(f, yield, sub_arg[3]);
4958         alarm(0);
4959         (void)fclose(f);
4960
4961         /* After a timeout, we restore the pointer in the result, that is,
4962         make sure we add nothing from the socket. */
4963
4964         if (sigalrm_seen)
4965           {
4966           yield->ptr = save_ptr;
4967           expand_string_message = US "socket read timed out";
4968           goto SOCK_FAIL;
4969           }
4970         }
4971
4972       /* The whole thing has worked (or we were skipping). If there is a
4973       failure string following, we need to skip it. */
4974
4975       if (*s == '{')
4976         {
4977         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
4978           goto EXPAND_FAILED;
4979         if (*s++ != '}')
4980           {
4981           expand_string_message = US"missing '}' closing failstring for readsocket";
4982           goto EXPAND_FAILED_CURLY;
4983           }
4984         while (isspace(*s)) s++;
4985         }
4986
4987     READSOCK_DONE:
4988       if (*s++ != '}')
4989         {
4990         expand_string_message = US"missing '}' closing readsocket";
4991         goto EXPAND_FAILED_CURLY;
4992         }
4993       continue;
4994
4995       /* Come here on failure to create socket, connect socket, write to the
4996       socket, or timeout on reading. If another substring follows, expand and
4997       use it. Otherwise, those conditions give expand errors. */
4998
4999     SOCK_FAIL:
5000       if (*s != '{') goto EXPAND_FAILED;
5001       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
5002       if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok)))
5003         goto EXPAND_FAILED;
5004       yield = string_cat(yield, arg);
5005       if (*s++ != '}')
5006         {
5007         expand_string_message = US"missing '}' closing failstring for readsocket";
5008         goto EXPAND_FAILED_CURLY;
5009         }
5010       while (isspace(*s)) s++;
5011       goto READSOCK_DONE;
5012       }
5013
5014     /* Handle "run" to execute a program. */
5015
5016     case EITEM_RUN:
5017       {
5018       FILE *f;
5019       uschar *arg;
5020       const uschar **argv;
5021       pid_t pid;
5022       int fd_in, fd_out;
5023
5024       if ((expand_forbid & RDO_RUN) != 0)
5025         {
5026         expand_string_message = US"running a command is not permitted";
5027         goto EXPAND_FAILED;
5028         }
5029
5030       while (isspace(*s)) s++;
5031       if (*s != '{')
5032         {
5033         expand_string_message = US"missing '{' for command arg of run";
5034         goto EXPAND_FAILED_CURLY;
5035         }
5036       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5037       if (arg == NULL) goto EXPAND_FAILED;
5038       while (isspace(*s)) s++;
5039       if (*s++ != '}')
5040         {
5041         expand_string_message = US"missing '}' closing command arg of run";
5042         goto EXPAND_FAILED_CURLY;
5043         }
5044
5045       if (skipping)   /* Just pretend it worked when we're skipping */
5046         {
5047         runrc = 0;
5048         lookup_value = NULL;
5049         }
5050       else
5051         {
5052         if (!transport_set_up_command(&argv,    /* anchor for arg list */
5053             arg,                                /* raw command */
5054             FALSE,                              /* don't expand the arguments */
5055             0,                                  /* not relevant when... */
5056             NULL,                               /* no transporting address */
5057             US"${run} expansion",               /* for error messages */
5058             &expand_string_message))            /* where to put error message */
5059           goto EXPAND_FAILED;
5060
5061         /* Create the child process, making it a group leader. */
5062
5063         if ((pid = child_open(USS argv, NULL, 0077, &fd_in, &fd_out, TRUE)) < 0)
5064           {
5065           expand_string_message =
5066             string_sprintf("couldn't create child process: %s", strerror(errno));
5067           goto EXPAND_FAILED;
5068           }
5069
5070         /* Nothing is written to the standard input. */
5071
5072         (void)close(fd_in);
5073
5074         /* Read the pipe to get the command's output into $value (which is kept
5075         in lookup_value). Read during execution, so that if the output exceeds
5076         the OS pipe buffer limit, we don't block forever. Remember to not release
5077         memory just allocated for $value. */
5078
5079         resetok = FALSE;
5080         f = fdopen(fd_out, "rb");
5081         sigalrm_seen = FALSE;
5082         alarm(60);
5083         lookup_value = string_from_gstring(cat_file(f, NULL, NULL));
5084         alarm(0);
5085         (void)fclose(f);
5086
5087         /* Wait for the process to finish, applying the timeout, and inspect its
5088         return code for serious disasters. Simple non-zero returns are passed on.
5089         */
5090
5091         if (sigalrm_seen || (runrc = child_close(pid, 30)) < 0)
5092           {
5093           if (sigalrm_seen || runrc == -256)
5094             {
5095             expand_string_message = string_sprintf("command timed out");
5096             killpg(pid, SIGKILL);       /* Kill the whole process group */
5097             }
5098
5099           else if (runrc == -257)
5100             expand_string_message = string_sprintf("wait() failed: %s",
5101               strerror(errno));
5102
5103           else
5104             expand_string_message = string_sprintf("command killed by signal %d",
5105               -runrc);
5106
5107           goto EXPAND_FAILED;
5108           }
5109         }
5110
5111       /* Process the yes/no strings; $value may be useful in both cases */
5112
5113       switch(process_yesno(
5114                skipping,                     /* were previously skipping */
5115                runrc == 0,                   /* success/failure indicator */
5116                lookup_value,                 /* value to reset for string2 */
5117                &s,                           /* input pointer */
5118                &yield,                       /* output pointer */
5119                US"run",                      /* condition type */
5120                &resetok))
5121         {
5122         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5123         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5124         }
5125
5126       continue;
5127       }
5128
5129     /* Handle character translation for "tr" */
5130
5131     case EITEM_TR:
5132       {
5133       int oldptr = yield->ptr;
5134       int o2m;
5135       uschar *sub[3];
5136
5137       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
5138         {
5139         case 1: goto EXPAND_FAILED_CURLY;
5140         case 2:
5141         case 3: goto EXPAND_FAILED;
5142         }
5143
5144       yield = string_cat(yield, sub[0]);
5145       o2m = Ustrlen(sub[2]) - 1;
5146
5147       if (o2m >= 0) for (; oldptr < yield->ptr; oldptr++)
5148         {
5149         uschar *m = Ustrrchr(sub[1], yield->s[oldptr]);
5150         if (m != NULL)
5151           {
5152           int o = m - sub[1];
5153           yield->s[oldptr] = sub[2][(o < o2m)? o : o2m];
5154           }
5155         }
5156
5157       continue;
5158       }
5159
5160     /* Handle "hash", "length", "nhash", and "substr" when they are given with
5161     expanded arguments. */
5162
5163     case EITEM_HASH:
5164     case EITEM_LENGTH:
5165     case EITEM_NHASH:
5166     case EITEM_SUBSTR:
5167       {
5168       int i;
5169       int len;
5170       uschar *ret;
5171       int val[2] = { 0, -1 };
5172       uschar *sub[3];
5173
5174       /* "length" takes only 2 arguments whereas the others take 2 or 3.
5175       Ensure that sub[2] is set in the ${length } case. */
5176
5177       sub[2] = NULL;
5178       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
5179              TRUE, name, &resetok))
5180         {
5181         case 1: goto EXPAND_FAILED_CURLY;
5182         case 2:
5183         case 3: goto EXPAND_FAILED;
5184         }
5185
5186       /* Juggle the arguments if there are only two of them: always move the
5187       string to the last position and make ${length{n}{str}} equivalent to
5188       ${substr{0}{n}{str}}. See the defaults for val[] above. */
5189
5190       if (sub[2] == NULL)
5191         {
5192         sub[2] = sub[1];
5193         sub[1] = NULL;
5194         if (item_type == EITEM_LENGTH)
5195           {
5196           sub[1] = sub[0];
5197           sub[0] = NULL;
5198           }
5199         }
5200
5201       for (i = 0; i < 2; i++)
5202         {
5203         if (sub[i] == NULL) continue;
5204         val[i] = (int)Ustrtol(sub[i], &ret, 10);
5205         if (*ret != 0 || (i != 0 && val[i] < 0))
5206           {
5207           expand_string_message = string_sprintf("\"%s\" is not a%s number "
5208             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
5209           goto EXPAND_FAILED;
5210           }
5211         }
5212
5213       ret =
5214         (item_type == EITEM_HASH)?
5215           compute_hash(sub[2], val[0], val[1], &len) :
5216         (item_type == EITEM_NHASH)?
5217           compute_nhash(sub[2], val[0], val[1], &len) :
5218           extract_substr(sub[2], val[0], val[1], &len);
5219
5220       if (ret == NULL) goto EXPAND_FAILED;
5221       yield = string_catn(yield, ret, len);
5222       continue;
5223       }
5224
5225     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
5226     This code originally contributed by Steve Haslam. It currently supports
5227     the use of MD5 and SHA-1 hashes.
5228
5229     We need some workspace that is large enough to handle all the supported
5230     hash types. Use macros to set the sizes rather than be too elaborate. */
5231
5232     #define MAX_HASHLEN      20
5233     #define MAX_HASHBLOCKLEN 64
5234
5235     case EITEM_HMAC:
5236       {
5237       uschar *sub[3];
5238       md5 md5_base;
5239       hctx sha1_ctx;
5240       void *use_base;
5241       int type, i;
5242       int hashlen;      /* Number of octets for the hash algorithm's output */
5243       int hashblocklen; /* Number of octets the hash algorithm processes */
5244       uschar *keyptr, *p;
5245       unsigned int keylen;
5246
5247       uschar keyhash[MAX_HASHLEN];
5248       uschar innerhash[MAX_HASHLEN];
5249       uschar finalhash[MAX_HASHLEN];
5250       uschar finalhash_hex[2*MAX_HASHLEN];
5251       uschar innerkey[MAX_HASHBLOCKLEN];
5252       uschar outerkey[MAX_HASHBLOCKLEN];
5253
5254       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
5255         {
5256         case 1: goto EXPAND_FAILED_CURLY;
5257         case 2:
5258         case 3: goto EXPAND_FAILED;
5259         }
5260
5261       if (!skipping)
5262         {
5263         if (Ustrcmp(sub[0], "md5") == 0)
5264           {
5265           type = HMAC_MD5;
5266           use_base = &md5_base;
5267           hashlen = 16;
5268           hashblocklen = 64;
5269           }
5270         else if (Ustrcmp(sub[0], "sha1") == 0)
5271           {
5272           type = HMAC_SHA1;
5273           use_base = &sha1_ctx;
5274           hashlen = 20;
5275           hashblocklen = 64;
5276           }
5277         else
5278           {
5279           expand_string_message =
5280             string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
5281           goto EXPAND_FAILED;
5282           }
5283
5284         keyptr = sub[1];
5285         keylen = Ustrlen(keyptr);
5286
5287         /* If the key is longer than the hash block length, then hash the key
5288         first */
5289
5290         if (keylen > hashblocklen)
5291           {
5292           chash_start(type, use_base);
5293           chash_end(type, use_base, keyptr, keylen, keyhash);
5294           keyptr = keyhash;
5295           keylen = hashlen;
5296           }
5297
5298         /* Now make the inner and outer key values */
5299
5300         memset(innerkey, 0x36, hashblocklen);
5301         memset(outerkey, 0x5c, hashblocklen);
5302
5303         for (i = 0; i < keylen; i++)
5304           {
5305           innerkey[i] ^= keyptr[i];
5306           outerkey[i] ^= keyptr[i];
5307           }
5308
5309         /* Now do the hashes */
5310
5311         chash_start(type, use_base);
5312         chash_mid(type, use_base, innerkey);
5313         chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5314
5315         chash_start(type, use_base);
5316         chash_mid(type, use_base, outerkey);
5317         chash_end(type, use_base, innerhash, hashlen, finalhash);
5318
5319         /* Encode the final hash as a hex string */
5320
5321         p = finalhash_hex;
5322         for (i = 0; i < hashlen; i++)
5323           {
5324           *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5325           *p++ = hex_digits[finalhash[i] & 0x0f];
5326           }
5327
5328         DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%s)=%.*s\n",
5329           sub[0], (int)keylen, keyptr, sub[2], hashlen*2, finalhash_hex);
5330
5331         yield = string_catn(yield, finalhash_hex, hashlen*2);
5332         }
5333       continue;
5334       }
5335
5336     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5337     We have to save the numerical variables and restore them afterwards. */
5338
5339     case EITEM_SG:
5340       {
5341       const pcre *re;
5342       int moffset, moffsetextra, slen;
5343       int roffset;
5344       int emptyopt;
5345       const uschar *rerror;
5346       uschar *subject;
5347       uschar *sub[3];
5348       int save_expand_nmax =
5349         save_expand_strings(save_expand_nstring, save_expand_nlength);
5350
5351       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5352         {
5353         case 1: goto EXPAND_FAILED_CURLY;
5354         case 2:
5355         case 3: goto EXPAND_FAILED;
5356         }
5357
5358       /* Compile the regular expression */
5359
5360       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5361         NULL);
5362
5363       if (re == NULL)
5364         {
5365         expand_string_message = string_sprintf("regular expression error in "
5366           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5367         goto EXPAND_FAILED;
5368         }
5369
5370       /* Now run a loop to do the substitutions as often as necessary. It ends
5371       when there are no more matches. Take care over matches of the null string;
5372       do the same thing as Perl does. */
5373
5374       subject = sub[0];
5375       slen = Ustrlen(sub[0]);
5376       moffset = moffsetextra = 0;
5377       emptyopt = 0;
5378
5379       for (;;)
5380         {
5381         int ovector[3*(EXPAND_MAXN+1)];
5382         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5383           PCRE_EOPT | emptyopt, ovector, nelem(ovector));
5384         int nn;
5385         uschar *insert;
5386
5387         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5388         is not necessarily the end. We want to repeat the match from one
5389         character further along, but leaving the basic offset the same (for
5390         copying below). We can't be at the end of the string - that was checked
5391         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5392         finished; copy the remaining string and end the loop. */
5393
5394         if (n < 0)
5395           {
5396           if (emptyopt != 0)
5397             {
5398             moffsetextra = 1;
5399             emptyopt = 0;
5400             continue;
5401             }
5402           yield = string_catn(yield, subject+moffset, slen-moffset);
5403           break;
5404           }
5405
5406         /* Match - set up for expanding the replacement. */
5407
5408         if (n == 0) n = EXPAND_MAXN + 1;
5409         expand_nmax = 0;
5410         for (nn = 0; nn < n*2; nn += 2)
5411           {
5412           expand_nstring[expand_nmax] = subject + ovector[nn];
5413           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5414           }
5415         expand_nmax--;
5416
5417         /* Copy the characters before the match, plus the expanded insertion. */
5418
5419         yield = string_catn(yield, subject + moffset, ovector[0] - moffset);
5420         insert = expand_string(sub[2]);
5421         if (insert == NULL) goto EXPAND_FAILED;
5422         yield = string_cat(yield, insert);
5423
5424         moffset = ovector[1];
5425         moffsetextra = 0;
5426         emptyopt = 0;
5427
5428         /* If we have matched an empty string, first check to see if we are at
5429         the end of the subject. If so, the loop is over. Otherwise, mimic
5430         what Perl's /g options does. This turns out to be rather cunning. First
5431         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5432         string at the same point. If this fails (picked up above) we advance to
5433         the next character. */
5434
5435         if (ovector[0] == ovector[1])
5436           {
5437           if (ovector[0] == slen) break;
5438           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5439           }
5440         }
5441
5442       /* All done - restore numerical variables. */
5443
5444       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5445         save_expand_nlength);
5446       continue;
5447       }
5448
5449     /* Handle keyed and numbered substring extraction. If the first argument
5450     consists entirely of digits, then a numerical extraction is assumed. */
5451
5452     case EITEM_EXTRACT:
5453       {
5454       int i;
5455       int j;
5456       int field_number = 1;
5457       BOOL field_number_set = FALSE;
5458       uschar *save_lookup_value = lookup_value;
5459       uschar *sub[3];
5460       int save_expand_nmax =
5461         save_expand_strings(save_expand_nstring, save_expand_nlength);
5462
5463       /* While skipping we cannot rely on the data for expansions being
5464       available (eg. $item) hence cannot decide on numeric vs. keyed.
5465       Read a maximum of 5 arguments (including the yes/no) */
5466
5467       if (skipping)
5468         {
5469         while (isspace(*s)) s++;
5470         for (j = 5; j > 0 && *s == '{'; j--)
5471           {
5472           if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))
5473             goto EXPAND_FAILED;                                 /*{*/
5474           if (*s++ != '}')
5475             {
5476             expand_string_message = US"missing '{' for arg of extract";
5477             goto EXPAND_FAILED_CURLY;
5478             }
5479           while (isspace(*s)) s++;
5480           }
5481         if (  Ustrncmp(s, "fail", 4) == 0
5482            && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4])
5483            )
5484           {
5485           s += 4;
5486           while (isspace(*s)) s++;
5487           }
5488         if (*s != '}')
5489           {
5490           expand_string_message = US"missing '}' closing extract";
5491           goto EXPAND_FAILED_CURLY;
5492           }
5493         }
5494
5495       else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */
5496         {
5497         while (isspace(*s)) s++;
5498         if (*s == '{')                                          /*}*/
5499           {
5500           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5501           if (sub[i] == NULL) goto EXPAND_FAILED;               /*{*/
5502           if (*s++ != '}')
5503             {
5504             expand_string_message = string_sprintf(
5505               "missing '}' closing arg %d of extract", i+1);
5506             goto EXPAND_FAILED_CURLY;
5507             }
5508
5509           /* After removal of leading and trailing white space, the first
5510           argument must not be empty; if it consists entirely of digits
5511           (optionally preceded by a minus sign), this is a numerical
5512           extraction, and we expect 3 arguments. */
5513
5514           if (i == 0)
5515             {
5516             int len;
5517             int x = 0;
5518             uschar *p = sub[0];
5519
5520             while (isspace(*p)) p++;
5521             sub[0] = p;
5522
5523             len = Ustrlen(p);
5524             while (len > 0 && isspace(p[len-1])) len--;
5525             p[len] = 0;
5526
5527             if (*p == 0)
5528               {
5529               expand_string_message = US"first argument of \"extract\" must "
5530                 "not be empty";
5531               goto EXPAND_FAILED;
5532               }
5533
5534             if (*p == '-')
5535               {
5536               field_number = -1;
5537               p++;
5538               }
5539             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5540             if (*p == 0)
5541               {
5542               field_number *= x;
5543               j = 3;               /* Need 3 args */
5544               field_number_set = TRUE;
5545               }
5546             }
5547           }
5548         else
5549           {
5550           expand_string_message = string_sprintf(
5551             "missing '{' for arg %d of extract", i+1);
5552           goto EXPAND_FAILED_CURLY;
5553           }
5554         }
5555
5556       /* Extract either the numbered or the keyed substring into $value. If
5557       skipping, just pretend the extraction failed. */
5558
5559       lookup_value = skipping? NULL : field_number_set?
5560         expand_gettokened(field_number, sub[1], sub[2]) :
5561         expand_getkeyed(sub[0], sub[1]);
5562
5563       /* If no string follows, $value gets substituted; otherwise there can
5564       be yes/no strings, as for lookup or if. */
5565
5566       switch(process_yesno(
5567                skipping,                     /* were previously skipping */
5568                lookup_value != NULL,         /* success/failure indicator */
5569                save_lookup_value,            /* value to reset for string2 */
5570                &s,                           /* input pointer */
5571                &yield,                       /* output pointer */
5572                US"extract",                  /* condition type */
5573                &resetok))
5574         {
5575         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5576         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5577         }
5578
5579       /* All done - restore numerical variables. */
5580
5581       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5582         save_expand_nlength);
5583
5584       continue;
5585       }
5586
5587     /* return the Nth item from a list */
5588
5589     case EITEM_LISTEXTRACT:
5590       {
5591       int i;
5592       int field_number = 1;
5593       uschar *save_lookup_value = lookup_value;
5594       uschar *sub[2];
5595       int save_expand_nmax =
5596         save_expand_strings(save_expand_nstring, save_expand_nlength);
5597
5598       /* Read the field & list arguments */
5599
5600       for (i = 0; i < 2; i++)
5601         {
5602         while (isspace(*s)) s++;
5603         if (*s != '{')                                  /*}*/
5604           {
5605           expand_string_message = string_sprintf(
5606             "missing '{' for arg %d of listextract", i+1);
5607           goto EXPAND_FAILED_CURLY;
5608           }
5609
5610         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5611         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5612         if (*s++ != '}')
5613           {
5614           expand_string_message = string_sprintf(
5615             "missing '}' closing arg %d of listextract", i+1);
5616           goto EXPAND_FAILED_CURLY;
5617           }
5618
5619         /* After removal of leading and trailing white space, the first
5620         argument must be numeric and nonempty. */
5621
5622         if (i == 0)
5623           {
5624           int len;
5625           int x = 0;
5626           uschar *p = sub[0];
5627
5628           while (isspace(*p)) p++;
5629           sub[0] = p;
5630
5631           len = Ustrlen(p);
5632           while (len > 0 && isspace(p[len-1])) len--;
5633           p[len] = 0;
5634
5635           if (!*p && !skipping)
5636             {
5637             expand_string_message = US"first argument of \"listextract\" must "
5638               "not be empty";
5639             goto EXPAND_FAILED;
5640             }
5641
5642           if (*p == '-')
5643             {
5644             field_number = -1;
5645             p++;
5646             }
5647           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5648           if (*p)
5649             {
5650             expand_string_message = US"first argument of \"listextract\" must "
5651               "be numeric";
5652             goto EXPAND_FAILED;
5653             }
5654           field_number *= x;
5655           }
5656         }
5657
5658       /* Extract the numbered element into $value. If
5659       skipping, just pretend the extraction failed. */
5660
5661       lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]);
5662
5663       /* If no string follows, $value gets substituted; otherwise there can
5664       be yes/no strings, as for lookup or if. */
5665
5666       switch(process_yesno(
5667                skipping,                     /* were previously skipping */
5668                lookup_value != NULL,         /* success/failure indicator */
5669                save_lookup_value,            /* value to reset for string2 */
5670                &s,                           /* input pointer */
5671                &yield,                       /* output pointer */
5672                US"listextract",              /* condition type */
5673                &resetok))
5674         {
5675         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5676         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5677         }
5678
5679       /* All done - restore numerical variables. */
5680
5681       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5682         save_expand_nlength);
5683
5684       continue;
5685       }
5686
5687 #ifdef SUPPORT_TLS
5688     case EITEM_CERTEXTRACT:
5689       {
5690       uschar *save_lookup_value = lookup_value;
5691       uschar *sub[2];
5692       int save_expand_nmax =
5693         save_expand_strings(save_expand_nstring, save_expand_nlength);
5694
5695       /* Read the field argument */
5696       while (isspace(*s)) s++;
5697       if (*s != '{')                                    /*}*/
5698         {
5699         expand_string_message = US"missing '{' for field arg of certextract";
5700         goto EXPAND_FAILED_CURLY;
5701         }
5702       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5703       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
5704       if (*s++ != '}')
5705         {
5706         expand_string_message = US"missing '}' closing field arg of certextract";
5707         goto EXPAND_FAILED_CURLY;
5708         }
5709       /* strip spaces fore & aft */
5710       {
5711       int len;
5712       uschar *p = sub[0];
5713
5714       while (isspace(*p)) p++;
5715       sub[0] = p;
5716
5717       len = Ustrlen(p);
5718       while (len > 0 && isspace(p[len-1])) len--;
5719       p[len] = 0;
5720       }
5721
5722       /* inspect the cert argument */
5723       while (isspace(*s)) s++;
5724       if (*s != '{')                                    /*}*/
5725         {
5726         expand_string_message = US"missing '{' for cert variable arg of certextract";
5727         goto EXPAND_FAILED_CURLY;
5728         }
5729       if (*++s != '$')
5730         {
5731         expand_string_message = US"second argument of \"certextract\" must "
5732           "be a certificate variable";
5733         goto EXPAND_FAILED;
5734         }
5735       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
5736       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
5737       if (*s++ != '}')
5738         {
5739         expand_string_message = US"missing '}' closing cert variable arg of certextract";
5740         goto EXPAND_FAILED_CURLY;
5741         }
5742
5743       if (skipping)
5744         lookup_value = NULL;
5745       else
5746         {
5747         lookup_value = expand_getcertele(sub[0], sub[1]);
5748         if (*expand_string_message) goto EXPAND_FAILED;
5749         }
5750       switch(process_yesno(
5751                skipping,                     /* were previously skipping */
5752                lookup_value != NULL,         /* success/failure indicator */
5753                save_lookup_value,            /* value to reset for string2 */
5754                &s,                           /* input pointer */
5755                &yield,                       /* output pointer */
5756                US"certextract",              /* condition type */
5757                &resetok))
5758         {
5759         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5760         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5761         }
5762
5763       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5764         save_expand_nlength);
5765       continue;
5766       }
5767 #endif  /*SUPPORT_TLS*/
5768
5769     /* Handle list operations */
5770
5771     case EITEM_FILTER:
5772     case EITEM_MAP:
5773     case EITEM_REDUCE:
5774       {
5775       int sep = 0;
5776       int save_ptr = yield->ptr;
5777       uschar outsep[2] = { '\0', '\0' };
5778       const uschar *list, *expr, *temp;
5779       uschar *save_iterate_item = iterate_item;
5780       uschar *save_lookup_value = lookup_value;
5781
5782       while (isspace(*s)) s++;
5783       if (*s++ != '{')
5784         {
5785         expand_string_message =
5786           string_sprintf("missing '{' for first arg of %s", name);
5787         goto EXPAND_FAILED_CURLY;
5788         }
5789
5790       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5791       if (list == NULL) goto EXPAND_FAILED;
5792       if (*s++ != '}')
5793         {
5794         expand_string_message =
5795           string_sprintf("missing '}' closing first arg of %s", name);
5796         goto EXPAND_FAILED_CURLY;
5797         }
5798
5799       if (item_type == EITEM_REDUCE)
5800         {
5801         uschar * t;
5802         while (isspace(*s)) s++;
5803         if (*s++ != '{')
5804           {
5805           expand_string_message = US"missing '{' for second arg of reduce";
5806           goto EXPAND_FAILED_CURLY;
5807           }
5808         t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5809         if (!t) goto EXPAND_FAILED;
5810         lookup_value = t;
5811         if (*s++ != '}')
5812           {
5813           expand_string_message = US"missing '}' closing second arg of reduce";
5814           goto EXPAND_FAILED_CURLY;
5815           }
5816         }
5817
5818       while (isspace(*s)) s++;
5819       if (*s++ != '{')
5820         {
5821         expand_string_message =
5822           string_sprintf("missing '{' for last arg of %s", name);
5823         goto EXPAND_FAILED_CURLY;
5824         }
5825
5826       expr = s;
5827
5828       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
5829       if scanning a "false" part). This allows us to find the end of the
5830       condition, because if the list is empty, we won't actually evaluate the
5831       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
5832       the normal internal expansion function. */
5833
5834       if (item_type == EITEM_FILTER)
5835         {
5836         temp = eval_condition(expr, &resetok, NULL);
5837         if (temp != NULL) s = temp;
5838         }
5839       else
5840         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5841
5842       if (temp == NULL)
5843         {
5844         expand_string_message = string_sprintf("%s inside \"%s\" item",
5845           expand_string_message, name);
5846         goto EXPAND_FAILED;
5847         }
5848
5849       while (isspace(*s)) s++;
5850       if (*s++ != '}')
5851         {                                               /*{*/
5852         expand_string_message = string_sprintf("missing } at end of condition "
5853           "or expression inside \"%s\"; could be an unquoted } in the content",
5854           name);
5855         goto EXPAND_FAILED;
5856         }
5857
5858       while (isspace(*s)) s++;                          /*{*/
5859       if (*s++ != '}')
5860         {                                               /*{*/
5861         expand_string_message = string_sprintf("missing } at end of \"%s\"",
5862           name);
5863         goto EXPAND_FAILED;
5864         }
5865
5866       /* If we are skipping, we can now just move on to the next item. When
5867       processing for real, we perform the iteration. */
5868
5869       if (skipping) continue;
5870       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
5871         {
5872         *outsep = (uschar)sep;      /* Separator as a string */
5873
5874         DEBUG(D_expand) debug_printf_indent("%s: $item = '%s'  $value = '%s'\n",
5875                           name, iterate_item, lookup_value);
5876
5877         if (item_type == EITEM_FILTER)
5878           {
5879           BOOL condresult;
5880           if (eval_condition(expr, &resetok, &condresult) == NULL)
5881             {
5882             iterate_item = save_iterate_item;
5883             lookup_value = save_lookup_value;
5884             expand_string_message = string_sprintf("%s inside \"%s\" condition",
5885               expand_string_message, name);
5886             goto EXPAND_FAILED;
5887             }
5888           DEBUG(D_expand) debug_printf_indent("%s: condition is %s\n", name,
5889             condresult? "true":"false");
5890           if (condresult)
5891             temp = iterate_item;    /* TRUE => include this item */
5892           else
5893             continue;               /* FALSE => skip this item */
5894           }
5895
5896         /* EITEM_MAP and EITEM_REDUCE */
5897
5898         else
5899           {
5900           uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
5901           temp = t;
5902           if (temp == NULL)
5903             {
5904             iterate_item = save_iterate_item;
5905             expand_string_message = string_sprintf("%s inside \"%s\" item",
5906               expand_string_message, name);
5907             goto EXPAND_FAILED;
5908             }
5909           if (item_type == EITEM_REDUCE)
5910             {
5911             lookup_value = t;         /* Update the value of $value */
5912             continue;                 /* and continue the iteration */
5913             }
5914           }
5915
5916         /* We reach here for FILTER if the condition is true, always for MAP,
5917         and never for REDUCE. The value in "temp" is to be added to the output
5918         list that is being created, ensuring that any occurrences of the
5919         separator character are doubled. Unless we are dealing with the first
5920         item of the output list, add in a space if the new item begins with the
5921         separator character, or is an empty string. */
5922
5923         if (yield->ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
5924           yield = string_catn(yield, US" ", 1);
5925
5926         /* Add the string in "temp" to the output list that we are building,
5927         This is done in chunks by searching for the separator character. */
5928
5929         for (;;)
5930           {
5931           size_t seglen = Ustrcspn(temp, outsep);
5932
5933           yield = string_catn(yield, temp, seglen + 1);
5934
5935           /* If we got to the end of the string we output one character
5936           too many; backup and end the loop. Otherwise arrange to double the
5937           separator. */
5938
5939           if (temp[seglen] == '\0') { yield->ptr--; break; }
5940           yield = string_catn(yield, outsep, 1);
5941           temp += seglen + 1;
5942           }
5943
5944         /* Output a separator after the string: we will remove the redundant
5945         final one at the end. */
5946
5947         yield = string_catn(yield, outsep, 1);
5948         }   /* End of iteration over the list loop */
5949
5950       /* REDUCE has generated no output above: output the final value of
5951       $value. */
5952
5953       if (item_type == EITEM_REDUCE)
5954         {
5955         yield = string_cat(yield, lookup_value);
5956         lookup_value = save_lookup_value;  /* Restore $value */
5957         }
5958
5959       /* FILTER and MAP generate lists: if they have generated anything, remove
5960       the redundant final separator. Even though an empty item at the end of a
5961       list does not count, this is tidier. */
5962
5963       else if (yield->ptr != save_ptr) yield->ptr--;
5964
5965       /* Restore preserved $item */
5966
5967       iterate_item = save_iterate_item;
5968       continue;
5969       }
5970
5971     case EITEM_SORT:
5972       {
5973       int sep = 0;
5974       const uschar *srclist, *cmp, *xtract;
5975       uschar *srcitem;
5976       const uschar *dstlist = NULL, *dstkeylist = NULL;
5977       uschar * tmp;
5978       uschar *save_iterate_item = iterate_item;
5979
5980       while (isspace(*s)) s++;
5981       if (*s++ != '{')
5982         {
5983         expand_string_message = US"missing '{' for list arg of sort";
5984         goto EXPAND_FAILED_CURLY;
5985         }
5986
5987       srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5988       if (!srclist) goto EXPAND_FAILED;
5989       if (*s++ != '}')
5990         {
5991         expand_string_message = US"missing '}' closing list arg of sort";
5992         goto EXPAND_FAILED_CURLY;
5993         }
5994
5995       while (isspace(*s)) s++;
5996       if (*s++ != '{')
5997         {
5998         expand_string_message = US"missing '{' for comparator arg of sort";
5999         goto EXPAND_FAILED_CURLY;
6000         }
6001
6002       cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok);
6003       if (!cmp) goto EXPAND_FAILED;
6004       if (*s++ != '}')
6005         {
6006         expand_string_message = US"missing '}' closing comparator arg of sort";
6007         goto EXPAND_FAILED_CURLY;
6008         }
6009
6010       while (isspace(*s)) s++;
6011       if (*s++ != '{')
6012         {
6013         expand_string_message = US"missing '{' for extractor arg of sort";
6014         goto EXPAND_FAILED_CURLY;
6015         }
6016
6017       xtract = s;
6018       tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
6019       if (!tmp) goto EXPAND_FAILED;
6020       xtract = string_copyn(xtract, s - xtract);
6021
6022       if (*s++ != '}')
6023         {
6024         expand_string_message = US"missing '}' closing extractor arg of sort";
6025         goto EXPAND_FAILED_CURLY;
6026         }
6027                                                         /*{*/
6028       if (*s++ != '}')
6029         {                                               /*{*/
6030         expand_string_message = US"missing } at end of \"sort\"";
6031         goto EXPAND_FAILED;
6032         }
6033
6034       if (skipping) continue;
6035
6036       while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0)))
6037         {
6038         uschar * dstitem;
6039         gstring * newlist = NULL;
6040         gstring * newkeylist = NULL;
6041         uschar * srcfield;
6042
6043         DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, srcitem);
6044
6045         /* extract field for comparisons */
6046         iterate_item = srcitem;
6047         if (  !(srcfield = expand_string_internal(xtract, FALSE, NULL, FALSE,
6048                                           TRUE, &resetok))
6049            || !*srcfield)
6050           {
6051           expand_string_message = string_sprintf(
6052               "field-extract in sort: \"%s\"", xtract);
6053           goto EXPAND_FAILED;
6054           }
6055
6056         /* Insertion sort */
6057
6058         /* copy output list until new-item < list-item */
6059         while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6060           {
6061           uschar * dstfield;
6062           uschar * expr;
6063           BOOL before;
6064
6065           /* field for comparison */
6066           if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6067             goto sort_mismatch;
6068
6069           /* build and run condition string */
6070           expr = string_sprintf("%s{%s}{%s}", cmp, srcfield, dstfield);
6071
6072           DEBUG(D_expand) debug_printf_indent("%s: cond = \"%s\"\n", name, expr);
6073           if (!eval_condition(expr, &resetok, &before))
6074             {
6075             expand_string_message = string_sprintf("comparison in sort: %s",
6076                 expr);
6077             goto EXPAND_FAILED;
6078             }
6079
6080           if (before)
6081             {
6082             /* New-item sorts before this dst-item.  Append new-item,
6083             then dst-item, then remainder of dst list. */
6084
6085             newlist = string_append_listele(newlist, sep, srcitem);
6086             newkeylist = string_append_listele(newkeylist, sep, srcfield);
6087             srcitem = NULL;
6088
6089             newlist = string_append_listele(newlist, sep, dstitem);
6090             newkeylist = string_append_listele(newkeylist, sep, dstfield);
6091
6092             while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6093               {
6094               if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6095                 goto sort_mismatch;
6096               newlist = string_append_listele(newlist, sep, dstitem);
6097               newkeylist = string_append_listele(newkeylist, sep, dstfield);
6098               }
6099
6100             break;
6101             }
6102
6103           newlist = string_append_listele(newlist, sep, dstitem);
6104           newkeylist = string_append_listele(newkeylist, sep, dstfield);
6105           }
6106
6107         /* If we ran out of dstlist without consuming srcitem, append it */
6108         if (srcitem)
6109           {
6110           newlist = string_append_listele(newlist, sep, srcitem);
6111           newkeylist = string_append_listele(newkeylist, sep, srcfield);
6112           }
6113
6114         dstlist = newlist->s;
6115         dstkeylist = newkeylist->s;
6116
6117         DEBUG(D_expand) debug_printf_indent("%s: dstlist = \"%s\"\n", name, dstlist);
6118         DEBUG(D_expand) debug_printf_indent("%s: dstkeylist = \"%s\"\n", name, dstkeylist);
6119         }
6120
6121       if (dstlist)
6122         yield = string_cat(yield, dstlist);
6123
6124       /* Restore preserved $item */
6125       iterate_item = save_iterate_item;
6126       continue;
6127
6128       sort_mismatch:
6129         expand_string_message = US"Internal error in sort (list mismatch)";
6130         goto EXPAND_FAILED;
6131       }
6132
6133
6134     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
6135     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
6136     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
6137     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
6138
6139     #define EXPAND_DLFUNC_MAX_ARGS 8
6140
6141     case EITEM_DLFUNC:
6142 #ifndef EXPAND_DLFUNC
6143       expand_string_message = US"\"${dlfunc\" encountered, but this facility "  /*}*/
6144         "is not included in this binary";
6145       goto EXPAND_FAILED;
6146
6147 #else   /* EXPAND_DLFUNC */
6148       {
6149       tree_node *t;
6150       exim_dlfunc_t *func;
6151       uschar *result;
6152       int status, argc;
6153       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
6154
6155       if ((expand_forbid & RDO_DLFUNC) != 0)
6156         {
6157         expand_string_message =
6158           US"dynamically-loaded functions are not permitted";
6159         goto EXPAND_FAILED;
6160         }
6161
6162       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
6163            TRUE, US"dlfunc", &resetok))
6164         {
6165         case 1: goto EXPAND_FAILED_CURLY;
6166         case 2:
6167         case 3: goto EXPAND_FAILED;
6168         }
6169
6170       /* If skipping, we don't actually do anything */
6171
6172       if (skipping) continue;
6173
6174       /* Look up the dynamically loaded object handle in the tree. If it isn't
6175       found, dlopen() the file and put the handle in the tree for next time. */
6176
6177       t = tree_search(dlobj_anchor, argv[0]);
6178       if (t == NULL)
6179         {
6180         void *handle = dlopen(CS argv[0], RTLD_LAZY);
6181         if (handle == NULL)
6182           {
6183           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
6184             argv[0], dlerror());
6185           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6186           goto EXPAND_FAILED;
6187           }
6188         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
6189         Ustrcpy(t->name, argv[0]);
6190         t->data.ptr = handle;
6191         (void)tree_insertnode(&dlobj_anchor, t);
6192         }
6193
6194       /* Having obtained the dynamically loaded object handle, look up the
6195       function pointer. */
6196
6197       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
6198       if (func == NULL)
6199         {
6200         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
6201           "%s", argv[1], argv[0], dlerror());
6202         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6203         goto EXPAND_FAILED;
6204         }
6205
6206       /* Call the function and work out what to do with the result. If it
6207       returns OK, we have a replacement string; if it returns DEFER then
6208       expansion has failed in a non-forced manner; if it returns FAIL then
6209       failure was forced; if it returns ERROR or any other value there's a
6210       problem, so panic slightly. In any case, assume that the function has
6211       side-effects on the store that must be preserved. */
6212
6213       resetok = FALSE;
6214       result = NULL;
6215       for (argc = 0; argv[argc] != NULL; argc++);
6216       status = func(&result, argc - 2, &argv[2]);
6217       if(status == OK)
6218         {
6219         if (result == NULL) result = US"";
6220         yield = string_cat(yield, result);
6221         continue;
6222         }
6223       else
6224         {
6225         expand_string_message = result == NULL ? US"(no message)" : result;
6226         if(status == FAIL_FORCED) expand_string_forcedfail = TRUE;
6227           else if(status != FAIL)
6228             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
6229               argv[0], argv[1], status, expand_string_message);
6230         goto EXPAND_FAILED;
6231         }
6232       }
6233 #endif /* EXPAND_DLFUNC */
6234
6235     case EITEM_ENV:     /* ${env {name} {val_if_found} {val_if_unfound}} */
6236       {
6237       uschar * key;
6238       uschar *save_lookup_value = lookup_value;
6239
6240       while (isspace(*s)) s++;
6241       if (*s != '{')                                    /*}*/
6242         goto EXPAND_FAILED;
6243
6244       key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6245       if (!key) goto EXPAND_FAILED;                     /*{*/
6246       if (*s++ != '}')
6247         {
6248         expand_string_message = US"missing '{' for name arg of env";
6249         goto EXPAND_FAILED_CURLY;
6250         }
6251
6252       lookup_value = US getenv(CS key);
6253
6254       switch(process_yesno(
6255                skipping,                     /* were previously skipping */
6256                lookup_value != NULL,         /* success/failure indicator */
6257                save_lookup_value,            /* value to reset for string2 */
6258                &s,                           /* input pointer */
6259                &yield,                       /* output pointer */
6260                US"env",                      /* condition type */
6261                &resetok))
6262         {
6263         case 1: goto EXPAND_FAILED;          /* when all is well, the */
6264         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
6265         }
6266       continue;
6267       }
6268     }   /* EITEM_* switch */
6269
6270   /* Control reaches here if the name is not recognized as one of the more
6271   complicated expansion items. Check for the "operator" syntax (name terminated
6272   by a colon). Some of the operators have arguments, separated by _ from the
6273   name. */
6274
6275   if (*s == ':')
6276     {
6277     int c;
6278     uschar *arg = NULL;
6279     uschar *sub;
6280     var_entry *vp = NULL;
6281
6282     /* Owing to an historical mis-design, an underscore may be part of the
6283     operator name, or it may introduce arguments.  We therefore first scan the
6284     table of names that contain underscores. If there is no match, we cut off
6285     the arguments and then scan the main table. */
6286
6287     if ((c = chop_match(name, op_table_underscore,
6288                         nelem(op_table_underscore))) < 0)
6289       {
6290       arg = Ustrchr(name, '_');
6291       if (arg != NULL) *arg = 0;
6292       c = chop_match(name, op_table_main, nelem(op_table_main));
6293       if (c >= 0) c += nelem(op_table_underscore);
6294       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
6295       }
6296
6297     /* Deal specially with operators that might take a certificate variable
6298     as we do not want to do the usual expansion. For most, expand the string.*/
6299     switch(c)
6300       {
6301 #ifdef SUPPORT_TLS
6302       case EOP_MD5:
6303       case EOP_SHA1:
6304       case EOP_SHA256:
6305       case EOP_BASE64:
6306         if (s[1] == '$')
6307           {
6308           const uschar * s1 = s;
6309           sub = expand_string_internal(s+2, TRUE, &s1, skipping,
6310                   FALSE, &resetok);
6311           if (!sub)       goto EXPAND_FAILED;           /*{*/
6312           if (*s1 != '}')
6313             {
6314             expand_string_message =
6315               string_sprintf("missing '}' closing cert arg of %s", name);
6316             goto EXPAND_FAILED_CURLY;
6317             }
6318           if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
6319             {
6320             s = s1+1;
6321             break;
6322             }
6323           vp = NULL;
6324           }
6325         /*FALLTHROUGH*/
6326 #endif
6327       default:
6328         sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6329         if (!sub) goto EXPAND_FAILED;
6330         s++;
6331         break;
6332       }
6333
6334     /* If we are skipping, we don't need to perform the operation at all.
6335     This matters for operations like "mask", because the data may not be
6336     in the correct format when skipping. For example, the expression may test
6337     for the existence of $sender_host_address before trying to mask it. For
6338     other operations, doing them may not fail, but it is a waste of time. */
6339
6340     if (skipping && c >= 0) continue;
6341
6342     /* Otherwise, switch on the operator type */
6343
6344     switch(c)
6345       {
6346       case EOP_BASE32:
6347         {
6348         uschar *t;
6349         unsigned long int n = Ustrtoul(sub, &t, 10);
6350         gstring * g = NULL;
6351
6352         if (*t != 0)
6353           {
6354           expand_string_message = string_sprintf("argument for base32 "
6355             "operator is \"%s\", which is not a decimal number", sub);
6356           goto EXPAND_FAILED;
6357           }
6358         for ( ; n; n >>= 5)
6359           g = string_catn(g, &base32_chars[n & 0x1f], 1);
6360
6361         if (g) while (g->ptr > 0) yield = string_catn(yield, &g->s[--g->ptr], 1);
6362         continue;
6363         }
6364
6365       case EOP_BASE32D:
6366         {
6367         uschar *tt = sub;
6368         unsigned long int n = 0;
6369         uschar * s;
6370         while (*tt)
6371           {
6372           uschar * t = Ustrchr(base32_chars, *tt++);
6373           if (t == NULL)
6374             {
6375             expand_string_message = string_sprintf("argument for base32d "
6376               "operator is \"%s\", which is not a base 32 number", sub);
6377             goto EXPAND_FAILED;
6378             }
6379           n = n * 32 + (t - base32_chars);
6380           }
6381         s = string_sprintf("%ld", n);
6382         yield = string_cat(yield, s);
6383         continue;
6384         }
6385
6386       case EOP_BASE62:
6387         {
6388         uschar *t;
6389         unsigned long int n = Ustrtoul(sub, &t, 10);
6390         if (*t != 0)
6391           {
6392           expand_string_message = string_sprintf("argument for base62 "
6393             "operator is \"%s\", which is not a decimal number", sub);
6394           goto EXPAND_FAILED;
6395           }
6396         t = string_base62(n);
6397         yield = string_cat(yield, t);
6398         continue;
6399         }
6400
6401       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
6402
6403       case EOP_BASE62D:
6404         {
6405         uschar buf[16];
6406         uschar *tt = sub;
6407         unsigned long int n = 0;
6408         while (*tt != 0)
6409           {
6410           uschar *t = Ustrchr(base62_chars, *tt++);
6411           if (t == NULL)
6412             {
6413             expand_string_message = string_sprintf("argument for base62d "
6414               "operator is \"%s\", which is not a base %d number", sub,
6415               BASE_62);
6416             goto EXPAND_FAILED;
6417             }
6418           n = n * BASE_62 + (t - base62_chars);
6419           }
6420         (void)sprintf(CS buf, "%ld", n);
6421         yield = string_cat(yield, buf);
6422         continue;
6423         }
6424
6425       case EOP_EXPAND:
6426         {
6427         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
6428         if (expanded == NULL)
6429           {
6430           expand_string_message =
6431             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
6432               expand_string_message);
6433           goto EXPAND_FAILED;
6434           }
6435         yield = string_cat(yield, expanded);
6436         continue;
6437         }
6438
6439       case EOP_LC:
6440         {
6441         int count = 0;
6442         uschar *t = sub - 1;
6443         while (*(++t) != 0) { *t = tolower(*t); count++; }
6444         yield = string_catn(yield, sub, count);
6445         continue;
6446         }
6447
6448       case EOP_UC:
6449         {
6450         int count = 0;
6451         uschar *t = sub - 1;
6452         while (*(++t) != 0) { *t = toupper(*t); count++; }
6453         yield = string_catn(yield, sub, count);
6454         continue;
6455         }
6456
6457       case EOP_MD5:
6458 #ifdef SUPPORT_TLS
6459         if (vp && *(void **)vp->value)
6460           {
6461           uschar * cp = tls_cert_fprt_md5(*(void **)vp->value);
6462           yield = string_cat(yield, cp);
6463           }
6464         else
6465 #endif
6466           {
6467           md5 base;
6468           uschar digest[16];
6469           int j;
6470           char st[33];
6471           md5_start(&base);
6472           md5_end(&base, sub, Ustrlen(sub), digest);
6473           for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]);
6474           yield = string_cat(yield, US st);
6475           }
6476         continue;
6477
6478       case EOP_SHA1:
6479 #ifdef SUPPORT_TLS
6480         if (vp && *(void **)vp->value)
6481           {
6482           uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value);
6483           yield = string_cat(yield, cp);
6484           }
6485         else
6486 #endif
6487           {
6488           hctx h;
6489           uschar digest[20];
6490           int j;
6491           char st[41];
6492           sha1_start(&h);
6493           sha1_end(&h, sub, Ustrlen(sub), digest);
6494           for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
6495           yield = string_catn(yield, US st, 40);
6496           }
6497         continue;
6498
6499       case EOP_SHA256:
6500 #ifdef EXIM_HAVE_SHA2
6501         if (vp && *(void **)vp->value)
6502           {
6503           uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value);
6504           yield = string_cat(yield, cp);
6505           }
6506         else
6507           {
6508           hctx h;
6509           blob b;
6510           char st[3];
6511
6512           if (!exim_sha_init(&h, HASH_SHA2_256))
6513             {
6514             expand_string_message = US"unrecognised sha256 variant";
6515             goto EXPAND_FAILED;
6516             }
6517           exim_sha_update(&h, sub, Ustrlen(sub));
6518           exim_sha_finish(&h, &b);
6519           while (b.len-- > 0)
6520             {
6521             sprintf(st, "%02X", *b.data++);
6522             yield = string_catn(yield, US st, 2);
6523             }
6524           }
6525 #else
6526           expand_string_message = US"sha256 only supported with TLS";
6527 #endif
6528         continue;
6529
6530       case EOP_SHA3:
6531 #ifdef EXIM_HAVE_SHA3
6532         {
6533         hctx h;
6534         blob b;
6535         char st[3];
6536         hashmethod m = !arg ? HASH_SHA3_256
6537           : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224
6538           : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256
6539           : Ustrcmp(arg, "384") == 0 ? HASH_SHA3_384
6540           : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
6541           : HASH_BADTYPE;
6542
6543         if (m == HASH_BADTYPE || !exim_sha_init(&h, m))
6544           {
6545           expand_string_message = US"unrecognised sha3 variant";
6546           goto EXPAND_FAILED;
6547           }
6548
6549         exim_sha_update(&h, sub, Ustrlen(sub));
6550         exim_sha_finish(&h, &b);
6551         while (b.len-- > 0)
6552           {
6553           sprintf(st, "%02X", *b.data++);
6554           yield = string_catn(yield, US st, 2);
6555           }
6556         }
6557         continue;
6558 #else
6559         expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 + or OpenSSL 1.1.1 +";
6560         goto EXPAND_FAILED;
6561 #endif
6562
6563       /* Convert hex encoding to base64 encoding */
6564
6565       case EOP_HEX2B64:
6566         {
6567         int c = 0;
6568         int b = -1;
6569         uschar *in = sub;
6570         uschar *out = sub;
6571         uschar *enc;
6572
6573         for (enc = sub; *enc != 0; enc++)
6574           {
6575           if (!isxdigit(*enc))
6576             {
6577             expand_string_message = string_sprintf("\"%s\" is not a hex "
6578               "string", sub);
6579             goto EXPAND_FAILED;
6580             }
6581           c++;
6582           }
6583
6584         if ((c & 1) != 0)
6585           {
6586           expand_string_message = string_sprintf("\"%s\" contains an odd "
6587             "number of characters", sub);
6588           goto EXPAND_FAILED;
6589           }
6590
6591         while ((c = *in++) != 0)
6592           {
6593           if (isdigit(c)) c -= '0';
6594           else c = toupper(c) - 'A' + 10;
6595           if (b == -1)
6596             {
6597             b = c << 4;
6598             }
6599           else
6600             {
6601             *out++ = b | c;
6602             b = -1;
6603             }
6604           }
6605
6606         enc = b64encode(sub, out - sub);
6607         yield = string_cat(yield, enc);
6608         continue;
6609         }
6610
6611       /* Convert octets outside 0x21..0x7E to \xXX form */
6612
6613       case EOP_HEXQUOTE:
6614         {
6615         uschar *t = sub - 1;
6616         while (*(++t) != 0)
6617           {
6618           if (*t < 0x21 || 0x7E < *t)
6619             yield = string_catn(yield, string_sprintf("\\x%02x", *t), 4);
6620           else
6621             yield = string_catn(yield, t, 1);
6622           }
6623         continue;
6624         }
6625
6626       /* count the number of list elements */
6627
6628       case EOP_LISTCOUNT:
6629         {
6630         int cnt = 0;
6631         int sep = 0;
6632         uschar * cp;
6633         uschar buffer[256];
6634
6635         while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
6636         cp = string_sprintf("%d", cnt);
6637         yield = string_cat(yield, cp);
6638         continue;
6639         }
6640
6641       /* expand a named list given the name */
6642       /* handles nested named lists; requotes as colon-sep list */
6643
6644       case EOP_LISTNAMED:
6645         {
6646         tree_node *t = NULL;
6647         const uschar * list;
6648         int sep = 0;
6649         uschar * item;
6650         uschar * suffix = US"";
6651         BOOL needsep = FALSE;
6652         uschar buffer[256];
6653
6654         if (*sub == '+') sub++;
6655         if (arg == NULL)        /* no-argument version */
6656           {
6657           if (!(t = tree_search(addresslist_anchor, sub)) &&
6658               !(t = tree_search(domainlist_anchor,  sub)) &&
6659               !(t = tree_search(hostlist_anchor,    sub)))
6660             t = tree_search(localpartlist_anchor, sub);
6661           }
6662         else switch(*arg)       /* specific list-type version */
6663           {
6664           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
6665           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
6666           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
6667           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
6668           default:
6669             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
6670             goto EXPAND_FAILED;
6671           }
6672
6673         if(!t)
6674           {
6675           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
6676             sub, !arg?""
6677               : *arg=='a'?"address "
6678               : *arg=='d'?"domain "
6679               : *arg=='h'?"host "
6680               : *arg=='l'?"localpart "
6681               : 0);
6682           goto EXPAND_FAILED;
6683           }
6684
6685         list = ((namedlist_block *)(t->data.ptr))->string;
6686
6687         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))))
6688           {
6689           uschar * buf = US" : ";
6690           if (needsep)
6691             yield = string_catn(yield, buf, 3);
6692           else
6693             needsep = TRUE;
6694
6695           if (*item == '+')     /* list item is itself a named list */
6696             {
6697             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
6698             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
6699             }
6700           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
6701             {
6702             char * cp;
6703             char tok[3];
6704             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
6705             while ((cp= strpbrk(CCS item, tok)))
6706               {
6707               yield = string_catn(yield, item, cp - CS item);
6708               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
6709                 {
6710                 yield = string_catn(yield, US"::", 2);
6711                 item = US cp;
6712                 }
6713               else              /* sep in item; should already be doubled; emit once */
6714                 {
6715                 yield = string_catn(yield, US tok, 1);
6716                 if (*cp == sep) cp++;
6717                 item = US cp;
6718                 }
6719               }
6720             }
6721           yield = string_cat(yield, item);
6722           }
6723         continue;
6724         }
6725
6726       /* mask applies a mask to an IP address; for example the result of
6727       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
6728
6729       case EOP_MASK:
6730         {
6731         int count;
6732         uschar *endptr;
6733         int binary[4];
6734         int mask, maskoffset;
6735         int type = string_is_ip_address(sub, &maskoffset);
6736         uschar buffer[64];
6737
6738         if (type == 0)
6739           {
6740           expand_string_message = string_sprintf("\"%s\" is not an IP address",
6741            sub);
6742           goto EXPAND_FAILED;
6743           }
6744
6745         if (maskoffset == 0)
6746           {
6747           expand_string_message = string_sprintf("missing mask value in \"%s\"",
6748             sub);
6749           goto EXPAND_FAILED;
6750           }
6751
6752         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
6753
6754         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
6755           {
6756           expand_string_message = string_sprintf("mask value too big in \"%s\"",
6757             sub);
6758           goto EXPAND_FAILED;
6759           }
6760
6761         /* Convert the address to binary integer(s) and apply the mask */
6762
6763         sub[maskoffset] = 0;
6764         count = host_aton(sub, binary);
6765         host_mask(count, binary, mask);
6766
6767         /* Convert to masked textual format and add to output. */
6768
6769         yield = string_catn(yield, buffer,
6770           host_nmtoa(count, binary, mask, buffer, '.'));
6771         continue;
6772         }
6773
6774       case EOP_IPV6NORM:
6775       case EOP_IPV6DENORM:
6776         {
6777         int type = string_is_ip_address(sub, NULL);
6778         int binary[4];
6779         uschar buffer[44];
6780
6781         switch (type)
6782           {
6783           case 6:
6784             (void) host_aton(sub, binary);
6785             break;
6786
6787           case 4:       /* convert to IPv4-mapped IPv6 */
6788             binary[0] = binary[1] = 0;
6789             binary[2] = 0x0000ffff;
6790             (void) host_aton(sub, binary+3);
6791             break;
6792
6793           case 0:
6794             expand_string_message =
6795               string_sprintf("\"%s\" is not an IP address", sub);
6796             goto EXPAND_FAILED;
6797           }
6798
6799         yield = string_catn(yield, buffer, c == EOP_IPV6NORM
6800                     ? ipv6_nmtoa(binary, buffer)
6801                     : host_nmtoa(4, binary, -1, buffer, ':')
6802                   );
6803         continue;
6804         }
6805
6806       case EOP_ADDRESS:
6807       case EOP_LOCAL_PART:
6808       case EOP_DOMAIN:
6809         {
6810         uschar * error;
6811         int start, end, domain;
6812         uschar * t = parse_extract_address(sub, &error, &start, &end, &domain,
6813           FALSE);
6814         if (t)
6815           if (c != EOP_DOMAIN)
6816             {
6817             if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1;
6818             yield = string_catn(yield, sub+start, end-start);
6819             }
6820           else if (domain != 0)
6821             {
6822             domain += start;
6823             yield = string_catn(yield, sub+domain, end-domain);
6824             }
6825         continue;
6826         }
6827
6828       case EOP_ADDRESSES:
6829         {
6830         uschar outsep[2] = { ':', '\0' };
6831         uschar *address, *error;
6832         int save_ptr = yield->ptr;
6833         int start, end, domain;  /* Not really used */
6834
6835         while (isspace(*sub)) sub++;
6836         if (*sub == '>')
6837           if (*outsep = *++sub) ++sub;
6838           else
6839             {
6840             expand_string_message = string_sprintf("output separator "
6841               "missing in expanding ${addresses:%s}", --sub);
6842             goto EXPAND_FAILED;
6843             }
6844         parse_allow_group = TRUE;
6845
6846         for (;;)
6847           {
6848           uschar *p = parse_find_address_end(sub, FALSE);
6849           uschar saveend = *p;
6850           *p = '\0';
6851           address = parse_extract_address(sub, &error, &start, &end, &domain,
6852             FALSE);
6853           *p = saveend;
6854
6855           /* Add the address to the output list that we are building. This is
6856           done in chunks by searching for the separator character. At the
6857           start, unless we are dealing with the first address of the output
6858           list, add in a space if the new address begins with the separator
6859           character, or is an empty string. */
6860
6861           if (address != NULL)
6862             {
6863             if (yield->ptr != save_ptr && address[0] == *outsep)
6864               yield = string_catn(yield, US" ", 1);
6865
6866             for (;;)
6867               {
6868               size_t seglen = Ustrcspn(address, outsep);
6869               yield = string_catn(yield, address, seglen + 1);
6870
6871               /* If we got to the end of the string we output one character
6872               too many. */
6873
6874               if (address[seglen] == '\0') { yield->ptr--; break; }
6875               yield = string_catn(yield, outsep, 1);
6876               address += seglen + 1;
6877               }
6878
6879             /* Output a separator after the string: we will remove the
6880             redundant final one at the end. */
6881
6882             yield = string_catn(yield, outsep, 1);
6883             }
6884
6885           if (saveend == '\0') break;
6886           sub = p + 1;
6887           }
6888
6889         /* If we have generated anything, remove the redundant final
6890         separator. */
6891
6892         if (yield->ptr != save_ptr) yield->ptr--;
6893         parse_allow_group = FALSE;
6894         continue;
6895         }
6896
6897
6898       /* quote puts a string in quotes if it is empty or contains anything
6899       other than alphamerics, underscore, dot, or hyphen.
6900
6901       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
6902       be quoted in order to be a valid local part.
6903
6904       In both cases, newlines and carriage returns are converted into \n and \r
6905       respectively */
6906
6907       case EOP_QUOTE:
6908       case EOP_QUOTE_LOCAL_PART:
6909       if (arg == NULL)
6910         {
6911         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
6912         uschar *t = sub - 1;
6913
6914         if (c == EOP_QUOTE)
6915           {
6916           while (!needs_quote && *(++t) != 0)
6917             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
6918           }
6919         else  /* EOP_QUOTE_LOCAL_PART */
6920           {
6921           while (!needs_quote && *(++t) != 0)
6922             needs_quote = !isalnum(*t) &&
6923               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
6924               (*t != '.' || t == sub || t[1] == 0);
6925           }
6926
6927         if (needs_quote)
6928           {
6929           yield = string_catn(yield, US"\"", 1);
6930           t = sub - 1;
6931           while (*(++t) != 0)
6932             {
6933             if (*t == '\n')
6934               yield = string_catn(yield, US"\\n", 2);
6935             else if (*t == '\r')
6936               yield = string_catn(yield, US"\\r", 2);
6937             else
6938               {
6939               if (*t == '\\' || *t == '"')
6940                 yield = string_catn(yield, US"\\", 1);
6941               yield = string_catn(yield, t, 1);
6942               }
6943             }
6944           yield = string_catn(yield, US"\"", 1);
6945           }
6946         else yield = string_cat(yield, sub);
6947         continue;
6948         }
6949
6950       /* quote_lookuptype does lookup-specific quoting */
6951
6952       else
6953         {
6954         int n;
6955         uschar *opt = Ustrchr(arg, '_');
6956
6957         if (opt != NULL) *opt++ = 0;
6958
6959         n = search_findtype(arg, Ustrlen(arg));
6960         if (n < 0)
6961           {
6962           expand_string_message = search_error_message;
6963           goto EXPAND_FAILED;
6964           }
6965
6966         if (lookup_list[n]->quote != NULL)
6967           sub = (lookup_list[n]->quote)(sub, opt);
6968         else if (opt != NULL) sub = NULL;
6969
6970         if (sub == NULL)
6971           {
6972           expand_string_message = string_sprintf(
6973             "\"%s\" unrecognized after \"${quote_%s\"",
6974             opt, arg);
6975           goto EXPAND_FAILED;
6976           }
6977
6978         yield = string_cat(yield, sub);
6979         continue;
6980         }
6981
6982       /* rx quote sticks in \ before any non-alphameric character so that
6983       the insertion works in a regular expression. */
6984
6985       case EOP_RXQUOTE:
6986         {
6987         uschar *t = sub - 1;
6988         while (*(++t) != 0)
6989           {
6990           if (!isalnum(*t))
6991             yield = string_catn(yield, US"\\", 1);
6992           yield = string_catn(yield, t, 1);
6993           }
6994         continue;
6995         }
6996
6997       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
6998       prescribed by the RFC, if there are characters that need to be encoded */
6999
7000       case EOP_RFC2047:
7001         {
7002         uschar buffer[2048];
7003         const uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset,
7004           buffer, sizeof(buffer), FALSE);
7005         yield = string_cat(yield, string);
7006         continue;
7007         }
7008
7009       /* RFC 2047 decode */
7010
7011       case EOP_RFC2047D:
7012         {
7013         int len;
7014         uschar *error;
7015         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
7016           headers_charset, '?', &len, &error);
7017         if (error != NULL)
7018           {
7019           expand_string_message = error;
7020           goto EXPAND_FAILED;
7021           }
7022         yield = string_catn(yield, decoded, len);
7023         continue;
7024         }
7025
7026       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
7027       underscores */
7028
7029       case EOP_FROM_UTF8:
7030         {
7031         while (*sub != 0)
7032           {
7033           int c;
7034           uschar buff[4];
7035           GETUTF8INC(c, sub);
7036           if (c > 255) c = '_';
7037           buff[0] = c;
7038           yield = string_catn(yield, buff, 1);
7039           }
7040         continue;
7041         }
7042
7043           /* replace illegal UTF-8 sequences by replacement character  */
7044
7045       #define UTF8_REPLACEMENT_CHAR US"?"
7046
7047       case EOP_UTF8CLEAN:
7048         {
7049         int seq_len = 0, index = 0;
7050         int bytes_left = 0;
7051         long codepoint = -1;
7052         uschar seq_buff[4];                     /* accumulate utf-8 here */
7053
7054         while (*sub != 0)
7055           {
7056           int complete = 0;
7057           uschar c = *sub++;
7058
7059           if (bytes_left)
7060             {
7061             if ((c & 0xc0) != 0x80)
7062                     /* wrong continuation byte; invalidate all bytes */
7063               complete = 1; /* error */
7064             else
7065               {
7066               codepoint = (codepoint << 6) | (c & 0x3f);
7067               seq_buff[index++] = c;
7068               if (--bytes_left == 0)            /* codepoint complete */
7069                 if(codepoint > 0x10FFFF)        /* is it too large? */
7070                   complete = -1;        /* error (RFC3629 limit) */
7071                 else
7072                   {             /* finished; output utf-8 sequence */
7073                   yield = string_catn(yield, seq_buff, seq_len);
7074                   index = 0;
7075                   }
7076               }
7077             }
7078           else  /* no bytes left: new sequence */
7079             {
7080             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
7081               {
7082               yield = string_catn(yield, &c, 1);
7083               continue;
7084               }
7085             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
7086               {
7087               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
7088                 complete = -1;
7089               else
7090                 {
7091                   bytes_left = 1;
7092                   codepoint = c & 0x1f;
7093                 }
7094               }
7095             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
7096               {
7097               bytes_left = 2;
7098               codepoint = c & 0x0f;
7099               }
7100             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
7101               {
7102               bytes_left = 3;
7103               codepoint = c & 0x07;
7104               }
7105             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
7106               complete = -1;
7107
7108             seq_buff[index++] = c;
7109             seq_len = bytes_left + 1;
7110             }           /* if(bytes_left) */
7111
7112           if (complete != 0)
7113             {
7114             bytes_left = index = 0;
7115             yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1);
7116             }
7117           if ((complete == 1) && ((c & 0x80) == 0))
7118                         /* ASCII character follows incomplete sequence */
7119               yield = string_catn(yield, &c, 1);
7120           }
7121         continue;
7122         }
7123
7124 #ifdef SUPPORT_I18N
7125       case EOP_UTF8_DOMAIN_TO_ALABEL:
7126         {
7127         uschar * error = NULL;
7128         uschar * s = string_domain_utf8_to_alabel(sub, &error);
7129         if (error)
7130           {
7131           expand_string_message = string_sprintf(
7132             "error converting utf8 (%s) to alabel: %s",
7133             string_printing(sub), error);
7134           goto EXPAND_FAILED;
7135           }
7136         yield = string_cat(yield, s);
7137         continue;
7138         }
7139
7140       case EOP_UTF8_DOMAIN_FROM_ALABEL:
7141         {
7142         uschar * error = NULL;
7143         uschar * s = string_domain_alabel_to_utf8(sub, &error);
7144         if (error)
7145           {
7146           expand_string_message = string_sprintf(
7147             "error converting alabel (%s) to utf8: %s",
7148             string_printing(sub), error);
7149           goto EXPAND_FAILED;
7150           }
7151         yield = string_cat(yield, s);
7152         continue;
7153         }
7154
7155       case EOP_UTF8_LOCALPART_TO_ALABEL:
7156         {
7157         uschar * error = NULL;
7158         uschar * s = string_localpart_utf8_to_alabel(sub, &error);
7159         if (error)
7160           {
7161           expand_string_message = string_sprintf(
7162             "error converting utf8 (%s) to alabel: %s",
7163             string_printing(sub), error);
7164           goto EXPAND_FAILED;
7165           }
7166         yield = string_cat(yield, s);
7167         DEBUG(D_expand) debug_printf_indent("yield: '%s'\n", yield->s);
7168         continue;
7169         }
7170
7171       case EOP_UTF8_LOCALPART_FROM_ALABEL:
7172         {
7173         uschar * error = NULL;
7174         uschar * s = string_localpart_alabel_to_utf8(sub, &error);
7175         if (error)
7176           {
7177           expand_string_message = string_sprintf(
7178             "error converting alabel (%s) to utf8: %s",
7179             string_printing(sub), error);
7180           goto EXPAND_FAILED;
7181           }
7182         yield = string_cat(yield, s);
7183         continue;
7184         }
7185 #endif  /* EXPERIMENTAL_INTERNATIONAL */
7186
7187       /* escape turns all non-printing characters into escape sequences. */
7188
7189       case EOP_ESCAPE:
7190         {
7191         const uschar * t = string_printing(sub);
7192         yield = string_cat(yield, t);
7193         continue;
7194         }
7195
7196       case EOP_ESCAPE8BIT:
7197         {
7198         const uschar * s = sub;
7199         uschar c;
7200
7201         for (s = sub; (c = *s); s++)
7202           yield = c < 127 && c != '\\'
7203             ? string_catn(yield, s, 1)
7204             : string_catn(yield, string_sprintf("\\%03o", c), 4);
7205         continue;
7206         }
7207
7208       /* Handle numeric expression evaluation */
7209
7210       case EOP_EVAL:
7211       case EOP_EVAL10:
7212         {
7213         uschar *save_sub = sub;
7214         uschar *error = NULL;
7215         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
7216         if (error != NULL)
7217           {
7218           expand_string_message = string_sprintf("error in expression "
7219             "evaluation: %s (after processing \"%.*s\")", error,
7220             (int)(sub-save_sub), save_sub);
7221           goto EXPAND_FAILED;
7222           }
7223         sprintf(CS var_buffer, PR_EXIM_ARITH, n);
7224         yield = string_cat(yield, var_buffer);
7225         continue;
7226         }
7227
7228       /* Handle time period formating */
7229
7230       case EOP_TIME_EVAL:
7231         {
7232         int n = readconf_readtime(sub, 0, FALSE);
7233         if (n < 0)
7234           {
7235           expand_string_message = string_sprintf("string \"%s\" is not an "
7236             "Exim time interval in \"%s\" operator", sub, name);
7237           goto EXPAND_FAILED;
7238           }
7239         sprintf(CS var_buffer, "%d", n);
7240         yield = string_cat(yield, var_buffer);
7241         continue;
7242         }
7243
7244       case EOP_TIME_INTERVAL:
7245         {
7246         int n;
7247         uschar *t = read_number(&n, sub);
7248         if (*t != 0) /* Not A Number*/
7249           {
7250           expand_string_message = string_sprintf("string \"%s\" is not a "
7251             "positive number in \"%s\" operator", sub, name);
7252           goto EXPAND_FAILED;
7253           }
7254         t = readconf_printtime(n);
7255         yield = string_cat(yield, t);
7256         continue;
7257         }
7258
7259       /* Convert string to base64 encoding */
7260
7261       case EOP_STR2B64:
7262       case EOP_BASE64:
7263         {
7264 #ifdef SUPPORT_TLS
7265         uschar * s = vp && *(void **)vp->value
7266           ? tls_cert_der_b64(*(void **)vp->value)
7267           : b64encode(sub, Ustrlen(sub));
7268 #else
7269         uschar * s = b64encode(sub, Ustrlen(sub));
7270 #endif
7271         yield = string_cat(yield, s);
7272         continue;
7273         }
7274
7275       case EOP_BASE64D:
7276         {
7277         uschar * s;
7278         int len = b64decode(sub, &s);
7279         if (len < 0)
7280           {
7281           expand_string_message = string_sprintf("string \"%s\" is not "
7282             "well-formed for \"%s\" operator", sub, name);
7283           goto EXPAND_FAILED;
7284           }
7285         yield = string_cat(yield, s);
7286         continue;
7287         }
7288
7289       /* strlen returns the length of the string */
7290
7291       case EOP_STRLEN:
7292         {
7293         uschar buff[24];
7294         (void)sprintf(CS buff, "%d", Ustrlen(sub));
7295         yield = string_cat(yield, buff);
7296         continue;
7297         }
7298
7299       /* length_n or l_n takes just the first n characters or the whole string,
7300       whichever is the shorter;
7301
7302       substr_m_n, and s_m_n take n characters from offset m; negative m take
7303       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
7304       takes the rest, either to the right or to the left.
7305
7306       hash_n or h_n makes a hash of length n from the string, yielding n
7307       characters from the set a-z; hash_n_m makes a hash of length n, but
7308       uses m characters from the set a-zA-Z0-9.
7309
7310       nhash_n returns a single number between 0 and n-1 (in text form), while
7311       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
7312       between 0 and n-1 and the second between 0 and m-1. */
7313
7314       case EOP_LENGTH:
7315       case EOP_L:
7316       case EOP_SUBSTR:
7317       case EOP_S:
7318       case EOP_HASH:
7319       case EOP_H:
7320       case EOP_NHASH:
7321       case EOP_NH:
7322         {
7323         int sign = 1;
7324         int value1 = 0;
7325         int value2 = -1;
7326         int *pn;
7327         int len;
7328         uschar *ret;
7329
7330         if (arg == NULL)
7331           {
7332           expand_string_message = string_sprintf("missing values after %s",
7333             name);
7334           goto EXPAND_FAILED;
7335           }
7336
7337         /* "length" has only one argument, effectively being synonymous with
7338         substr_0_n. */
7339
7340         if (c == EOP_LENGTH || c == EOP_L)
7341           {
7342           pn = &value2;
7343           value2 = 0;
7344           }
7345
7346         /* The others have one or two arguments; for "substr" the first may be
7347         negative. The second being negative means "not supplied". */
7348
7349         else
7350           {
7351           pn = &value1;
7352           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
7353           }
7354
7355         /* Read up to two numbers, separated by underscores */
7356
7357         ret = arg;
7358         while (*arg != 0)
7359           {
7360           if (arg != ret && *arg == '_' && pn == &value1)
7361             {
7362             pn = &value2;
7363             value2 = 0;
7364             if (arg[1] != 0) arg++;
7365             }
7366           else if (!isdigit(*arg))
7367             {
7368             expand_string_message =
7369               string_sprintf("non-digit after underscore in \"%s\"", name);
7370             goto EXPAND_FAILED;
7371             }
7372           else *pn = (*pn)*10 + *arg++ - '0';
7373           }
7374         value1 *= sign;
7375
7376         /* Perform the required operation */
7377
7378         ret =
7379           (c == EOP_HASH || c == EOP_H)?
7380              compute_hash(sub, value1, value2, &len) :
7381           (c == EOP_NHASH || c == EOP_NH)?
7382              compute_nhash(sub, value1, value2, &len) :
7383              extract_substr(sub, value1, value2, &len);
7384
7385         if (ret == NULL) goto EXPAND_FAILED;
7386         yield = string_catn(yield, ret, len);
7387         continue;
7388         }
7389
7390       /* Stat a path */
7391
7392       case EOP_STAT:
7393         {
7394         uschar *s;
7395         uschar smode[12];
7396         uschar **modetable[3];
7397         int i;
7398         mode_t mode;
7399         struct stat st;
7400
7401         if ((expand_forbid & RDO_EXISTS) != 0)
7402           {
7403           expand_string_message = US"Use of the stat() expansion is not permitted";
7404           goto EXPAND_FAILED;
7405           }
7406
7407         if (stat(CS sub, &st) < 0)
7408           {
7409           expand_string_message = string_sprintf("stat(%s) failed: %s",
7410             sub, strerror(errno));
7411           goto EXPAND_FAILED;
7412           }
7413         mode = st.st_mode;
7414         switch (mode & S_IFMT)
7415           {
7416           case S_IFIFO: smode[0] = 'p'; break;
7417           case S_IFCHR: smode[0] = 'c'; break;
7418           case S_IFDIR: smode[0] = 'd'; break;
7419           case S_IFBLK: smode[0] = 'b'; break;
7420           case S_IFREG: smode[0] = '-'; break;
7421           default: smode[0] = '?'; break;
7422           }
7423
7424         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
7425         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
7426         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
7427
7428         for (i = 0; i < 3; i++)
7429           {
7430           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
7431           mode >>= 3;
7432           }
7433
7434         smode[10] = 0;
7435         s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
7436           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
7437           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
7438           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
7439           (long)st.st_gid, st.st_size, (long)st.st_atime,
7440           (long)st.st_mtime, (long)st.st_ctime);
7441         yield = string_cat(yield, s);
7442         continue;
7443         }
7444
7445       /* vaguely random number less than N */
7446
7447       case EOP_RANDINT:
7448         {
7449         int_eximarith_t max;
7450         uschar *s;
7451
7452         max = expanded_string_integer(sub, TRUE);
7453         if (expand_string_message != NULL)
7454           goto EXPAND_FAILED;
7455         s = string_sprintf("%d", vaguely_random_number((int)max));
7456         yield = string_cat(yield, s);
7457         continue;
7458         }
7459
7460       /* Reverse IP, including IPv6 to dotted-nibble */
7461
7462       case EOP_REVERSE_IP:
7463         {
7464         int family, maskptr;
7465         uschar reversed[128];
7466
7467         family = string_is_ip_address(sub, &maskptr);
7468         if (family == 0)
7469           {
7470           expand_string_message = string_sprintf(
7471               "reverse_ip() not given an IP address [%s]", sub);
7472           goto EXPAND_FAILED;
7473           }
7474         invert_address(reversed, sub);
7475         yield = string_cat(yield, reversed);
7476         continue;
7477         }
7478
7479       /* Unknown operator */
7480
7481       default:
7482       expand_string_message =
7483         string_sprintf("unknown expansion operator \"%s\"", name);
7484       goto EXPAND_FAILED;
7485       }
7486     }
7487
7488   /* Handle a plain name. If this is the first thing in the expansion, release
7489   the pre-allocated buffer. If the result data is known to be in a new buffer,
7490   newsize will be set to the size of that buffer, and we can just point at that
7491   store instead of copying. Many expansion strings contain just one reference,
7492   so this is a useful optimization, especially for humungous headers
7493   ($message_headers). */
7494                                                 /*{*/
7495   if (*s++ == '}')
7496     {
7497     int len;
7498     int newsize = 0;
7499     gstring * g = NULL;
7500
7501     if (!yield)
7502       g = store_get(sizeof(gstring));
7503     else if (yield->ptr == 0)
7504       {
7505       if (resetok) store_reset(yield);
7506       yield = NULL;
7507       g = store_get(sizeof(gstring));   /* alloc _before_ calling find_variable() */
7508       }
7509     if (!(value = find_variable(name, FALSE, skipping, &newsize)))
7510       {
7511       expand_string_message =
7512         string_sprintf("unknown variable in \"${%s}\"", name);
7513       check_variable_error_message(name);
7514       goto EXPAND_FAILED;
7515       }
7516     len = Ustrlen(value);
7517     if (!yield && newsize)
7518       {
7519       yield = g;
7520       yield->size = newsize;
7521       yield->ptr = len;
7522       yield->s = value;
7523       }
7524     else
7525       yield = string_catn(yield, value, len);
7526     continue;
7527     }
7528
7529   /* Else there's something wrong */
7530
7531   expand_string_message =
7532     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
7533     "in a variable reference)", name);
7534   goto EXPAND_FAILED;
7535   }
7536
7537 /* If we hit the end of the string when ket_ends is set, there is a missing
7538 terminating brace. */
7539
7540 if (ket_ends && *s == 0)
7541   {
7542   expand_string_message = malformed_header
7543     ? US"missing } at end of string - could be header name not terminated by colon"
7544     : US"missing } at end of string";
7545   goto EXPAND_FAILED;
7546   }
7547
7548 /* Expansion succeeded; yield may still be NULL here if nothing was actually
7549 added to the string. If so, set up an empty string. Add a terminating zero. If
7550 left != NULL, return a pointer to the terminator. */
7551
7552 if (!yield)
7553   yield = string_get(1);
7554 (void) string_from_gstring(yield);
7555 if (left) *left = s;
7556
7557 /* Any stacking store that was used above the final string is no longer needed.
7558 In many cases the final string will be the first one that was got and so there
7559 will be optimal store usage. */
7560
7561 if (resetok) store_reset(yield->s + (yield->size = yield->ptr + 1));
7562 else if (resetok_p) *resetok_p = FALSE;
7563
7564 DEBUG(D_expand)
7565   {
7566   debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
7567     "expanding: %.*s\n",
7568     (int)(s - string), string);
7569   debug_printf_indent("%s"
7570     UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7571     "result: %s\n",
7572     skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7573     yield->s);
7574   if (skipping)
7575     debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7576       "skipping: result is not used\n");
7577   }
7578 expand_level--;
7579 return yield->s;
7580
7581 /* This is the failure exit: easiest to program with a goto. We still need
7582 to update the pointer to the terminator, for cases of nested calls with "fail".
7583 */
7584
7585 EXPAND_FAILED_CURLY:
7586 if (malformed_header)
7587   expand_string_message =
7588     US"missing or misplaced { or } - could be header name not terminated by colon";
7589
7590 else if (!expand_string_message || !*expand_string_message)
7591   expand_string_message = US"missing or misplaced { or }";
7592
7593 /* At one point, Exim reset the store to yield (if yield was not NULL), but
7594 that is a bad idea, because expand_string_message is in dynamic store. */
7595
7596 EXPAND_FAILED:
7597 if (left) *left = s;
7598 DEBUG(D_expand)
7599   {
7600   debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n",
7601     string);
7602   debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7603     "error message: %s\n",
7604     expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7605     expand_string_message);
7606   if (expand_string_forcedfail)
7607     debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n");
7608   }
7609 if (resetok_p && !resetok) *resetok_p = FALSE;
7610 expand_level--;
7611 return NULL;
7612 }
7613
7614
7615 /* This is the external function call. Do a quick check for any expansion
7616 metacharacters, and if there are none, just return the input string.
7617
7618 Argument: the string to be expanded
7619 Returns:  the expanded string, or NULL if expansion failed; if failure was
7620           due to a lookup deferring, search_find_defer will be TRUE
7621 */
7622
7623 const uschar *
7624 expand_cstring(const uschar * string)
7625 {
7626 if (Ustrpbrk(string, "$\\") != NULL)
7627   {
7628   int old_pool = store_pool;
7629   uschar * s;
7630
7631   search_find_defer = FALSE;
7632   malformed_header = FALSE;
7633   store_pool = POOL_MAIN;
7634     s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7635   store_pool = old_pool;
7636   return s;
7637   }
7638 return string;
7639 }
7640
7641
7642 uschar *
7643 expand_string(uschar * string)
7644 {
7645 return US expand_cstring(CUS string);
7646 }
7647
7648
7649
7650
7651
7652 /*************************************************
7653 *              Expand and copy                   *
7654 *************************************************/
7655
7656 /* Now and again we want to expand a string and be sure that the result is in a
7657 new bit of store. This function does that.
7658 Since we know it has been copied, the de-const cast is safe.
7659
7660 Argument: the string to be expanded
7661 Returns:  the expanded string, always in a new bit of store, or NULL
7662 */
7663
7664 uschar *
7665 expand_string_copy(const uschar *string)
7666 {
7667 const uschar *yield = expand_cstring(string);
7668 if (yield == string) yield = string_copy(string);
7669 return US yield;
7670 }
7671
7672
7673
7674 /*************************************************
7675 *        Expand and interpret as an integer      *
7676 *************************************************/
7677
7678 /* Expand a string, and convert the result into an integer.
7679
7680 Arguments:
7681   string  the string to be expanded
7682   isplus  TRUE if a non-negative number is expected
7683
7684 Returns:  the integer value, or
7685           -1 for an expansion error               ) in both cases, message in
7686           -2 for an integer interpretation error  ) expand_string_message
7687           expand_string_message is set NULL for an OK integer
7688 */
7689
7690 int_eximarith_t
7691 expand_string_integer(uschar *string, BOOL isplus)
7692 {
7693 return expanded_string_integer(expand_string(string), isplus);
7694 }
7695
7696
7697 /*************************************************
7698  *         Interpret string as an integer        *
7699  *************************************************/
7700
7701 /* Convert a string (that has already been expanded) into an integer.
7702
7703 This function is used inside the expansion code.
7704
7705 Arguments:
7706   s       the string to be expanded
7707   isplus  TRUE if a non-negative number is expected
7708
7709 Returns:  the integer value, or
7710           -1 if string is NULL (which implies an expansion error)
7711           -2 for an integer interpretation error
7712           expand_string_message is set NULL for an OK integer
7713 */
7714
7715 static int_eximarith_t
7716 expanded_string_integer(const uschar *s, BOOL isplus)
7717 {
7718 int_eximarith_t value;
7719 uschar *msg = US"invalid integer \"%s\"";
7720 uschar *endptr;
7721
7722 /* If expansion failed, expand_string_message will be set. */
7723
7724 if (s == NULL) return -1;
7725
7726 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
7727 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
7728 systems, so we set it zero ourselves. */
7729
7730 errno = 0;
7731 expand_string_message = NULL;               /* Indicates no error */
7732
7733 /* Before Exim 4.64, strings consisting entirely of whitespace compared
7734 equal to 0.  Unfortunately, people actually relied upon that, so preserve
7735 the behaviour explicitly.  Stripping leading whitespace is a harmless
7736 noop change since strtol skips it anyway (provided that there is a number
7737 to find at all). */
7738 if (isspace(*s))
7739   {
7740   while (isspace(*s)) ++s;
7741   if (*s == '\0')
7742     {
7743       DEBUG(D_expand)
7744        debug_printf_indent("treating blank string as number 0\n");
7745       return 0;
7746     }
7747   }
7748
7749 value = strtoll(CS s, CSS &endptr, 10);
7750
7751 if (endptr == s)
7752   {
7753   msg = US"integer expected but \"%s\" found";
7754   }
7755 else if (value < 0 && isplus)
7756   {
7757   msg = US"non-negative integer expected but \"%s\" found";
7758   }
7759 else
7760   {
7761   switch (tolower(*endptr))
7762     {
7763     default:
7764       break;
7765     case 'k':
7766       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
7767       else value *= 1024;
7768       endptr++;
7769       break;
7770     case 'm':
7771       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
7772       else value *= 1024*1024;
7773       endptr++;
7774       break;
7775     case 'g':
7776       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
7777       else value *= 1024*1024*1024;
7778       endptr++;
7779       break;
7780     }
7781   if (errno == ERANGE)
7782     msg = US"absolute value of integer \"%s\" is too large (overflow)";
7783   else
7784     {
7785     while (isspace(*endptr)) endptr++;
7786     if (*endptr == 0) return value;
7787     }
7788   }
7789
7790 expand_string_message = string_sprintf(CS msg, s);
7791 return -2;
7792 }
7793
7794
7795 /* These values are usually fixed boolean values, but they are permitted to be
7796 expanded strings.
7797
7798 Arguments:
7799   addr       address being routed
7800   mtype      the module type
7801   mname      the module name
7802   dbg_opt    debug selectors
7803   oname      the option name
7804   bvalue     the router's boolean value
7805   svalue     the router's string value
7806   rvalue     where to put the returned value
7807
7808 Returns:     OK     value placed in rvalue
7809              DEFER  expansion failed
7810 */
7811
7812 int
7813 exp_bool(address_item *addr,
7814   uschar *mtype, uschar *mname, unsigned dbg_opt,
7815   uschar *oname, BOOL bvalue,
7816   uschar *svalue, BOOL *rvalue)
7817 {
7818 uschar *expanded;
7819 if (svalue == NULL) { *rvalue = bvalue; return OK; }
7820
7821 expanded = expand_string(svalue);
7822 if (expanded == NULL)
7823   {
7824   if (expand_string_forcedfail)
7825     {
7826     DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname);
7827     *rvalue = bvalue;
7828     return OK;
7829     }
7830   addr->message = string_sprintf("failed to expand \"%s\" in %s %s: %s",
7831       oname, mname, mtype, expand_string_message);
7832   DEBUG(dbg_opt) debug_printf("%s\n", addr->message);
7833   return DEFER;
7834   }
7835
7836 DEBUG(dbg_opt) debug_printf("expansion of \"%s\" yields \"%s\"\n", oname,
7837   expanded);
7838
7839 if (strcmpic(expanded, US"true") == 0 || strcmpic(expanded, US"yes") == 0)
7840   *rvalue = TRUE;
7841 else if (strcmpic(expanded, US"false") == 0 || strcmpic(expanded, US"no") == 0)
7842   *rvalue = FALSE;
7843 else
7844   {
7845   addr->message = string_sprintf("\"%s\" is not a valid value for the "
7846     "\"%s\" option in the %s %s", expanded, oname, mname, mtype);
7847   return DEFER;
7848   }
7849
7850 return OK;
7851 }
7852
7853
7854
7855 /* Avoid potentially exposing a password in a string about to be logged */
7856
7857 uschar *
7858 expand_hide_passwords(uschar * s)
7859 {
7860 return (  (  Ustrstr(s, "failed to expand") != NULL
7861           || Ustrstr(s, "expansion of ")    != NULL
7862           )
7863        && (  Ustrstr(s, "mysql")   != NULL
7864           || Ustrstr(s, "pgsql")   != NULL
7865           || Ustrstr(s, "redis")   != NULL
7866           || Ustrstr(s, "sqlite")  != NULL
7867           || Ustrstr(s, "ldap:")   != NULL
7868           || Ustrstr(s, "ldaps:")  != NULL
7869           || Ustrstr(s, "ldapi:")  != NULL
7870           || Ustrstr(s, "ldapdn:") != NULL
7871           || Ustrstr(s, "ldapm:")  != NULL
7872        )  )
7873   ? US"Temporary internal error" : s;
7874 }
7875
7876
7877 /* Read given named file into big_buffer.  Use for keying material etc.
7878 The content will have an ascii NUL appended.
7879
7880 Arguments:
7881  filename       as it says
7882
7883 Return:  pointer to buffer, or NULL on error.
7884 */
7885
7886 uschar *
7887 expand_file_big_buffer(const uschar * filename)
7888 {
7889 int fd, off = 0, len;
7890
7891 if ((fd = open(CS filename, O_RDONLY)) < 0)
7892   {
7893   log_write(0, LOG_MAIN | LOG_PANIC, "unable to open file for reading: %s",
7894              filename);
7895   return NULL;
7896   }
7897
7898 do
7899   {
7900   if ((len = read(fd, big_buffer + off, big_buffer_size - 2 - off)) < 0)
7901     {
7902     (void) close(fd);
7903     log_write(0, LOG_MAIN|LOG_PANIC, "unable to read file: %s", filename);
7904     return NULL;
7905     }
7906   off += len;
7907   }
7908 while (len > 0);
7909
7910 (void) close(fd);
7911 big_buffer[off] = '\0';
7912 return big_buffer;
7913 }
7914
7915
7916
7917 /*************************************************
7918 * Error-checking for testsuite                   *
7919 *************************************************/
7920 typedef struct {
7921   uschar *      region_start;
7922   uschar *      region_end;
7923   const uschar *var_name;
7924   const uschar *var_data;
7925 } err_ctx;
7926
7927 static void
7928 assert_variable_notin(uschar * var_name, uschar * var_data, void * ctx)
7929 {
7930 err_ctx * e = ctx;
7931 if (var_data >= e->region_start  &&  var_data < e->region_end)
7932   {
7933   e->var_name = CUS var_name;
7934   e->var_data = CUS var_data;
7935   }
7936 }
7937
7938 void
7939 assert_no_variables(void * ptr, int len, const char * filename, int linenumber)
7940 {
7941 err_ctx e = { .region_start = ptr, .region_end = US ptr + len,
7942               .var_name = NULL, .var_data = NULL };
7943 int i;
7944 var_entry * v;
7945
7946 /* check acl_ variables */
7947 tree_walk(acl_var_c, assert_variable_notin, &e);
7948 tree_walk(acl_var_m, assert_variable_notin, &e);
7949
7950 /* check auth<n> variables */
7951 for (i = 0; i < AUTH_VARS; i++) if (auth_vars[i])
7952   assert_variable_notin(US"auth<n>", auth_vars[i], &e);
7953
7954 /* check regex<n> variables */
7955 for (i = 0; i < REGEX_VARS; i++) if (regex_vars[i])
7956   assert_variable_notin(US"regex<n>", regex_vars[i], &e);
7957
7958 /* check known-name variables */
7959 for (v = var_table; v < var_table + var_table_size; v++)
7960   if (v->type == vtype_stringptr)
7961     assert_variable_notin(US v->name, *(USS v->value), &e);
7962
7963 /* check dns and address trees */
7964 tree_walk(tree_dns_fails,     assert_variable_notin, &e);
7965 tree_walk(tree_duplicates,    assert_variable_notin, &e);
7966 tree_walk(tree_nonrecipients, assert_variable_notin, &e);
7967 tree_walk(tree_unusable,      assert_variable_notin, &e);
7968
7969 if (e.var_name)
7970   log_write(0, LOG_MAIN|LOG_PANIC_DIE,
7971     "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'",
7972     e.var_name, filename, linenumber, e.var_data);
7973 }
7974
7975
7976
7977 /*************************************************
7978 **************************************************
7979 *             Stand-alone test program           *
7980 **************************************************
7981 *************************************************/
7982
7983 #ifdef STAND_ALONE
7984
7985
7986 BOOL
7987 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
7988 {
7989 int ovector[3*(EXPAND_MAXN+1)];
7990 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
7991   ovector, nelem(ovector));
7992 BOOL yield = n >= 0;
7993 if (n == 0) n = EXPAND_MAXN + 1;
7994 if (yield)
7995   {
7996   int nn;
7997   expand_nmax = (setup < 0)? 0 : setup + 1;
7998   for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2)
7999     {
8000     expand_nstring[expand_nmax] = subject + ovector[nn];
8001     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
8002     }
8003   expand_nmax--;
8004   }
8005 return yield;
8006 }
8007
8008
8009 int main(int argc, uschar **argv)
8010 {
8011 int i;
8012 uschar buffer[1024];
8013
8014 debug_selector = D_v;
8015 debug_file = stderr;
8016 debug_fd = fileno(debug_file);
8017 big_buffer = malloc(big_buffer_size);
8018
8019 for (i = 1; i < argc; i++)
8020   {
8021   if (argv[i][0] == '+')
8022     {
8023     debug_trace_memory = 2;
8024     argv[i]++;
8025     }
8026   if (isdigit(argv[i][0]))
8027     debug_selector = Ustrtol(argv[i], NULL, 0);
8028   else
8029     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
8030         Ustrlen(argv[i]))
8031       {
8032 #ifdef LOOKUP_LDAP
8033       eldap_default_servers = argv[i];
8034 #endif
8035 #ifdef LOOKUP_MYSQL
8036       mysql_servers = argv[i];
8037 #endif
8038 #ifdef LOOKUP_PGSQL
8039       pgsql_servers = argv[i];
8040 #endif
8041 #ifdef LOOKUP_REDIS
8042       redis_servers = argv[i];
8043 #endif
8044       }
8045 #ifdef EXIM_PERL
8046   else opt_perl_startup = argv[i];
8047 #endif
8048   }
8049
8050 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
8051
8052 expand_nstring[1] = US"string 1....";
8053 expand_nlength[1] = 8;
8054 expand_nmax = 1;
8055
8056 #ifdef EXIM_PERL
8057 if (opt_perl_startup != NULL)
8058   {
8059   uschar *errstr;
8060   printf("Starting Perl interpreter\n");
8061   errstr = init_perl(opt_perl_startup);
8062   if (errstr != NULL)
8063     {
8064     printf("** error in perl_startup code: %s\n", errstr);
8065     return EXIT_FAILURE;
8066     }
8067   }
8068 #endif /* EXIM_PERL */
8069
8070 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
8071   {
8072   void *reset_point = store_get(0);
8073   uschar *yield = expand_string(buffer);
8074   if (yield != NULL)
8075     {
8076     printf("%s\n", yield);
8077     store_reset(reset_point);
8078     }
8079   else
8080     {
8081     if (search_find_defer) printf("search_find deferred\n");
8082     printf("Failed: %s\n", expand_string_message);
8083     if (expand_string_forcedfail) printf("Forced failure\n");
8084     printf("\n");
8085     }
8086   }
8087
8088 search_tidyup();
8089
8090 return 0;
8091 }
8092
8093 #endif
8094
8095 /* vi: aw ai sw=2
8096 */
8097 /* End of expand.c */