src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2018 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOOL, BOOL, BOOL *);
  17 static int_eximarith_t expanded_string_integer(const uschar *, BOOL);
  18
  19 #ifdef STAND_ALONE
  20 # ifndef SUPPORT_CRYPTEQ
  21 #  define SUPPORT_CRYPTEQ
  22 # endif
  23 #endif
  24
  25 #ifdef LOOKUP_LDAP
  26 # include "lookups/ldap.h"
  27 #endif
  28
  29 #ifdef SUPPORT_CRYPTEQ
  30 # ifdef CRYPT_H
  31 #  include <crypt.h>
  32 # endif
  33 # ifndef HAVE_CRYPT16
  34 extern char* crypt16(char*, char*);
  35 # endif
  36 #endif
  37
  38 /* The handling of crypt16() is a mess. I will record below the analysis of the
  39 mess that was sent to me. We decided, however, to make changing this very low
  40 priority, because in practice people are moving away from the crypt()
  41 algorithms nowadays, so it doesn't seem worth it.
  42
  43 <quote>
  44 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  45 the first 8 characters of the password using a 20-round version of crypt
  46 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  47 or an empty block if the password is less than 9 characters, using a
  48 20-round version of crypt and the same salt as was used for the first
  49 block.  Characters after the first 16 are ignored.  It always generates
  50 a 16-byte hash, which is expressed together with the salt as a string
  51 of 24 base 64 digits.  Here are some links to peruse:
  52
  53         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  54         http://seclists.org/bugtraq/1999/Mar/0076.html
  55
  56 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  57 and OSF/1.  This is the same as the standard crypt if given a password
  58 of 8 characters or less.  If given more, it first does the same as crypt
  59 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  60 using as salt the first two base 64 digits from the first hash block.
  61 If the password is more than 16 characters then it crypts the 17th to 24th
  62 characters using as salt the first two base 64 digits from the second hash
  63 block.  And so on: I've seen references to it cutting off the password at
  64 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  65
  66         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  67         http://seclists.org/bugtraq/1999/Mar/0109.html
  68         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  69              TET1_html/sec.c222.html#no_id_208
  70
  71 Exim has something it calls "crypt16".  It will either use a native
  72 crypt16 or its own implementation.  A native crypt16 will presumably
  73 be the one that I called "crypt16" above.  The internal "crypt16"
  74 function, however, is a two-block-maximum implementation of what I called
  75 "bigcrypt".  The documentation matches the internal code.
  76
  77 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  78 that crypt16 and bigcrypt were different things.
  79
  80 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  81 to whatever it is using under that name.  This unfortunately sets a
  82 precedent for using "{crypt16}" to identify two incompatible algorithms
  83 whose output can't be distinguished.  With "{crypt16}" thus rendered
  84 ambiguous, I suggest you deprecate it and invent two new identifiers
  85 for the two algorithms.
  86
  87 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  88 of the password separately means they can be cracked separately, so
  89 the double-length hash only doubles the cracking effort instead of
  90 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  91 bcrypt ({CRYPT}$2a$).
  92 </quote>
  93 */
  94
  95
  96
  97 /*************************************************
  98 *            Local statics and tables            *
  99 *************************************************/
 100
 101 /* Table of item names, and corresponding switch numbers. The names must be in
 102 alphabetical order. */
 103
 104 static uschar *item_table[] = {
 105   US"acl",
 106   US"authresults",
 107   US"certextract",
 108   US"dlfunc",
 109   US"env",
 110   US"extract",
 111   US"filter",
 112   US"hash",
 113   US"hmac",
 114   US"if",
 115 #ifdef SUPPORT_I18N
 116   US"imapfolder",
 117 #endif
 118   US"length",
 119   US"listextract",
 120   US"lookup",
 121   US"map",
 122   US"nhash",
 123   US"perl",
 124   US"prvs",
 125   US"prvscheck",
 126   US"readfile",
 127   US"readsocket",
 128   US"reduce",
 129   US"run",
 130   US"sg",
 131   US"sort",
 132   US"substr",
 133   US"tr" };
 134
 135 enum {
 136   EITEM_ACL,
 137   EITEM_AUTHRESULTS,
 138   EITEM_CERTEXTRACT,
 139   EITEM_DLFUNC,
 140   EITEM_ENV,
 141   EITEM_EXTRACT,
 142   EITEM_FILTER,
 143   EITEM_HASH,
 144   EITEM_HMAC,
 145   EITEM_IF,
 146 #ifdef SUPPORT_I18N
 147   EITEM_IMAPFOLDER,
 148 #endif
 149   EITEM_LENGTH,
 150   EITEM_LISTEXTRACT,
 151   EITEM_LOOKUP,
 152   EITEM_MAP,
 153   EITEM_NHASH,
 154   EITEM_PERL,
 155   EITEM_PRVS,
 156   EITEM_PRVSCHECK,
 157   EITEM_READFILE,
 158   EITEM_READSOCK,
 159   EITEM_REDUCE,
 160   EITEM_RUN,
 161   EITEM_SG,
 162   EITEM_SORT,
 163   EITEM_SUBSTR,
 164   EITEM_TR };
 165
 166 /* Tables of operator names, and corresponding switch numbers. The names must be
 167 in alphabetical order. There are two tables, because underscore is used in some
 168 cases to introduce arguments, whereas for other it is part of the name. This is
 169 an historical mis-design. */
 170
 171 static uschar *op_table_underscore[] = {
 172   US"from_utf8",
 173   US"local_part",
 174   US"quote_local_part",
 175   US"reverse_ip",
 176   US"time_eval",
 177   US"time_interval"
 178 #ifdef SUPPORT_I18N
 179  ,US"utf8_domain_from_alabel",
 180   US"utf8_domain_to_alabel",
 181   US"utf8_localpart_from_alabel",
 182   US"utf8_localpart_to_alabel"
 183 #endif
 184   };
 185
 186 enum {
 187   EOP_FROM_UTF8,
 188   EOP_LOCAL_PART,
 189   EOP_QUOTE_LOCAL_PART,
 190   EOP_REVERSE_IP,
 191   EOP_TIME_EVAL,
 192   EOP_TIME_INTERVAL
 193 #ifdef SUPPORT_I18N
 194  ,EOP_UTF8_DOMAIN_FROM_ALABEL,
 195   EOP_UTF8_DOMAIN_TO_ALABEL,
 196   EOP_UTF8_LOCALPART_FROM_ALABEL,
 197   EOP_UTF8_LOCALPART_TO_ALABEL
 198 #endif
 199   };
 200
 201 static uschar *op_table_main[] = {
 202   US"address",
 203   US"addresses",
 204   US"base32",
 205   US"base32d",
 206   US"base62",
 207   US"base62d",
 208   US"base64",
 209   US"base64d",
 210   US"domain",
 211   US"escape",
 212   US"escape8bit",
 213   US"eval",
 214   US"eval10",
 215   US"expand",
 216   US"h",
 217   US"hash",
 218   US"hex2b64",
 219   US"hexquote",
 220   US"ipv6denorm",
 221   US"ipv6norm",
 222   US"l",
 223   US"lc",
 224   US"length",
 225   US"listcount",
 226   US"listnamed",
 227   US"mask",
 228   US"md5",
 229   US"nh",
 230   US"nhash",
 231   US"quote",
 232   US"randint",
 233   US"rfc2047",
 234   US"rfc2047d",
 235   US"rxquote",
 236   US"s",
 237   US"sha1",
 238   US"sha2",
 239   US"sha256",
 240   US"sha3",
 241   US"stat",
 242   US"str2b64",
 243   US"strlen",
 244   US"substr",
 245   US"uc",
 246   US"utf8clean" };
 247
 248 enum {
 249   EOP_ADDRESS =  nelem(op_table_underscore),
 250   EOP_ADDRESSES,
 251   EOP_BASE32,
 252   EOP_BASE32D,
 253   EOP_BASE62,
 254   EOP_BASE62D,
 255   EOP_BASE64,
 256   EOP_BASE64D,
 257   EOP_DOMAIN,
 258   EOP_ESCAPE,
 259   EOP_ESCAPE8BIT,
 260   EOP_EVAL,
 261   EOP_EVAL10,
 262   EOP_EXPAND,
 263   EOP_H,
 264   EOP_HASH,
 265   EOP_HEX2B64,
 266   EOP_HEXQUOTE,
 267   EOP_IPV6DENORM,
 268   EOP_IPV6NORM,
 269   EOP_L,
 270   EOP_LC,
 271   EOP_LENGTH,
 272   EOP_LISTCOUNT,
 273   EOP_LISTNAMED,
 274   EOP_MASK,
 275   EOP_MD5,
 276   EOP_NH,
 277   EOP_NHASH,
 278   EOP_QUOTE,
 279   EOP_RANDINT,
 280   EOP_RFC2047,
 281   EOP_RFC2047D,
 282   EOP_RXQUOTE,
 283   EOP_S,
 284   EOP_SHA1,
 285   EOP_SHA2,
 286   EOP_SHA256,
 287   EOP_SHA3,
 288   EOP_STAT,
 289   EOP_STR2B64,
 290   EOP_STRLEN,
 291   EOP_SUBSTR,
 292   EOP_UC,
 293   EOP_UTF8CLEAN };
 294
 295
 296 /* Table of condition names, and corresponding switch numbers. The names must
 297 be in alphabetical order. */
 298
 299 static uschar *cond_table[] = {
 300   US"<",
 301   US"<=",
 302   US"=",
 303   US"==",     /* Backward compatibility */
 304   US">",
 305   US">=",
 306   US"acl",
 307   US"and",
 308   US"bool",
 309   US"bool_lax",
 310   US"crypteq",
 311   US"def",
 312   US"eq",
 313   US"eqi",
 314   US"exists",
 315   US"first_delivery",
 316   US"forall",
 317   US"forall_json",
 318   US"forall_jsons",
 319   US"forany",
 320   US"forany_json",
 321   US"forany_jsons",
 322   US"ge",
 323   US"gei",
 324   US"gt",
 325   US"gti",
 326   US"inlist",
 327   US"inlisti",
 328   US"isip",
 329   US"isip4",
 330   US"isip6",
 331   US"ldapauth",
 332   US"le",
 333   US"lei",
 334   US"lt",
 335   US"lti",
 336   US"match",
 337   US"match_address",
 338   US"match_domain",
 339   US"match_ip",
 340   US"match_local_part",
 341   US"or",
 342   US"pam",
 343   US"pwcheck",
 344   US"queue_running",
 345   US"radius",
 346   US"saslauthd"
 347 };
 348
 349 enum {
 350   ECOND_NUM_L,
 351   ECOND_NUM_LE,
 352   ECOND_NUM_E,
 353   ECOND_NUM_EE,
 354   ECOND_NUM_G,
 355   ECOND_NUM_GE,
 356   ECOND_ACL,
 357   ECOND_AND,
 358   ECOND_BOOL,
 359   ECOND_BOOL_LAX,
 360   ECOND_CRYPTEQ,
 361   ECOND_DEF,
 362   ECOND_STR_EQ,
 363   ECOND_STR_EQI,
 364   ECOND_EXISTS,
 365   ECOND_FIRST_DELIVERY,
 366   ECOND_FORALL,
 367   ECOND_FORALL_JSON,
 368   ECOND_FORALL_JSONS,
 369   ECOND_FORANY,
 370   ECOND_FORANY_JSON,
 371   ECOND_FORANY_JSONS,
 372   ECOND_STR_GE,
 373   ECOND_STR_GEI,
 374   ECOND_STR_GT,
 375   ECOND_STR_GTI,
 376   ECOND_INLIST,
 377   ECOND_INLISTI,
 378   ECOND_ISIP,
 379   ECOND_ISIP4,
 380   ECOND_ISIP6,
 381   ECOND_LDAPAUTH,
 382   ECOND_STR_LE,
 383   ECOND_STR_LEI,
 384   ECOND_STR_LT,
 385   ECOND_STR_LTI,
 386   ECOND_MATCH,
 387   ECOND_MATCH_ADDRESS,
 388   ECOND_MATCH_DOMAIN,
 389   ECOND_MATCH_IP,
 390   ECOND_MATCH_LOCAL_PART,
 391   ECOND_OR,
 392   ECOND_PAM,
 393   ECOND_PWCHECK,
 394   ECOND_QUEUE_RUNNING,
 395   ECOND_RADIUS,
 396   ECOND_SASLAUTHD
 397 };
 398
 399
 400 /* Types of table entry */
 401
 402 enum vtypes {
 403   vtype_int,            /* value is address of int */
 404   vtype_filter_int,     /* ditto, but recognized only when filtering */
 405   vtype_ino,            /* value is address of ino_t (not always an int) */
 406   vtype_uid,            /* value is address of uid_t (not always an int) */
 407   vtype_gid,            /* value is address of gid_t (not always an int) */
 408   vtype_bool,           /* value is address of bool */
 409   vtype_stringptr,      /* value is address of pointer to string */
 410   vtype_msgbody,        /* as stringptr, but read when first required */
 411   vtype_msgbody_end,    /* ditto, the end of the message */
 412   vtype_msgheaders,     /* the message's headers, processed */
 413   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 414   vtype_localpart,      /* extract local part from string */
 415   vtype_domain,         /* extract domain from string */
 416   vtype_string_func,    /* value is string returned by given function */
 417   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 418   vtype_tode,           /* value not used; generate tod in epoch format */
 419   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 420   vtype_todf,           /* value not used; generate full tod */
 421   vtype_todl,           /* value not used; generate log tod */
 422   vtype_todlf,          /* value not used; generate log file datestamp tod */
 423   vtype_todzone,        /* value not used; generate time zone only */
 424   vtype_todzulu,        /* value not used; generate zulu tod */
 425   vtype_reply,          /* value not used; get reply from headers */
 426   vtype_pid,            /* value not used; result is pid */
 427   vtype_host_lookup,    /* value not used; get host name */
 428   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 429   vtype_pspace,         /* partition space; value is T/F for spool/log */
 430   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 431   vtype_cert            /* SSL certificate */
 432   #ifndef DISABLE_DKIM
 433   ,vtype_dkim           /* Lookup of value in DKIM signature */
 434   #endif
 435 };
 436
 437 /* Type for main variable table */
 438
 439 typedef struct {
 440   const char *name;
 441   enum vtypes type;
 442   void       *value;
 443 } var_entry;
 444
 445 /* Type for entries pointing to address/length pairs. Not currently
 446 in use. */
 447
 448 typedef struct {
 449   uschar **address;
 450   int  *length;
 451 } alblock;
 452
 453 static uschar * fn_recipients(void);
 454
 455 /* This table must be kept in alphabetical order. */
 456
 457 static var_entry var_table[] = {
 458   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 459      they will be confused with user-creatable ACL variables. */
 460   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 461   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 462   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 463   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 464   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 465   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 466   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 467   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 468   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 469   { "acl_narg",            vtype_int,         &acl_narg },
 470   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 471   { "address_data",        vtype_stringptr,   &deliver_address_data },
 472   { "address_file",        vtype_stringptr,   &address_file },
 473   { "address_pipe",        vtype_stringptr,   &address_pipe },
 474 #ifdef EXPERIMENTAL_ARC
 475   { "arc_domains",         vtype_string_func, &fn_arc_domains },
 476   { "arc_oldest_pass",     vtype_int,         &arc_oldest_pass },
 477   { "arc_state",           vtype_stringptr,   &arc_state },
 478   { "arc_state_reason",    vtype_stringptr,   &arc_state_reason },
 479 #endif
 480   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 481   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 482   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 483   { "authentication_failed",vtype_int,        &authentication_failed },
 484 #ifdef WITH_CONTENT_SCAN
 485   { "av_failed",           vtype_int,         &av_failed },
 486 #endif
 487 #ifdef EXPERIMENTAL_BRIGHTMAIL
 488   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 489   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 490   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 491   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 492 #endif
 493   { "body_linecount",      vtype_int,         &body_linecount },
 494   { "body_zerocount",      vtype_int,         &body_zerocount },
 495   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 496   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 497   { "caller_gid",          vtype_gid,         &real_gid },
 498   { "caller_uid",          vtype_uid,         &real_uid },
 499   { "callout_address",     vtype_stringptr,   &callout_address },
 500   { "compile_date",        vtype_stringptr,   &version_date },
 501   { "compile_number",      vtype_stringptr,   &version_cnumber },
 502   { "config_dir",          vtype_stringptr,   &config_main_directory },
 503   { "config_file",         vtype_stringptr,   &config_main_filename },
 504   { "csa_status",          vtype_stringptr,   &csa_status },
 505 #ifdef EXPERIMENTAL_DCC
 506   { "dcc_header",          vtype_stringptr,   &dcc_header },
 507   { "dcc_result",          vtype_stringptr,   &dcc_result },
 508 #endif
 509 #ifndef DISABLE_DKIM
 510   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 511   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 512   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 513   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 514   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 515   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 516   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 517   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 518   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 519   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 520   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 521   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 522   { "dkim_key_length",     vtype_int,         &dkim_key_length },
 523   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 524   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 525   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 526   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 527   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 528   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 529   { "dkim_verify_reason",  vtype_stringptr,   &dkim_verify_reason },
 530   { "dkim_verify_status",  vtype_stringptr,   &dkim_verify_status },
 531 #endif
 532 #ifdef EXPERIMENTAL_DMARC
 533   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 534   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 535   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 536   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 537 #endif
 538   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 539   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 540   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 541   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 542   { "domain",              vtype_stringptr,   &deliver_domain },
 543   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 544 #ifndef DISABLE_EVENT
 545   { "event_data",          vtype_stringptr,   &event_data },
 546
 547   /*XXX want to use generic vars for as many of these as possible*/
 548   { "event_defer_errno",   vtype_int,         &event_defer_errno },
 549
 550   { "event_name",          vtype_stringptr,   &event_name },
 551 #endif
 552   { "exim_gid",            vtype_gid,         &exim_gid },
 553   { "exim_path",           vtype_stringptr,   &exim_path },
 554   { "exim_uid",            vtype_uid,         &exim_uid },
 555   { "exim_version",        vtype_stringptr,   &version_string },
 556   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 557   { "home",                vtype_stringptr,   &deliver_home },
 558   { "host",                vtype_stringptr,   &deliver_host },
 559   { "host_address",        vtype_stringptr,   &deliver_host_address },
 560   { "host_data",           vtype_stringptr,   &host_data },
 561   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 562   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 563   { "host_port",           vtype_int,         &deliver_host_port },
 564   { "initial_cwd",         vtype_stringptr,   &initial_cwd },
 565   { "inode",               vtype_ino,         &deliver_inode },
 566   { "interface_address",   vtype_stringptr,   &interface_address },
 567   { "interface_port",      vtype_int,         &interface_port },
 568   { "item",                vtype_stringptr,   &iterate_item },
 569   #ifdef LOOKUP_LDAP
 570   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 571   #endif
 572   { "load_average",        vtype_load_avg,    NULL },
 573   { "local_part",          vtype_stringptr,   &deliver_localpart },
 574   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 575   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 576   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 577 #ifdef HAVE_LOCAL_SCAN
 578   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 579 #endif
 580   { "local_user_gid",      vtype_gid,         &local_user_gid },
 581   { "local_user_uid",      vtype_uid,         &local_user_uid },
 582   { "localhost_number",    vtype_int,         &host_number },
 583   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 584   { "log_space",           vtype_pspace,      (void *)FALSE },
 585   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 586   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 587 #ifdef WITH_CONTENT_SCAN
 588   { "malware_name",        vtype_stringptr,   &malware_name },
 589 #endif
 590   { "max_received_linelength", vtype_int,     &max_received_linelength },
 591   { "message_age",         vtype_int,         &message_age },
 592   { "message_body",        vtype_msgbody,     &message_body },
 593   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 594   { "message_body_size",   vtype_int,         &message_body_size },
 595   { "message_exim_id",     vtype_stringptr,   &message_id },
 596   { "message_headers",     vtype_msgheaders,  NULL },
 597   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 598   { "message_id",          vtype_stringptr,   &message_id },
 599   { "message_linecount",   vtype_int,         &message_linecount },
 600   { "message_size",        vtype_int,         &message_size },
 601 #ifdef SUPPORT_I18N
 602   { "message_smtputf8",    vtype_bool,        &message_smtputf8 },
 603 #endif
 604 #ifdef WITH_CONTENT_SCAN
 605   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 606   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 607   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 608   { "mime_charset",        vtype_stringptr,   &mime_charset },
 609   { "mime_content_description", vtype_stringptr, &mime_content_description },
 610   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 611   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 612   { "mime_content_size",   vtype_int,         &mime_content_size },
 613   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 614   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 615   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 616   { "mime_filename",       vtype_stringptr,   &mime_filename },
 617   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 618   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 619   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 620   { "mime_part_count",     vtype_int,         &mime_part_count },
 621 #endif
 622   { "n0",                  vtype_filter_int,  &filter_n[0] },
 623   { "n1",                  vtype_filter_int,  &filter_n[1] },
 624   { "n2",                  vtype_filter_int,  &filter_n[2] },
 625   { "n3",                  vtype_filter_int,  &filter_n[3] },
 626   { "n4",                  vtype_filter_int,  &filter_n[4] },
 627   { "n5",                  vtype_filter_int,  &filter_n[5] },
 628   { "n6",                  vtype_filter_int,  &filter_n[6] },
 629   { "n7",                  vtype_filter_int,  &filter_n[7] },
 630   { "n8",                  vtype_filter_int,  &filter_n[8] },
 631   { "n9",                  vtype_filter_int,  &filter_n[9] },
 632   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 633   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 634   { "originator_gid",      vtype_gid,         &originator_gid },
 635   { "originator_uid",      vtype_uid,         &originator_uid },
 636   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 637   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 638   { "pid",                 vtype_pid,         NULL },
 639 #ifndef DISABLE_PRDR
 640   { "prdr_requested",      vtype_bool,        &prdr_requested },
 641 #endif
 642   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 643 #if defined(SUPPORT_PROXY) || defined(SUPPORT_SOCKS)
 644   { "proxy_external_address",vtype_stringptr, &proxy_external_address },
 645   { "proxy_external_port", vtype_int,         &proxy_external_port },
 646   { "proxy_local_address", vtype_stringptr,   &proxy_local_address },
 647   { "proxy_local_port",    vtype_int,         &proxy_local_port },
 648   { "proxy_session",       vtype_bool,        &proxy_session },
 649 #endif
 650   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 651   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 652   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 653   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 654   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 655   { "queue_name",          vtype_stringptr,   &queue_name },
 656   { "rcpt_count",          vtype_int,         &rcpt_count },
 657   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 658   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 659   { "received_count",      vtype_int,         &received_count },
 660   { "received_for",        vtype_stringptr,   &received_for },
 661   { "received_ip_address", vtype_stringptr,   &interface_address },
 662   { "received_port",       vtype_int,         &interface_port },
 663   { "received_protocol",   vtype_stringptr,   &received_protocol },
 664   { "received_time",       vtype_int,         &received_time.tv_sec },
 665   { "recipient_data",      vtype_stringptr,   &recipient_data },
 666   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 667   { "recipients",          vtype_string_func, &fn_recipients },
 668   { "recipients_count",    vtype_int,         &recipients_count },
 669 #ifdef WITH_CONTENT_SCAN
 670   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 671 #endif
 672   { "reply_address",       vtype_reply,       NULL },
 673   { "return_path",         vtype_stringptr,   &return_path },
 674   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 675   { "router_name",         vtype_stringptr,   &router_name },
 676   { "runrc",               vtype_int,         &runrc },
 677   { "self_hostname",       vtype_stringptr,   &self_hostname },
 678   { "sender_address",      vtype_stringptr,   &sender_address },
 679   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 680   { "sender_address_domain", vtype_domain,    &sender_address },
 681   { "sender_address_local_part", vtype_localpart, &sender_address },
 682   { "sender_data",         vtype_stringptr,   &sender_data },
 683   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 684   { "sender_helo_dnssec",  vtype_bool,        &sender_helo_dnssec },
 685   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 686   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 687   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 688   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 689   { "sender_host_name",    vtype_host_lookup, NULL },
 690   { "sender_host_port",    vtype_int,         &sender_host_port },
 691   { "sender_ident",        vtype_stringptr,   &sender_ident },
 692   { "sender_rate",         vtype_stringptr,   &sender_rate },
 693   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 694   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 695   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 696   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 697   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 698   { "sending_port",        vtype_int,         &sending_port },
 699   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 700   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 701   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 702   { "smtp_command_history", vtype_string_func, &smtp_cmd_hist },
 703   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 704   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 705   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 706   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 707   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 708   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 709   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 710   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 711   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 712   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 713   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 714   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 715 #ifdef WITH_CONTENT_SCAN
 716   { "spam_action",         vtype_stringptr,   &spam_action },
 717   { "spam_bar",            vtype_stringptr,   &spam_bar },
 718   { "spam_report",         vtype_stringptr,   &spam_report },
 719   { "spam_score",          vtype_stringptr,   &spam_score },
 720   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 721 #endif
 722 #ifdef SUPPORT_SPF
 723   { "spf_guess",           vtype_stringptr,   &spf_guess },
 724   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 725   { "spf_received",        vtype_stringptr,   &spf_received },
 726   { "spf_result",          vtype_stringptr,   &spf_result },
 727   { "spf_result_guessed",  vtype_bool,        &spf_result_guessed },
 728   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 729 #endif
 730   { "spool_directory",     vtype_stringptr,   &spool_directory },
 731   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 732   { "spool_space",         vtype_pspace,      (void *)TRUE },
 733 #ifdef EXPERIMENTAL_SRS
 734   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 735   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 736   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 737   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 738   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 739   { "srs_status",          vtype_stringptr,   &srs_status },
 740 #endif
 741   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 742
 743   /* The non-(in,out) variables are now deprecated */
 744   { "tls_bits",            vtype_int,         &tls_in.bits },
 745   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 746   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 747
 748   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 749   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 750   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 751   { "tls_in_cipher_std",   vtype_stringptr,   &tls_in.cipher_stdname },
 752   { "tls_in_ocsp",         vtype_int,         &tls_in.ocsp },
 753   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 754   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 755   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 756 #ifdef EXPERIMENTAL_TLS_RESUME
 757   { "tls_in_resumption",   vtype_int,         &tls_in.resumption },
 758 #endif
 759 #ifndef DISABLE_TLS
 760   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 761 #endif
 762   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 763   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 764   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 765   { "tls_out_cipher_std",  vtype_stringptr,   &tls_out.cipher_stdname },
 766 #ifdef SUPPORT_DANE
 767   { "tls_out_dane",        vtype_bool,        &tls_out.dane_verified },
 768 #endif
 769   { "tls_out_ocsp",        vtype_int,         &tls_out.ocsp },
 770   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 771   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 772   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 773 #ifdef EXPERIMENTAL_TLS_RESUME
 774   { "tls_out_resumption",  vtype_int,         &tls_out.resumption },
 775 #endif
 776 #ifndef DISABLE_TLS
 777   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 778 #endif
 779 #ifdef SUPPORT_DANE
 780   { "tls_out_tlsa_usage",  vtype_int,         &tls_out.tlsa_usage },
 781 #endif
 782
 783   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 784 #ifndef DISABLE_TLS
 785   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 786 #endif
 787
 788   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 789   { "tod_epoch",           vtype_tode,        NULL },
 790   { "tod_epoch_l",         vtype_todel,       NULL },
 791   { "tod_full",            vtype_todf,        NULL },
 792   { "tod_log",             vtype_todl,        NULL },
 793   { "tod_logfile",         vtype_todlf,       NULL },
 794   { "tod_zone",            vtype_todzone,     NULL },
 795   { "tod_zulu",            vtype_todzulu,     NULL },
 796   { "transport_name",      vtype_stringptr,   &transport_name },
 797   { "value",               vtype_stringptr,   &lookup_value },
 798   { "verify_mode",         vtype_stringptr,   &verify_mode },
 799   { "version_number",      vtype_stringptr,   &version_string },
 800   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 801   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 802   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 803   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 804   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 805   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 806 };
 807
 808 static int var_table_size = nelem(var_table);
 809 static uschar var_buffer[256];
 810 static BOOL malformed_header;
 811
 812 /* For textual hashes */
 813
 814 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 815                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 816                                "0123456789";
 817
 818 enum { HMAC_MD5, HMAC_SHA1 };
 819
 820 /* For numeric hashes */
 821
 822 static unsigned int prime[] = {
 823   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 824  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 825  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 826
 827 /* For printing modes in symbolic form */
 828
 829 static uschar *mtable_normal[] =
 830   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 831
 832 static uschar *mtable_setid[] =
 833   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 834
 835 static uschar *mtable_sticky[] =
 836   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 837
 838 /* flags for find_header() */
 839 #define FH_EXISTS_ONLY  BIT(0)
 840 #define FH_WANT_RAW     BIT(1)
 841 #define FH_WANT_LIST    BIT(2)
 842
 843
 844 /*************************************************
 845 *           Tables for UTF-8 support             *
 846 *************************************************/
 847
 848 /* Table of the number of extra characters, indexed by the first character
 849 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 850 0x3d. */
 851
 852 static uschar utf8_table1[] = {
 853   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 854   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 855   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 856   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 857
 858 /* These are the masks for the data bits in the first byte of a character,
 859 indexed by the number of additional bytes. */
 860
 861 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 862
 863 /* Get the next UTF-8 character, advancing the pointer. */
 864
 865 #define GETUTF8INC(c, ptr) \
 866   c = *ptr++; \
 867   if ((c & 0xc0) == 0xc0) \
 868     { \
 869     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 870     int s = 6*a; \
 871     c = (c & utf8_table2[a]) << s; \
 872     while (a-- > 0) \
 873       { \
 874       s -= 6; \
 875       c |= (*ptr++ & 0x3f) << s; \
 876       } \
 877     }
 878
 879
 880
 881 static uschar * base32_chars = US"abcdefghijklmnopqrstuvwxyz234567";
 882
 883 /*************************************************
 884 *           Binary chop search on a table        *
 885 *************************************************/
 886
 887 /* This is used for matching expansion items and operators.
 888
 889 Arguments:
 890   name        the name that is being sought
 891   table       the table to search
 892   table_size  the number of items in the table
 893
 894 Returns:      the offset in the table, or -1
 895 */
 896
 897 static int
 898 chop_match(uschar *name, uschar **table, int table_size)
 899 {
 900 uschar **bot = table;
 901 uschar **top = table + table_size;
 902
 903 while (top > bot)
 904   {
 905   uschar **mid = bot + (top - bot)/2;
 906   int c = Ustrcmp(name, *mid);
 907   if (c == 0) return mid - table;
 908   if (c > 0) bot = mid + 1; else top = mid;
 909   }
 910
 911 return -1;
 912 }
 913
 914
 915
 916 /*************************************************
 917 *          Check a condition string              *
 918 *************************************************/
 919
 920 /* This function is called to expand a string, and test the result for a "true"
 921 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 922 forced fail or lookup defer.
 923
 924 We used to release all store used, but this is not not safe due
 925 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 926 is reasonably careful to release what it can.
 927
 928 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 929
 930 Arguments:
 931   condition     the condition string
 932   m1            text to be incorporated in panic error
 933   m2            ditto
 934
 935 Returns:        TRUE if condition is met, FALSE if not
 936 */
 937
 938 BOOL
 939 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 940 {
 941 int rc;
 942 uschar *ss = expand_string(condition);
 943 if (ss == NULL)
 944   {
 945   if (!f.expand_string_forcedfail && !f.search_find_defer)
 946     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 947       "for %s %s: %s", condition, m1, m2, expand_string_message);
 948   return FALSE;
 949   }
 950 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 951   strcmpic(ss, US"false") != 0;
 952 return rc;
 953 }
 954
 955
 956
 957
 958 /*************************************************
 959 *        Pseudo-random number generation         *
 960 *************************************************/
 961
 962 /* Pseudo-random number generation.  The result is not "expected" to be
 963 cryptographically strong but not so weak that someone will shoot themselves
 964 in the foot using it as a nonce in some email header scheme or whatever
 965 weirdness they'll twist this into.  The result should ideally handle fork().
 966
 967 However, if we're stuck unable to provide this, then we'll fall back to
 968 appallingly bad randomness.
 969
 970 If DISABLE_TLS is not defined then this will not be used except as an emergency
 971 fallback.
 972
 973 Arguments:
 974   max       range maximum
 975 Returns     a random number in range [0, max-1]
 976 */
 977
 978 #ifndef DISABLE_TLS
 979 # define vaguely_random_number vaguely_random_number_fallback
 980 #endif
 981 int
 982 vaguely_random_number(int max)
 983 {
 984 #ifndef DISABLE_TLS
 985 # undef vaguely_random_number
 986 #endif
 987 static pid_t pid = 0;
 988 pid_t p2;
 989
 990 if ((p2 = getpid()) != pid)
 991   {
 992   if (pid != 0)
 993     {
 994
 995 #ifdef HAVE_ARC4RANDOM
 996     /* cryptographically strong randomness, common on *BSD platforms, not
 997     so much elsewhere.  Alas. */
 998 # ifndef NOT_HAVE_ARC4RANDOM_STIR
 999     arc4random_stir();
1000 # endif
1001 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
1002 # ifdef HAVE_SRANDOMDEV
1003     /* uses random(4) for seeding */
1004     srandomdev();
1005 # else
1006     {
1007     struct timeval tv;
1008     gettimeofday(&tv, NULL);
1009     srandom(tv.tv_sec | tv.tv_usec | getpid());
1010     }
1011 # endif
1012 #else
1013     /* Poor randomness and no seeding here */
1014 #endif
1015
1016     }
1017   pid = p2;
1018   }
1019
1020 #ifdef HAVE_ARC4RANDOM
1021 return arc4random() % max;
1022 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
1023 return random() % max;
1024 #else
1025 /* This one returns a 16-bit number, definitely not crypto-strong */
1026 return random_number(max);
1027 #endif
1028 }
1029
1030
1031
1032
1033 /*************************************************
1034 *             Pick out a name from a string      *
1035 *************************************************/
1036
1037 /* If the name is too long, it is silently truncated.
1038
1039 Arguments:
1040   name      points to a buffer into which to put the name
1041   max       is the length of the buffer
1042   s         points to the first alphabetic character of the name
1043   extras    chars other than alphanumerics to permit
1044
1045 Returns:    pointer to the first character after the name
1046
1047 Note: The test for *s != 0 in the while loop is necessary because
1048 Ustrchr() yields non-NULL if the character is zero (which is not something
1049 I expected). */
1050
1051 static const uschar *
1052 read_name(uschar *name, int max, const uschar *s, uschar *extras)
1053 {
1054 int ptr = 0;
1055 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
1056   {
1057   if (ptr < max-1) name[ptr++] = *s;
1058   s++;
1059   }
1060 name[ptr] = 0;
1061 return s;
1062 }
1063
1064
1065
1066 /*************************************************
1067 *     Pick out the rest of a header name         *
1068 *************************************************/
1069
1070 /* A variable name starting $header_ (or just $h_ for those who like
1071 abbreviations) might not be the complete header name because headers can
1072 contain any printing characters in their names, except ':'. This function is
1073 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
1074 on the end, if the name was terminated by white space.
1075
1076 Arguments:
1077   name      points to a buffer in which the name read so far exists
1078   max       is the length of the buffer
1079   s         points to the first character after the name so far, i.e. the
1080             first non-alphameric character after $header_xxxxx
1081
1082 Returns:    a pointer to the first character after the header name
1083 */
1084
1085 static const uschar *
1086 read_header_name(uschar *name, int max, const uschar *s)
1087 {
1088 int prelen = Ustrchr(name, '_') - name + 1;
1089 int ptr = Ustrlen(name) - prelen;
1090 if (ptr > 0) memmove(name, name+prelen, ptr);
1091 while (mac_isgraph(*s) && *s != ':')
1092   {
1093   if (ptr < max-1) name[ptr++] = *s;
1094   s++;
1095   }
1096 if (*s == ':') s++;
1097 name[ptr++] = ':';
1098 name[ptr] = 0;
1099 return s;
1100 }
1101
1102
1103
1104 /*************************************************
1105 *           Pick out a number from a string      *
1106 *************************************************/
1107
1108 /* Arguments:
1109   n     points to an integer into which to put the number
1110   s     points to the first digit of the number
1111
1112 Returns:  a pointer to the character after the last digit
1113 */
1114 /*XXX consider expanding to int_eximarith_t.  But the test for
1115 "overbig numbers" in 0002 still needs to overflow it. */
1116
1117 static uschar *
1118 read_number(int *n, uschar *s)
1119 {
1120 *n = 0;
1121 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1122 return s;
1123 }
1124
1125 static const uschar *
1126 read_cnumber(int *n, const uschar *s)
1127 {
1128 *n = 0;
1129 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1130 return s;
1131 }
1132
1133
1134
1135 /*************************************************
1136 *        Extract keyed subfield from a string    *
1137 *************************************************/
1138
1139 /* The yield is in dynamic store; NULL means that the key was not found.
1140
1141 Arguments:
1142   key       points to the name of the key
1143   s         points to the string from which to extract the subfield
1144
1145 Returns:    NULL if the subfield was not found, or
1146             a pointer to the subfield's data
1147 */
1148
1149 static uschar *
1150 expand_getkeyed(uschar * key, const uschar * s)
1151 {
1152 int length = Ustrlen(key);
1153 while (isspace(*s)) s++;
1154
1155 /* Loop to search for the key */
1156
1157 while (*s)
1158   {
1159   int dkeylength;
1160   uschar * data;
1161   const uschar * dkey = s;
1162
1163   while (*s && *s != '=' && !isspace(*s)) s++;
1164   dkeylength = s - dkey;
1165   while (isspace(*s)) s++;
1166   if (*s == '=') while (isspace((*(++s))));
1167
1168   data = string_dequote(&s);
1169   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1170     return data;
1171
1172   while (isspace(*s)) s++;
1173   }
1174
1175 return NULL;
1176 }
1177
1178
1179
1180 static var_entry *
1181 find_var_ent(uschar * name)
1182 {
1183 int first = 0;
1184 int last = var_table_size;
1185
1186 while (last > first)
1187   {
1188   int middle = (first + last)/2;
1189   int c = Ustrcmp(name, var_table[middle].name);
1190
1191   if (c > 0) { first = middle + 1; continue; }
1192   if (c < 0) { last = middle; continue; }
1193   return &var_table[middle];
1194   }
1195 return NULL;
1196 }
1197
1198 /*************************************************
1199 *   Extract numbered subfield from string        *
1200 *************************************************/
1201
1202 /* Extracts a numbered field from a string that is divided by tokens - for
1203 example a line from /etc/passwd is divided by colon characters.  First field is
1204 numbered one.  Negative arguments count from the right. Zero returns the whole
1205 string. Returns NULL if there are insufficient tokens in the string
1206
1207 ***WARNING***
1208 Modifies final argument - this is a dynamically generated string, so that's OK.
1209
1210 Arguments:
1211   field       number of field to be extracted,
1212                 first field = 1, whole string = 0, last field = -1
1213   separators  characters that are used to break string into tokens
1214   s           points to the string from which to extract the subfield
1215
1216 Returns:      NULL if the field was not found,
1217               a pointer to the field's data inside s (modified to add 0)
1218 */
1219
1220 static uschar *
1221 expand_gettokened (int field, uschar *separators, uschar *s)
1222 {
1223 int sep = 1;
1224 int count;
1225 uschar *ss = s;
1226 uschar *fieldtext = NULL;
1227
1228 if (field == 0) return s;
1229
1230 /* Break the line up into fields in place; for field > 0 we stop when we have
1231 done the number of fields we want. For field < 0 we continue till the end of
1232 the string, counting the number of fields. */
1233
1234 count = (field > 0)? field : INT_MAX;
1235
1236 while (count-- > 0)
1237   {
1238   size_t len;
1239
1240   /* Previous field was the last one in the string. For a positive field
1241   number, this means there are not enough fields. For a negative field number,
1242   check that there are enough, and scan back to find the one that is wanted. */
1243
1244   if (sep == 0)
1245     {
1246     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1247     if ((-field) == (INT_MAX - count - 1)) return s;
1248     while (field++ < 0)
1249       {
1250       ss--;
1251       while (ss[-1] != 0) ss--;
1252       }
1253     fieldtext = ss;
1254     break;
1255     }
1256
1257   /* Previous field was not last in the string; save its start and put a
1258   zero at its end. */
1259
1260   fieldtext = ss;
1261   len = Ustrcspn(ss, separators);
1262   sep = ss[len];
1263   ss[len] = 0;
1264   ss += len + 1;
1265   }
1266
1267 return fieldtext;
1268 }
1269
1270
1271 static uschar *
1272 expand_getlistele(int field, const uschar * list)
1273 {
1274 const uschar * tlist = list;
1275 int sep = 0;
1276 uschar dummy;
1277
1278 if (field < 0)
1279   {
1280   for (field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1281   sep = 0;
1282   }
1283 if (field == 0) return NULL;
1284 while (--field > 0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1285 return string_nextinlist(&list, &sep, NULL, 0);
1286 }
1287
1288
1289 /* Certificate fields, by name.  Worry about by-OID later */
1290 /* Names are chosen to not have common prefixes */
1291
1292 #ifndef DISABLE_TLS
1293 typedef struct
1294 {
1295 uschar * name;
1296 int      namelen;
1297 uschar * (*getfn)(void * cert, uschar * mod);
1298 } certfield;
1299 static certfield certfields[] =
1300 {                       /* linear search; no special order */
1301   { US"version",         7,  &tls_cert_version },
1302   { US"serial_number",   13, &tls_cert_serial_number },
1303   { US"subject",         7,  &tls_cert_subject },
1304   { US"notbefore",       9,  &tls_cert_not_before },
1305   { US"notafter",        8,  &tls_cert_not_after },
1306   { US"issuer",          6,  &tls_cert_issuer },
1307   { US"signature",       9,  &tls_cert_signature },
1308   { US"sig_algorithm",   13, &tls_cert_signature_algorithm },
1309   { US"subj_altname",    12, &tls_cert_subject_altname },
1310   { US"ocsp_uri",        8,  &tls_cert_ocsp_uri },
1311   { US"crl_uri",         7,  &tls_cert_crl_uri },
1312 };
1313
1314 static uschar *
1315 expand_getcertele(uschar * field, uschar * certvar)
1316 {
1317 var_entry * vp;
1318
1319 if (!(vp = find_var_ent(certvar)))
1320   {
1321   expand_string_message =
1322     string_sprintf("no variable named \"%s\"", certvar);
1323   return NULL;          /* Unknown variable name */
1324   }
1325 /* NB this stops us passing certs around in variable.  Might
1326 want to do that in future */
1327 if (vp->type != vtype_cert)
1328   {
1329   expand_string_message =
1330     string_sprintf("\"%s\" is not a certificate", certvar);
1331   return NULL;          /* Unknown variable name */
1332   }
1333 if (!*(void **)vp->value)
1334   return NULL;
1335
1336 if (*field >= '0' && *field <= '9')
1337   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1338
1339 for (certfield * cp = certfields;
1340      cp < certfields + nelem(certfields);
1341      cp++)
1342   if (Ustrncmp(cp->name, field, cp->namelen) == 0)
1343     {
1344     uschar * modifier = *(field += cp->namelen) == ','
1345       ? ++field : NULL;
1346     return (*cp->getfn)( *(void **)vp->value, modifier );
1347     }
1348
1349 expand_string_message =
1350   string_sprintf("bad field selector \"%s\" for certextract", field);
1351 return NULL;
1352 }
1353 #endif  /*DISABLE_TLS*/
1354
1355 /*************************************************
1356 *        Extract a substring from a string       *
1357 *************************************************/
1358
1359 /* Perform the ${substr or ${length expansion operations.
1360
1361 Arguments:
1362   subject     the input string
1363   value1      the offset from the start of the input string to the start of
1364                 the output string; if negative, count from the right.
1365   value2      the length of the output string, or negative (-1) for unset
1366                 if value1 is positive, unset means "all after"
1367                 if value1 is negative, unset means "all before"
1368   len         set to the length of the returned string
1369
1370 Returns:      pointer to the output string, or NULL if there is an error
1371 */
1372
1373 static uschar *
1374 extract_substr(uschar *subject, int value1, int value2, int *len)
1375 {
1376 int sublen = Ustrlen(subject);
1377
1378 if (value1 < 0)    /* count from right */
1379   {
1380   value1 += sublen;
1381
1382   /* If the position is before the start, skip to the start, and adjust the
1383   length. If the length ends up negative, the substring is null because nothing
1384   can precede. This falls out naturally when the length is unset, meaning "all
1385   to the left". */
1386
1387   if (value1 < 0)
1388     {
1389     value2 += value1;
1390     if (value2 < 0) value2 = 0;
1391     value1 = 0;
1392     }
1393
1394   /* Otherwise an unset length => characters before value1 */
1395
1396   else if (value2 < 0)
1397     {
1398     value2 = value1;
1399     value1 = 0;
1400     }
1401   }
1402
1403 /* For a non-negative offset, if the starting position is past the end of the
1404 string, the result will be the null string. Otherwise, an unset length means
1405 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1406
1407 else
1408   {
1409   if (value1 > sublen)
1410     {
1411     value1 = sublen;
1412     value2 = 0;
1413     }
1414   else if (value2 < 0) value2 = sublen;
1415   }
1416
1417 /* Cut the length down to the maximum possible for the offset value, and get
1418 the required characters. */
1419
1420 if (value1 + value2 > sublen) value2 = sublen - value1;
1421 *len = value2;
1422 return subject + value1;
1423 }
1424
1425
1426
1427
1428 /*************************************************
1429 *            Old-style hash of a string          *
1430 *************************************************/
1431
1432 /* Perform the ${hash expansion operation.
1433
1434 Arguments:
1435   subject     the input string (an expanded substring)
1436   value1      the length of the output string; if greater or equal to the
1437                 length of the input string, the input string is returned
1438   value2      the number of hash characters to use, or 26 if negative
1439   len         set to the length of the returned string
1440
1441 Returns:      pointer to the output string, or NULL if there is an error
1442 */
1443
1444 static uschar *
1445 compute_hash(uschar *subject, int value1, int value2, int *len)
1446 {
1447 int sublen = Ustrlen(subject);
1448
1449 if (value2 < 0) value2 = 26;
1450 else if (value2 > Ustrlen(hashcodes))
1451   {
1452   expand_string_message =
1453     string_sprintf("hash count \"%d\" too big", value2);
1454   return NULL;
1455   }
1456
1457 /* Calculate the hash text. We know it is shorter than the original string, so
1458 can safely place it in subject[] (we know that subject is always itself an
1459 expanded substring). */
1460
1461 if (value1 < sublen)
1462   {
1463   int c;
1464   int i = 0;
1465   int j = value1;
1466   while ((c = (subject[j])) != 0)
1467     {
1468     int shift = (c + j++) & 7;
1469     subject[i] ^= (c << shift) | (c >> (8-shift));
1470     if (++i >= value1) i = 0;
1471     }
1472   for (i = 0; i < value1; i++)
1473     subject[i] = hashcodes[(subject[i]) % value2];
1474   }
1475 else value1 = sublen;
1476
1477 *len = value1;
1478 return subject;
1479 }
1480
1481
1482
1483
1484 /*************************************************
1485 *             Numeric hash of a string           *
1486 *************************************************/
1487
1488 /* Perform the ${nhash expansion operation. The first characters of the
1489 string are treated as most important, and get the highest prime numbers.
1490
1491 Arguments:
1492   subject     the input string
1493   value1      the maximum value of the first part of the result
1494   value2      the maximum value of the second part of the result,
1495                 or negative to produce only a one-part result
1496   len         set to the length of the returned string
1497
1498 Returns:  pointer to the output string, or NULL if there is an error.
1499 */
1500
1501 static uschar *
1502 compute_nhash (uschar *subject, int value1, int value2, int *len)
1503 {
1504 uschar *s = subject;
1505 int i = 0;
1506 unsigned long int total = 0; /* no overflow */
1507
1508 while (*s != 0)
1509   {
1510   if (i == 0) i = nelem(prime) - 1;
1511   total += prime[i--] * (unsigned int)(*s++);
1512   }
1513
1514 /* If value2 is unset, just compute one number */
1515
1516 if (value2 < 0)
1517   s = string_sprintf("%lu", total % value1);
1518
1519 /* Otherwise do a div/mod hash */
1520
1521 else
1522   {
1523   total = total % (value1 * value2);
1524   s = string_sprintf("%lu/%lu", total/value2, total % value2);
1525   }
1526
1527 *len = Ustrlen(s);
1528 return s;
1529 }
1530
1531
1532
1533
1534
1535 /*************************************************
1536 *     Find the value of a header or headers      *
1537 *************************************************/
1538
1539 /* Multiple instances of the same header get concatenated, and this function
1540 can also return a concatenation of all the header lines. When concatenating
1541 specific headers that contain lists of addresses, a comma is inserted between
1542 them. Otherwise we use a straight concatenation. Because some messages can have
1543 pathologically large number of lines, there is a limit on the length that is
1544 returned.
1545
1546 Arguments:
1547   name          the name of the header, without the leading $header_ or $h_,
1548                 or NULL if a concatenation of all headers is required
1549   newsize       return the size of memory block that was obtained; may be NULL
1550                 if exists_only is TRUE
1551   flags         FH_EXISTS_ONLY
1552                   set if called from a def: test; don't need to build a string;
1553                   just return a string that is not "" and not "0" if the header
1554                   exists
1555                 FH_WANT_RAW
1556                   set if called for $rh_ or $rheader_ items; no processing,
1557                   other than concatenating, will be done on the header. Also used
1558                   for $message_headers_raw.
1559                 FH_WANT_LIST
1560                   Double colon chars in the content, and replace newline with
1561                   colon between each element when concatenating; returning a
1562                   colon-sep list (elements might contain newlines)
1563   charset       name of charset to translate MIME words to; used only if
1564                 want_raw is false; if NULL, no translation is done (this is
1565                 used for $bh_ and $bheader_)
1566
1567 Returns:        NULL if the header does not exist, else a pointer to a new
1568                 store block
1569 */
1570
1571 static uschar *
1572 find_header(uschar *name, int *newsize, unsigned flags, uschar *charset)
1573 {
1574 BOOL found = !name;
1575 int len = name ? Ustrlen(name) : 0;
1576 BOOL comma = FALSE;
1577 gstring * g = NULL;
1578
1579 for (header_line * h = header_list; h; h = h->next)
1580   if (h->type != htype_old && h->text)  /* NULL => Received: placeholder */
1581     if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1582       {
1583       uschar * s, * t;
1584       size_t inc;
1585
1586       if (flags & FH_EXISTS_ONLY)
1587         return US"1";  /* don't need actual string */
1588
1589       found = TRUE;
1590       s = h->text + len;                /* text to insert */
1591       if (!(flags & FH_WANT_RAW))       /* unless wanted raw, */
1592         while (isspace(*s)) s++;        /* remove leading white space */
1593       t = h->text + h->slen;            /* end-point */
1594
1595       /* Unless wanted raw, remove trailing whitespace, including the
1596       newline. */
1597
1598       if (flags & FH_WANT_LIST)
1599         while (t > s && t[-1] == '\n') t--;
1600       else if (!(flags & FH_WANT_RAW))
1601         {
1602         while (t > s && isspace(t[-1])) t--;
1603
1604         /* Set comma if handling a single header and it's one of those
1605         that contains an address list, except when asked for raw headers. Only
1606         need to do this once. */
1607
1608         if (name && !comma && Ustrchr("BCFRST", h->type)) comma = TRUE;
1609         }
1610
1611       /* Trim the header roughly if we're approaching limits */
1612       inc = t - s;
1613       if ((g ? g->ptr : 0) + inc > header_insert_maxlen)
1614         inc = header_insert_maxlen - (g ? g->ptr : 0);
1615
1616       /* For raw just copy the data; for a list, add the data as a colon-sep
1617       list-element; for comma-list add as an unchecked comma,newline sep
1618       list-elemment; for other nonraw add as an unchecked newline-sep list (we
1619       stripped trailing WS above including the newline). We ignore the potential
1620       expansion due to colon-doubling, just leaving the loop if the limit is met
1621       or exceeded. */
1622
1623       if (flags & FH_WANT_LIST)
1624         g = string_append_listele_n(g, ':', s, (unsigned)inc);
1625       else if (flags & FH_WANT_RAW)
1626         {
1627         g = string_catn(g, s, (unsigned)inc);
1628         (void) string_from_gstring(g);
1629         }
1630       else if (inc > 0)
1631         if (comma)
1632           g = string_append2_listele_n(g, US",\n", s, (unsigned)inc);
1633         else
1634           g = string_append2_listele_n(g, US"\n", s, (unsigned)inc);
1635
1636       if (g && g->ptr >= header_insert_maxlen) break;
1637       }
1638
1639 if (!found) return NULL;        /* No header found */
1640 if (!g) return US"";
1641
1642 /* That's all we do for raw header expansion. */
1643
1644 *newsize = g->size;
1645 if (flags & FH_WANT_RAW)
1646   return g->s;
1647
1648 /* Otherwise do RFC 2047 decoding, translating the charset if requested.
1649 The rfc2047_decode2() function can return an error with decoded data if the
1650 charset translation fails. If decoding fails, it returns NULL. */
1651
1652 else
1653   {
1654   uschar *decoded, *error;
1655
1656   decoded = rfc2047_decode2(g->s, check_rfc2047_length, charset, '?', NULL,
1657     newsize, &error);
1658   if (error)
1659     {
1660     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1661       "    input was: %s\n", error, g->s);
1662     }
1663   return decoded ? decoded : g->s;
1664   }
1665 }
1666
1667
1668
1669
1670 /* Append a "local" element to an Authentication-Results: header
1671 if this was a non-smtp message.
1672 */
1673
1674 static gstring *
1675 authres_local(gstring * g, const uschar * sysname)
1676 {
1677 if (!f.authentication_local)
1678   return g;
1679 g = string_append(g, 3, US";\n\tlocal=pass (non-smtp, ", sysname, US")");
1680 if (authenticated_id) g = string_append(g, 2, " u=", authenticated_id);
1681 return g;
1682 }
1683
1684
1685 /* Append an "iprev" element to an Authentication-Results: header
1686 if we have attempted to get the calling host's name.
1687 */
1688
1689 static gstring *
1690 authres_iprev(gstring * g)
1691 {
1692 if (sender_host_name)
1693   g = string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")");
1694 else if (host_lookup_deferred)
1695   g = string_catn(g, US";\n\tiprev=temperror", 19);
1696 else if (host_lookup_failed)
1697   g = string_catn(g, US";\n\tiprev=fail", 13);
1698 else
1699   return g;
1700
1701 if (sender_host_address)
1702   g = string_append(g, 2, US" smtp.remote-ip=", sender_host_address);
1703 return g;
1704 }
1705
1706
1707
1708 /*************************************************
1709 *               Return list of recipients        *
1710 *************************************************/
1711 /* A recipients list is available only during system message filtering,
1712 during ACL processing after DATA, and while expanding pipe commands
1713 generated from a system filter, but not elsewhere. */
1714
1715 static uschar *
1716 fn_recipients(void)
1717 {
1718 uschar * s;
1719 gstring * g = NULL;
1720
1721 if (!f.enable_dollar_recipients) return NULL;
1722
1723 for (int i = 0; i < recipients_count; i++)
1724   {
1725   s = recipients_list[i].address;
1726   g = string_append2_listele_n(g, US", ", s, Ustrlen(s));
1727   }
1728 return g ? g->s : NULL;
1729 }
1730
1731
1732 /*************************************************
1733 *               Find value of a variable         *
1734 *************************************************/
1735
1736 /* The table of variables is kept in alphabetic order, so we can search it
1737 using a binary chop. The "choplen" variable is nothing to do with the binary
1738 chop.
1739
1740 Arguments:
1741   name          the name of the variable being sought
1742   exists_only   TRUE if this is a def: test; passed on to find_header()
1743   skipping      TRUE => skip any processing evaluation; this is not the same as
1744                   exists_only because def: may test for values that are first
1745                   evaluated here
1746   newsize       pointer to an int which is initially zero; if the answer is in
1747                 a new memory buffer, *newsize is set to its size
1748
1749 Returns:        NULL if the variable does not exist, or
1750                 a pointer to the variable's contents, or
1751                 something non-NULL if exists_only is TRUE
1752 */
1753
1754 static uschar *
1755 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1756 {
1757 var_entry * vp;
1758 uschar *s, *domain;
1759 uschar **ss;
1760 void * val;
1761
1762 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1763 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1764 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1765 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1766 (this gave backwards compatibility at the changeover). There may be built-in
1767 variables whose names start acl_ but they should never start in this way. This
1768 slightly messy specification is a consequence of the history, needless to say.
1769
1770 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1771 set, in which case give an error. */
1772
1773 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1774      !isalpha(name[5]))
1775   {
1776   tree_node * node =
1777     tree_search(name[4] == 'c' ? acl_var_c : acl_var_m, name + 4);
1778   return node ? node->data.ptr : strict_acl_vars ? NULL : US"";
1779   }
1780 else if (Ustrncmp(name, "r_", 2) == 0)
1781   {
1782   tree_node * node = tree_search(router_var, name + 2);
1783   return node ? node->data.ptr : strict_acl_vars ? NULL : US"";
1784   }
1785
1786 /* Handle $auth<n> variables. */
1787
1788 if (Ustrncmp(name, "auth", 4) == 0)
1789   {
1790   uschar *endptr;
1791   int n = Ustrtoul(name + 4, &endptr, 10);
1792   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1793     return !auth_vars[n-1] ? US"" : auth_vars[n-1];
1794   }
1795 else if (Ustrncmp(name, "regex", 5) == 0)
1796   {
1797   uschar *endptr;
1798   int n = Ustrtoul(name + 5, &endptr, 10);
1799   if (*endptr == 0 && n != 0 && n <= REGEX_VARS)
1800     return !regex_vars[n-1] ? US"" : regex_vars[n-1];
1801   }
1802
1803 /* For all other variables, search the table */
1804
1805 if (!(vp = find_var_ent(name)))
1806   return NULL;          /* Unknown variable name */
1807
1808 /* Found an existing variable. If in skipping state, the value isn't needed,
1809 and we want to avoid processing (such as looking up the host name). */
1810
1811 if (skipping)
1812   return US"";
1813
1814 val = vp->value;
1815 switch (vp->type)
1816   {
1817   case vtype_filter_int:
1818     if (!f.filter_running) return NULL;
1819     /* Fall through */
1820     /* VVVVVVVVVVVV */
1821   case vtype_int:
1822     sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1823     return var_buffer;
1824
1825   case vtype_ino:
1826     sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1827     return var_buffer;
1828
1829   case vtype_gid:
1830     sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1831     return var_buffer;
1832
1833   case vtype_uid:
1834     sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1835     return var_buffer;
1836
1837   case vtype_bool:
1838     sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1839     return var_buffer;
1840
1841   case vtype_stringptr:                      /* Pointer to string */
1842     return (s = *((uschar **)(val))) ? s : US"";
1843
1844   case vtype_pid:
1845     sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1846     return var_buffer;
1847
1848   case vtype_load_avg:
1849     sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1850     return var_buffer;
1851
1852   case vtype_host_lookup:                    /* Lookup if not done so */
1853     if (  !sender_host_name && sender_host_address
1854        && !host_lookup_failed && host_name_lookup() == OK)
1855       host_build_sender_fullhost();
1856     return sender_host_name ? sender_host_name : US"";
1857
1858   case vtype_localpart:                      /* Get local part from address */
1859     s = *((uschar **)(val));
1860     if (s == NULL) return US"";
1861     domain = Ustrrchr(s, '@');
1862     if (domain == NULL) return s;
1863     if (domain - s > sizeof(var_buffer) - 1)
1864       log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1865           " in string expansion", sizeof(var_buffer));
1866     Ustrncpy(var_buffer, s, domain - s);
1867     var_buffer[domain - s] = 0;
1868     return var_buffer;
1869
1870   case vtype_domain:                         /* Get domain from address */
1871     s = *((uschar **)(val));
1872     if (s == NULL) return US"";
1873     domain = Ustrrchr(s, '@');
1874     return (domain == NULL)? US"" : domain + 1;
1875
1876   case vtype_msgheaders:
1877     return find_header(NULL, newsize, exists_only ? FH_EXISTS_ONLY : 0, NULL);
1878
1879   case vtype_msgheaders_raw:
1880     return find_header(NULL, newsize,
1881                 exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, NULL);
1882
1883   case vtype_msgbody:                        /* Pointer to msgbody string */
1884   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1885     ss = (uschar **)(val);
1886     if (!*ss && deliver_datafile >= 0)  /* Read body when needed */
1887       {
1888       uschar *body;
1889       off_t start_offset = SPOOL_DATA_START_OFFSET;
1890       int len = message_body_visible;
1891       if (len > message_size) len = message_size;
1892       *ss = body = store_malloc(len+1);
1893       body[0] = 0;
1894       if (vp->type == vtype_msgbody_end)
1895         {
1896         struct stat statbuf;
1897         if (fstat(deliver_datafile, &statbuf) == 0)
1898           {
1899           start_offset = statbuf.st_size - len;
1900           if (start_offset < SPOOL_DATA_START_OFFSET)
1901             start_offset = SPOOL_DATA_START_OFFSET;
1902           }
1903         }
1904       if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0)
1905         log_write(0, LOG_MAIN|LOG_PANIC_DIE, "deliver_datafile lseek: %s",
1906           strerror(errno));
1907       len = read(deliver_datafile, body, len);
1908       if (len > 0)
1909         {
1910         body[len] = 0;
1911         if (message_body_newlines)   /* Separate loops for efficiency */
1912           while (len > 0)
1913             { if (body[--len] == 0) body[len] = ' '; }
1914         else
1915           while (len > 0)
1916             { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1917         }
1918       }
1919     return *ss ? *ss : US"";
1920
1921   case vtype_todbsdin:                       /* BSD inbox time of day */
1922     return tod_stamp(tod_bsdin);
1923
1924   case vtype_tode:                           /* Unix epoch time of day */
1925     return tod_stamp(tod_epoch);
1926
1927   case vtype_todel:                          /* Unix epoch/usec time of day */
1928     return tod_stamp(tod_epoch_l);
1929
1930   case vtype_todf:                           /* Full time of day */
1931     return tod_stamp(tod_full);
1932
1933   case vtype_todl:                           /* Log format time of day */
1934     return tod_stamp(tod_log_bare);            /* (without timezone) */
1935
1936   case vtype_todzone:                        /* Time zone offset only */
1937     return tod_stamp(tod_zone);
1938
1939   case vtype_todzulu:                        /* Zulu time */
1940     return tod_stamp(tod_zulu);
1941
1942   case vtype_todlf:                          /* Log file datestamp tod */
1943     return tod_stamp(tod_log_datestamp_daily);
1944
1945   case vtype_reply:                          /* Get reply address */
1946     s = find_header(US"reply-to:", newsize,
1947                 exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW,
1948                 headers_charset);
1949     if (s) while (isspace(*s)) s++;
1950     if (!s || !*s)
1951       {
1952       *newsize = 0;                            /* For the *s==0 case */
1953       s = find_header(US"from:", newsize,
1954                 exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW,
1955                 headers_charset);
1956       }
1957     if (s)
1958       {
1959       uschar *t;
1960       while (isspace(*s)) s++;
1961       for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1962       while (t > s && isspace(t[-1])) t--;
1963       *t = 0;
1964       }
1965     return s ? s : US"";
1966
1967   case vtype_string_func:
1968     {
1969     uschar * (*fn)() = val;
1970     return fn();
1971     }
1972
1973   case vtype_pspace:
1974     {
1975     int inodes;
1976     sprintf(CS var_buffer, PR_EXIM_ARITH,
1977       receive_statvfs(val == (void *)TRUE, &inodes));
1978     }
1979   return var_buffer;
1980
1981   case vtype_pinodes:
1982     {
1983     int inodes;
1984     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1985     sprintf(CS var_buffer, "%d", inodes);
1986     }
1987   return var_buffer;
1988
1989   case vtype_cert:
1990     return *(void **)val ? US"<cert>" : US"";
1991
1992 #ifndef DISABLE_DKIM
1993   case vtype_dkim:
1994     return dkim_exim_expand_query((int)(long)val);
1995 #endif
1996
1997   }
1998
1999 return NULL;  /* Unknown variable. Silences static checkers. */
2000 }
2001
2002
2003
2004
2005 void
2006 modify_variable(uschar *name, void * value)
2007 {
2008 var_entry * vp;
2009 if ((vp = find_var_ent(name))) vp->value = value;
2010 return;          /* Unknown variable name, fail silently */
2011 }
2012
2013
2014
2015
2016
2017
2018 /*************************************************
2019 *           Read and expand substrings           *
2020 *************************************************/
2021
2022 /* This function is called to read and expand argument substrings for various
2023 expansion items. Some have a minimum requirement that is less than the maximum;
2024 in these cases, the first non-present one is set to NULL.
2025
2026 Arguments:
2027   sub        points to vector of pointers to set
2028   n          maximum number of substrings
2029   m          minimum required
2030   sptr       points to current string pointer
2031   skipping   the skipping flag
2032   check_end  if TRUE, check for final '}'
2033   name       name of item, for error message
2034   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
2035              the store.
2036
2037 Returns:     0 OK; string pointer updated
2038              1 curly bracketing error (too few arguments)
2039              2 too many arguments (only if check_end is set); message set
2040              3 other error (expansion failure)
2041 */
2042
2043 static int
2044 read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping,
2045   BOOL check_end, uschar *name, BOOL *resetok)
2046 {
2047 const uschar *s = *sptr;
2048
2049 while (isspace(*s)) s++;
2050 for (int i = 0; i < n; i++)
2051   {
2052   if (*s != '{')
2053     {
2054     if (i < m)
2055       {
2056       expand_string_message = string_sprintf("Not enough arguments for '%s' "
2057         "(min is %d)", name, m);
2058       return 1;
2059       }
2060     sub[i] = NULL;
2061     break;
2062     }
2063   if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok)))
2064     return 3;
2065   if (*s++ != '}') return 1;
2066   while (isspace(*s)) s++;
2067   }
2068 if (check_end && *s++ != '}')
2069   {
2070   if (s[-1] == '{')
2071     {
2072     expand_string_message = string_sprintf("Too many arguments for '%s' "
2073       "(max is %d)", name, n);
2074     return 2;
2075     }
2076   expand_string_message = string_sprintf("missing '}' after '%s'", name);
2077   return 1;
2078   }
2079
2080 *sptr = s;
2081 return 0;
2082 }
2083
2084
2085
2086
2087 /*************************************************
2088 *     Elaborate message for bad variable         *
2089 *************************************************/
2090
2091 /* For the "unknown variable" message, take a look at the variable's name, and
2092 give additional information about possible ACL variables. The extra information
2093 is added on to expand_string_message.
2094
2095 Argument:   the name of the variable
2096 Returns:    nothing
2097 */
2098
2099 static void
2100 check_variable_error_message(uschar *name)
2101 {
2102 if (Ustrncmp(name, "acl_", 4) == 0)
2103   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
2104     (name[4] == 'c' || name[4] == 'm')?
2105       (isalpha(name[5])?
2106         US"6th character of a user-defined ACL variable must be a digit or underscore" :
2107         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
2108       ) :
2109       US"user-defined ACL variables must start acl_c or acl_m");
2110 }
2111
2112
2113
2114 /*
2115 Load args from sub array to globals, and call acl_check().
2116 Sub array will be corrupted on return.
2117
2118 Returns:       OK         access is granted by an ACCEPT verb
2119                DISCARD    access is (apparently) granted by a DISCARD verb
2120                FAIL       access is denied
2121                FAIL_DROP  access is denied; drop the connection
2122                DEFER      can't tell at the moment
2123                ERROR      disaster
2124 */
2125 static int
2126 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
2127 {
2128 int i;
2129 int sav_narg = acl_narg;
2130 int ret;
2131 uschar * dummy_logmsg;
2132 extern int acl_where;
2133
2134 if(--nsub > nelem(acl_arg)) nsub = nelem(acl_arg);
2135 for (i = 0; i < nsub && sub[i+1]; i++)
2136   {
2137   uschar * tmp = acl_arg[i];
2138   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2139   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2140   }
2141 acl_narg = i;
2142 while (i < nsub)
2143   {
2144   sub[i+1] = acl_arg[i];
2145   acl_arg[i++] = NULL;
2146   }
2147
2148 DEBUG(D_expand)
2149   debug_printf_indent("expanding: acl: %s  arg: %s%s\n",
2150     sub[0],
2151     acl_narg>0 ? acl_arg[0] : US"<none>",
2152     acl_narg>1 ? " +more"   : "");
2153
2154 ret = acl_eval(acl_where, sub[0], user_msgp, &dummy_logmsg);
2155
2156 for (i = 0; i < nsub; i++)
2157   acl_arg[i] = sub[i+1];        /* restore old args */
2158 acl_narg = sav_narg;
2159
2160 return ret;
2161 }
2162
2163
2164
2165
2166 /* Return pointer to dewrapped string, with enclosing specified chars removed.
2167 The given string is modified on return.  Leading whitespace is skipped while
2168 looking for the opening wrap character, then the rest is scanned for the trailing
2169 (non-escaped) wrap character.  A backslash in the string will act as an escape.
2170
2171 A nul is written over the trailing wrap, and a pointer to the char after the
2172 leading wrap is returned.
2173
2174 Arguments:
2175   s     String for de-wrapping
2176   wrap  Two-char string, the first being the opener, second the closer wrapping
2177         character
2178 Return:
2179   Pointer to de-wrapped string, or NULL on error (with expand_string_message set).
2180 */
2181
2182 static uschar *
2183 dewrap(uschar * s, const uschar * wrap)
2184 {
2185 uschar * p = s;
2186 unsigned depth = 0;
2187 BOOL quotesmode = wrap[0] == wrap[1];
2188
2189 while (isspace(*p)) p++;
2190
2191 if (*p == *wrap)
2192   {
2193   s = ++p;
2194   wrap++;
2195   while (*p)
2196     {
2197     if (*p == '\\') p++;
2198     else if (!quotesmode && *p == wrap[-1]) depth++;
2199     else if (*p == *wrap)
2200       if (depth == 0)
2201         {
2202         *p = '\0';
2203         return s;
2204         }
2205       else
2206         depth--;
2207     p++;
2208     }
2209   }
2210 expand_string_message = string_sprintf("missing '%c'", *wrap);
2211 return NULL;
2212 }
2213
2214
2215 /* Pull off the leading array or object element, returning
2216 a copy in an allocated string.  Update the list pointer.
2217
2218 The element may itself be an abject or array.
2219 Return NULL when the list is empty.
2220 */
2221
2222 static uschar *
2223 json_nextinlist(const uschar ** list)
2224 {
2225 unsigned array_depth = 0, object_depth = 0;
2226 const uschar * s = *list, * item;
2227
2228 while (isspace(*s)) s++;
2229
2230 for (item = s;
2231      *s && (*s != ',' || array_depth != 0 || object_depth != 0);
2232      s++)
2233   switch (*s)
2234     {
2235     case '[': array_depth++; break;
2236     case ']': array_depth--; break;
2237     case '{': object_depth++; break;
2238     case '}': object_depth--; break;
2239     }
2240 *list = *s ? s+1 : s;
2241 if (item == s) return NULL;
2242 item = string_copyn(item, s - item);
2243 DEBUG(D_expand) debug_printf_indent("  json ele: '%s'\n", item);
2244 return US item;
2245 }
2246
2247
2248
2249 /*************************************************
2250 *        Read and evaluate a condition           *
2251 *************************************************/
2252
2253 /*
2254 Arguments:
2255   s        points to the start of the condition text
2256   resetok  points to a BOOL which is written false if it is unsafe to
2257            free memory. Certain condition types (acl) may have side-effect
2258            allocation which must be preserved.
2259   yield    points to a BOOL to hold the result of the condition test;
2260            if NULL, we are just reading through a condition that is
2261            part of an "or" combination to check syntax, or in a state
2262            where the answer isn't required
2263
2264 Returns:   a pointer to the first character after the condition, or
2265            NULL after an error
2266 */
2267
2268 static const uschar *
2269 eval_condition(const uschar *s, BOOL *resetok, BOOL *yield)
2270 {
2271 BOOL testfor = TRUE;
2272 BOOL tempcond, combined_cond;
2273 BOOL *subcondptr;
2274 BOOL sub2_honour_dollar = TRUE;
2275 BOOL is_forany, is_json, is_jsons;
2276 int rc, cond_type, roffset;
2277 int_eximarith_t num[2];
2278 struct stat statbuf;
2279 uschar name[256];
2280 const uschar *sub[10];
2281
2282 const pcre *re;
2283 const uschar *rerror;
2284
2285 for (;;)
2286   {
2287   while (isspace(*s)) s++;
2288   if (*s == '!') { testfor = !testfor; s++; } else break;
2289   }
2290
2291 /* Numeric comparisons are symbolic */
2292
2293 if (*s == '=' || *s == '>' || *s == '<')
2294   {
2295   int p = 0;
2296   name[p++] = *s++;
2297   if (*s == '=')
2298     {
2299     name[p++] = '=';
2300     s++;
2301     }
2302   name[p] = 0;
2303   }
2304
2305 /* All other conditions are named */
2306
2307 else s = read_name(name, 256, s, US"_");
2308
2309 /* If we haven't read a name, it means some non-alpha character is first. */
2310
2311 if (name[0] == 0)
2312   {
2313   expand_string_message = string_sprintf("condition name expected, "
2314     "but found \"%.16s\"", s);
2315   return NULL;
2316   }
2317
2318 /* Find which condition we are dealing with, and switch on it */
2319
2320 cond_type = chop_match(name, cond_table, nelem(cond_table));
2321 switch(cond_type)
2322   {
2323   /* def: tests for a non-empty variable, or for the existence of a header. If
2324   yield == NULL we are in a skipping state, and don't care about the answer. */
2325
2326   case ECOND_DEF:
2327     {
2328     uschar * t;
2329
2330     if (*s != ':')
2331       {
2332       expand_string_message = US"\":\" expected after \"def\"";
2333       return NULL;
2334       }
2335
2336     s = read_name(name, 256, s+1, US"_");
2337
2338     /* Test for a header's existence. If the name contains a closing brace
2339     character, this may be a user error where the terminating colon has been
2340     omitted. Set a flag to adjust a subsequent error message in this case. */
2341
2342     if (  ( *(t = name) == 'h'
2343           || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h'
2344           )
2345        && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0)
2346        )
2347       {
2348       s = read_header_name(name, 256, s);
2349       /* {-for-text-editors */
2350       if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2351       if (yield) *yield =
2352         (find_header(name, NULL, FH_EXISTS_ONLY, NULL) != NULL) == testfor;
2353       }
2354
2355     /* Test for a variable's having a non-empty value. A non-existent variable
2356     causes an expansion failure. */
2357
2358     else
2359       {
2360       if (!(t = find_variable(name, TRUE, yield == NULL, NULL)))
2361         {
2362         expand_string_message = (name[0] == 0)?
2363           string_sprintf("variable name omitted after \"def:\"") :
2364           string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2365         check_variable_error_message(name);
2366         return NULL;
2367         }
2368       if (yield) *yield = (t[0] != 0) == testfor;
2369       }
2370
2371     return s;
2372     }
2373
2374
2375   /* first_delivery tests for first delivery attempt */
2376
2377   case ECOND_FIRST_DELIVERY:
2378   if (yield != NULL) *yield = f.deliver_firsttime == testfor;
2379   return s;
2380
2381
2382   /* queue_running tests for any process started by a queue runner */
2383
2384   case ECOND_QUEUE_RUNNING:
2385   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2386   return s;
2387
2388
2389   /* exists:  tests for file existence
2390        isip:  tests for any IP address
2391       isip4:  tests for an IPv4 address
2392       isip6:  tests for an IPv6 address
2393         pam:  does PAM authentication
2394      radius:  does RADIUS authentication
2395    ldapauth:  does LDAP authentication
2396     pwcheck:  does Cyrus SASL pwcheck authentication
2397   */
2398
2399   case ECOND_EXISTS:
2400   case ECOND_ISIP:
2401   case ECOND_ISIP4:
2402   case ECOND_ISIP6:
2403   case ECOND_PAM:
2404   case ECOND_RADIUS:
2405   case ECOND_LDAPAUTH:
2406   case ECOND_PWCHECK:
2407
2408   while (isspace(*s)) s++;
2409   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2410
2411   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2412   if (sub[0] == NULL) return NULL;
2413   /* {-for-text-editors */
2414   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2415
2416   if (yield == NULL) return s;   /* No need to run the test if skipping */
2417
2418   switch(cond_type)
2419     {
2420     case ECOND_EXISTS:
2421     if ((expand_forbid & RDO_EXISTS) != 0)
2422       {
2423       expand_string_message = US"File existence tests are not permitted";
2424       return NULL;
2425       }
2426     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2427     break;
2428
2429     case ECOND_ISIP:
2430     case ECOND_ISIP4:
2431     case ECOND_ISIP6:
2432     rc = string_is_ip_address(sub[0], NULL);
2433     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2434              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2435     break;
2436
2437     /* Various authentication tests - all optionally compiled */
2438
2439     case ECOND_PAM:
2440     #ifdef SUPPORT_PAM
2441     rc = auth_call_pam(sub[0], &expand_string_message);
2442     goto END_AUTH;
2443     #else
2444     goto COND_FAILED_NOT_COMPILED;
2445     #endif  /* SUPPORT_PAM */
2446
2447     case ECOND_RADIUS:
2448     #ifdef RADIUS_CONFIG_FILE
2449     rc = auth_call_radius(sub[0], &expand_string_message);
2450     goto END_AUTH;
2451     #else
2452     goto COND_FAILED_NOT_COMPILED;
2453     #endif  /* RADIUS_CONFIG_FILE */
2454
2455     case ECOND_LDAPAUTH:
2456     #ifdef LOOKUP_LDAP
2457       {
2458       /* Just to keep the interface the same */
2459       BOOL do_cache;
2460       int old_pool = store_pool;
2461       store_pool = POOL_SEARCH;
2462       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2463         &expand_string_message, &do_cache);
2464       store_pool = old_pool;
2465       }
2466     goto END_AUTH;
2467     #else
2468     goto COND_FAILED_NOT_COMPILED;
2469     #endif  /* LOOKUP_LDAP */
2470
2471     case ECOND_PWCHECK:
2472     #ifdef CYRUS_PWCHECK_SOCKET
2473     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2474     goto END_AUTH;
2475     #else
2476     goto COND_FAILED_NOT_COMPILED;
2477     #endif  /* CYRUS_PWCHECK_SOCKET */
2478
2479     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2480         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2481     END_AUTH:
2482     if (rc == ERROR || rc == DEFER) return NULL;
2483     *yield = (rc == OK) == testfor;
2484     #endif
2485     }
2486   return s;
2487
2488
2489   /* call ACL (in a conditional context).  Accept true, deny false.
2490   Defer is a forced-fail.  Anything set by message= goes to $value.
2491   Up to ten parameters are used; we use the braces round the name+args
2492   like the saslauthd condition does, to permit a variable number of args.
2493   See also the expansion-item version EITEM_ACL and the traditional
2494   acl modifier ACLC_ACL.
2495   Since the ACL may allocate new global variables, tell our caller to not
2496   reclaim memory.
2497   */
2498
2499   case ECOND_ACL:
2500     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2501     {
2502     uschar *sub[10];
2503     uschar *user_msg;
2504     BOOL cond = FALSE;
2505
2506     while (isspace(*s)) s++;
2507     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2508
2509     switch(read_subs(sub, nelem(sub), 1,
2510       &s, yield == NULL, TRUE, US"acl", resetok))
2511       {
2512       case 1: expand_string_message = US"too few arguments or bracketing "
2513         "error for acl";
2514       case 2:
2515       case 3: return NULL;
2516       }
2517
2518     if (yield != NULL)
2519       {
2520       int rc;
2521       *resetok = FALSE; /* eval_acl() might allocate; do not reclaim */
2522       switch(rc = eval_acl(sub, nelem(sub), &user_msg))
2523         {
2524         case OK:
2525           cond = TRUE;
2526         case FAIL:
2527           lookup_value = NULL;
2528           if (user_msg)
2529             lookup_value = string_copy(user_msg);
2530           *yield = cond == testfor;
2531           break;
2532
2533         case DEFER:
2534           f.expand_string_forcedfail = TRUE;
2535           /*FALLTHROUGH*/
2536         default:
2537           expand_string_message = string_sprintf("%s from acl \"%s\"",
2538             rc_names[rc], sub[0]);
2539           return NULL;
2540         }
2541       }
2542     return s;
2543     }
2544
2545
2546   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2547
2548      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2549
2550   However, the last two are optional. That is why the whole set is enclosed
2551   in their own set of braces. */
2552
2553   case ECOND_SASLAUTHD:
2554 #ifndef CYRUS_SASLAUTHD_SOCKET
2555     goto COND_FAILED_NOT_COMPILED;
2556 #else
2557     {
2558     uschar *sub[4];
2559     while (isspace(*s)) s++;
2560     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2561     switch(read_subs(sub, nelem(sub), 2, &s, yield == NULL, TRUE, US"saslauthd",
2562                     resetok))
2563       {
2564       case 1: expand_string_message = US"too few arguments or bracketing "
2565         "error for saslauthd";
2566       case 2:
2567       case 3: return NULL;
2568       }
2569     if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2570     if (yield != NULL)
2571       {
2572       int rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2573         &expand_string_message);
2574       if (rc == ERROR || rc == DEFER) return NULL;
2575       *yield = (rc == OK) == testfor;
2576       }
2577     return s;
2578     }
2579 #endif /* CYRUS_SASLAUTHD_SOCKET */
2580
2581
2582   /* symbolic operators for numeric and string comparison, and a number of
2583   other operators, all requiring two arguments.
2584
2585   crypteq:           encrypts plaintext and compares against an encrypted text,
2586                        using crypt(), crypt16(), MD5 or SHA-1
2587   inlist/inlisti:    checks if first argument is in the list of the second
2588   match:             does a regular expression match and sets up the numerical
2589                        variables if it succeeds
2590   match_address:     matches in an address list
2591   match_domain:      matches in a domain list
2592   match_ip:          matches a host list that is restricted to IP addresses
2593   match_local_part:  matches in a local part list
2594   */
2595
2596   case ECOND_MATCH_ADDRESS:
2597   case ECOND_MATCH_DOMAIN:
2598   case ECOND_MATCH_IP:
2599   case ECOND_MATCH_LOCAL_PART:
2600 #ifndef EXPAND_LISTMATCH_RHS
2601     sub2_honour_dollar = FALSE;
2602 #endif
2603     /* FALLTHROUGH */
2604
2605   case ECOND_CRYPTEQ:
2606   case ECOND_INLIST:
2607   case ECOND_INLISTI:
2608   case ECOND_MATCH:
2609
2610   case ECOND_NUM_L:     /* Numerical comparisons */
2611   case ECOND_NUM_LE:
2612   case ECOND_NUM_E:
2613   case ECOND_NUM_EE:
2614   case ECOND_NUM_G:
2615   case ECOND_NUM_GE:
2616
2617   case ECOND_STR_LT:    /* String comparisons */
2618   case ECOND_STR_LTI:
2619   case ECOND_STR_LE:
2620   case ECOND_STR_LEI:
2621   case ECOND_STR_EQ:
2622   case ECOND_STR_EQI:
2623   case ECOND_STR_GT:
2624   case ECOND_STR_GTI:
2625   case ECOND_STR_GE:
2626   case ECOND_STR_GEI:
2627
2628   for (int i = 0; i < 2; i++)
2629     {
2630     /* Sometimes, we don't expand substrings; too many insecure configurations
2631     created using match_address{}{} and friends, where the second param
2632     includes information from untrustworthy sources. */
2633     BOOL honour_dollar = TRUE;
2634     if ((i > 0) && !sub2_honour_dollar)
2635       honour_dollar = FALSE;
2636
2637     while (isspace(*s)) s++;
2638     if (*s != '{')
2639       {
2640       if (i == 0) goto COND_FAILED_CURLY_START;
2641       expand_string_message = string_sprintf("missing 2nd string in {} "
2642         "after \"%s\"", name);
2643       return NULL;
2644       }
2645     if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2646         honour_dollar, resetok)))
2647       return NULL;
2648     DEBUG(D_expand) if (i == 1 && !sub2_honour_dollar && Ustrchr(sub[1], '$'))
2649       debug_printf_indent("WARNING: the second arg is NOT expanded,"
2650                         " for security reasons\n");
2651     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2652
2653     /* Convert to numerical if required; we know that the names of all the
2654     conditions that compare numbers do not start with a letter. This just saves
2655     checking for them individually. */
2656
2657     if (!isalpha(name[0]) && yield != NULL)
2658       if (sub[i][0] == 0)
2659         {
2660         num[i] = 0;
2661         DEBUG(D_expand)
2662           debug_printf_indent("empty string cast to zero for numerical comparison\n");
2663         }
2664       else
2665         {
2666         num[i] = expanded_string_integer(sub[i], FALSE);
2667         if (expand_string_message != NULL) return NULL;
2668         }
2669     }
2670
2671   /* Result not required */
2672
2673   if (yield == NULL) return s;
2674
2675   /* Do an appropriate comparison */
2676
2677   switch(cond_type)
2678     {
2679     case ECOND_NUM_E:
2680     case ECOND_NUM_EE:
2681     tempcond = (num[0] == num[1]);
2682     break;
2683
2684     case ECOND_NUM_G:
2685     tempcond = (num[0] > num[1]);
2686     break;
2687
2688     case ECOND_NUM_GE:
2689     tempcond = (num[0] >= num[1]);
2690     break;
2691
2692     case ECOND_NUM_L:
2693     tempcond = (num[0] < num[1]);
2694     break;
2695
2696     case ECOND_NUM_LE:
2697     tempcond = (num[0] <= num[1]);
2698     break;
2699
2700     case ECOND_STR_LT:
2701     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2702     break;
2703
2704     case ECOND_STR_LTI:
2705     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2706     break;
2707
2708     case ECOND_STR_LE:
2709     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2710     break;
2711
2712     case ECOND_STR_LEI:
2713     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2714     break;
2715
2716     case ECOND_STR_EQ:
2717     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2718     break;
2719
2720     case ECOND_STR_EQI:
2721     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2722     break;
2723
2724     case ECOND_STR_GT:
2725     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2726     break;
2727
2728     case ECOND_STR_GTI:
2729     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2730     break;
2731
2732     case ECOND_STR_GE:
2733     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2734     break;
2735
2736     case ECOND_STR_GEI:
2737     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2738     break;
2739
2740     case ECOND_MATCH:   /* Regular expression match */
2741     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2742       NULL);
2743     if (re == NULL)
2744       {
2745       expand_string_message = string_sprintf("regular expression error in "
2746         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2747       return NULL;
2748       }
2749     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2750     break;
2751
2752     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2753     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2754     goto MATCHED_SOMETHING;
2755
2756     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2757     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2758       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2759     goto MATCHED_SOMETHING;
2760
2761     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2762     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2763       {
2764       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2765         sub[0]);
2766       return NULL;
2767       }
2768     else
2769       {
2770       unsigned int *nullcache = NULL;
2771       check_host_block cb;
2772
2773       cb.host_name = US"";
2774       cb.host_address = sub[0];
2775
2776       /* If the host address starts off ::ffff: it is an IPv6 address in
2777       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2778       addresses. */
2779
2780       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2781         cb.host_address + 7 : cb.host_address;
2782
2783       rc = match_check_list(
2784              &sub[1],                   /* the list */
2785              0,                         /* separator character */
2786              &hostlist_anchor,          /* anchor pointer */
2787              &nullcache,                /* cache pointer */
2788              check_host,                /* function for testing */
2789              &cb,                       /* argument for function */
2790              MCL_HOST,                  /* type of check */
2791              sub[0],                    /* text for debugging */
2792              NULL);                     /* where to pass back data */
2793       }
2794     goto MATCHED_SOMETHING;
2795
2796     case ECOND_MATCH_LOCAL_PART:
2797     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2798       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2799     /* Fall through */
2800     /* VVVVVVVVVVVV */
2801     MATCHED_SOMETHING:
2802     switch(rc)
2803       {
2804       case OK:
2805       tempcond = TRUE;
2806       break;
2807
2808       case FAIL:
2809       tempcond = FALSE;
2810       break;
2811
2812       case DEFER:
2813       expand_string_message = string_sprintf("unable to complete match "
2814         "against \"%s\": %s", sub[1], search_error_message);
2815       return NULL;
2816       }
2817
2818     break;
2819
2820     /* Various "encrypted" comparisons. If the second string starts with
2821     "{" then an encryption type is given. Default to crypt() or crypt16()
2822     (build-time choice). */
2823     /* }-for-text-editors */
2824
2825     case ECOND_CRYPTEQ:
2826     #ifndef SUPPORT_CRYPTEQ
2827     goto COND_FAILED_NOT_COMPILED;
2828     #else
2829     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2830       {
2831       int sublen = Ustrlen(sub[1]+5);
2832       md5 base;
2833       uschar digest[16];
2834
2835       md5_start(&base);
2836       md5_end(&base, sub[0], Ustrlen(sub[0]), digest);
2837
2838       /* If the length that we are comparing against is 24, the MD5 digest
2839       is expressed as a base64 string. This is the way LDAP does it. However,
2840       some other software uses a straightforward hex representation. We assume
2841       this if the length is 32. Other lengths fail. */
2842
2843       if (sublen == 24)
2844         {
2845         uschar *coded = b64encode(CUS digest, 16);
2846         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2847           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2848         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2849         }
2850       else if (sublen == 32)
2851         {
2852         uschar coded[36];
2853         for (int i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2854         coded[32] = 0;
2855         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2856           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2857         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2858         }
2859       else
2860         {
2861         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2862           "fail\n  crypted=%s\n", sub[1]+5);
2863         tempcond = FALSE;
2864         }
2865       }
2866
2867     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2868       {
2869       int sublen = Ustrlen(sub[1]+6);
2870       hctx h;
2871       uschar digest[20];
2872
2873       sha1_start(&h);
2874       sha1_end(&h, sub[0], Ustrlen(sub[0]), digest);
2875
2876       /* If the length that we are comparing against is 28, assume the SHA1
2877       digest is expressed as a base64 string. If the length is 40, assume a
2878       straightforward hex representation. Other lengths fail. */
2879
2880       if (sublen == 28)
2881         {
2882         uschar *coded = b64encode(CUS digest, 20);
2883         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2884           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2885         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2886         }
2887       else if (sublen == 40)
2888         {
2889         uschar coded[44];
2890         for (int i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2891         coded[40] = 0;
2892         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2893           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2894         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2895         }
2896       else
2897         {
2898         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2899           "fail\n  crypted=%s\n", sub[1]+6);
2900         tempcond = FALSE;
2901         }
2902       }
2903
2904     else   /* {crypt} or {crypt16} and non-{ at start */
2905            /* }-for-text-editors */
2906       {
2907       int which = 0;
2908       uschar *coded;
2909
2910       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2911         {
2912         sub[1] += 7;
2913         which = 1;
2914         }
2915       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2916         {
2917         sub[1] += 9;
2918         which = 2;
2919         }
2920       else if (sub[1][0] == '{')                /* }-for-text-editors */
2921         {
2922         expand_string_message = string_sprintf("unknown encryption mechanism "
2923           "in \"%s\"", sub[1]);
2924         return NULL;
2925         }
2926
2927       switch(which)
2928         {
2929         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2930         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2931         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2932         }
2933
2934       #define STR(s) # s
2935       #define XSTR(s) STR(s)
2936       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2937         "  subject=%s\n  crypted=%s\n",
2938         which == 0 ? XSTR(DEFAULT_CRYPT) : which == 1 ? "crypt" : "crypt16",
2939         coded, sub[1]);
2940       #undef STR
2941       #undef XSTR
2942
2943       /* If the encrypted string contains fewer than two characters (for the
2944       salt), force failure. Otherwise we get false positives: with an empty
2945       string the yield of crypt() is an empty string! */
2946
2947       if (coded)
2948         tempcond = Ustrlen(sub[1]) < 2 ? FALSE : Ustrcmp(coded, sub[1]) == 0;
2949       else if (errno == EINVAL)
2950         tempcond = FALSE;
2951       else
2952         {
2953         expand_string_message = string_sprintf("crypt error: %s\n",
2954           US strerror(errno));
2955         return NULL;
2956         }
2957       }
2958     break;
2959     #endif  /* SUPPORT_CRYPTEQ */
2960
2961     case ECOND_INLIST:
2962     case ECOND_INLISTI:
2963       {
2964       const uschar * list = sub[1];
2965       int sep = 0;
2966       uschar *save_iterate_item = iterate_item;
2967       int (*compare)(const uschar *, const uschar *);
2968
2969       DEBUG(D_expand) debug_printf_indent("condition: %s  item: %s\n", name, sub[0]);
2970
2971       tempcond = FALSE;
2972       compare = cond_type == ECOND_INLISTI
2973         ? strcmpic : (int (*)(const uschar *, const uschar *)) strcmp;
2974
2975       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
2976         {
2977         DEBUG(D_expand) debug_printf_indent(" compare %s\n", iterate_item);
2978         if (compare(sub[0], iterate_item) == 0)
2979           {
2980           tempcond = TRUE;
2981           break;
2982           }
2983         }
2984       iterate_item = save_iterate_item;
2985       }
2986
2987     }   /* Switch for comparison conditions */
2988
2989   *yield = tempcond == testfor;
2990   return s;    /* End of comparison conditions */
2991
2992
2993   /* and/or: computes logical and/or of several conditions */
2994
2995   case ECOND_AND:
2996   case ECOND_OR:
2997   subcondptr = (yield == NULL)? NULL : &tempcond;
2998   combined_cond = (cond_type == ECOND_AND);
2999
3000   while (isspace(*s)) s++;
3001   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
3002
3003   for (;;)
3004     {
3005     while (isspace(*s)) s++;
3006     /* {-for-text-editors */
3007     if (*s == '}') break;
3008     if (*s != '{')                                      /* }-for-text-editors */
3009       {
3010       expand_string_message = string_sprintf("each subcondition "
3011         "inside an \"%s{...}\" condition must be in its own {}", name);
3012       return NULL;
3013       }
3014
3015     if (!(s = eval_condition(s+1, resetok, subcondptr)))
3016       {
3017       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
3018         expand_string_message, name);
3019       return NULL;
3020       }
3021     while (isspace(*s)) s++;
3022
3023     /* {-for-text-editors */
3024     if (*s++ != '}')
3025       {
3026       /* {-for-text-editors */
3027       expand_string_message = string_sprintf("missing } at end of condition "
3028         "inside \"%s\" group", name);
3029       return NULL;
3030       }
3031
3032     if (yield != NULL)
3033       {
3034       if (cond_type == ECOND_AND)
3035         {
3036         combined_cond &= tempcond;
3037         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
3038         }                                       /* evaluate any more */
3039       else
3040         {
3041         combined_cond |= tempcond;
3042         if (combined_cond) subcondptr = NULL;   /* once true, don't */
3043         }                                       /* evaluate any more */
3044       }
3045     }
3046
3047   if (yield != NULL) *yield = (combined_cond == testfor);
3048   return ++s;
3049
3050
3051   /* forall/forany: iterates a condition with different values */
3052
3053   case ECOND_FORALL:      is_forany = FALSE;  is_json = FALSE; is_jsons = FALSE; goto FORMANY;
3054   case ECOND_FORANY:      is_forany = TRUE;   is_json = FALSE; is_jsons = FALSE; goto FORMANY;
3055   case ECOND_FORALL_JSON: is_forany = FALSE;  is_json = TRUE;  is_jsons = FALSE; goto FORMANY;
3056   case ECOND_FORANY_JSON: is_forany = TRUE;   is_json = TRUE;  is_jsons = FALSE; goto FORMANY;
3057   case ECOND_FORALL_JSONS: is_forany = FALSE; is_json = TRUE;  is_jsons = TRUE;  goto FORMANY;
3058   case ECOND_FORANY_JSONS: is_forany = TRUE;  is_json = TRUE;  is_jsons = TRUE;  goto FORMANY;
3059
3060   FORMANY:
3061     {
3062     const uschar * list;
3063     int sep = 0;
3064     uschar *save_iterate_item = iterate_item;
3065
3066     DEBUG(D_expand) debug_printf_indent("condition: %s\n", name);
3067
3068     while (isspace(*s)) s++;
3069     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
3070     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
3071     if (sub[0] == NULL) return NULL;
3072     /* {-for-text-editors */
3073     if (*s++ != '}') goto COND_FAILED_CURLY_END;
3074
3075     while (isspace(*s)) s++;
3076     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
3077
3078     sub[1] = s;
3079
3080     /* Call eval_condition once, with result discarded (as if scanning a
3081     "false" part). This allows us to find the end of the condition, because if
3082     the list it empty, we won't actually evaluate the condition for real. */
3083
3084     if (!(s = eval_condition(sub[1], resetok, NULL)))
3085       {
3086       expand_string_message = string_sprintf("%s inside \"%s\" condition",
3087         expand_string_message, name);
3088       return NULL;
3089       }
3090     while (isspace(*s)) s++;
3091
3092     /* {-for-text-editors */
3093     if (*s++ != '}')
3094       {
3095       /* {-for-text-editors */
3096       expand_string_message = string_sprintf("missing } at end of condition "
3097         "inside \"%s\"", name);
3098       return NULL;
3099       }
3100
3101     if (yield) *yield = !testfor;
3102     list = sub[0];
3103     if (is_json) list = dewrap(string_copy(list), US"[]");
3104     while ((iterate_item = is_json
3105       ? json_nextinlist(&list) : string_nextinlist(&list, &sep, NULL, 0)))
3106       {
3107       if (is_jsons)
3108         if (!(iterate_item = dewrap(iterate_item, US"\"\"")))
3109           {
3110           expand_string_message =
3111             string_sprintf("%s wrapping string result for extract jsons",
3112               expand_string_message);
3113           iterate_item = save_iterate_item;
3114           return NULL;
3115           }
3116
3117       DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, iterate_item);
3118       if (!eval_condition(sub[1], resetok, &tempcond))
3119         {
3120         expand_string_message = string_sprintf("%s inside \"%s\" condition",
3121           expand_string_message, name);
3122         iterate_item = save_iterate_item;
3123         return NULL;
3124         }
3125       DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", name,
3126         tempcond? "true":"false");
3127
3128       if (yield) *yield = (tempcond == testfor);
3129       if (tempcond == is_forany) break;
3130       }
3131
3132     iterate_item = save_iterate_item;
3133     return s;
3134     }
3135
3136
3137   /* The bool{} expansion condition maps a string to boolean.
3138   The values supported should match those supported by the ACL condition
3139   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
3140   of true/false.  Note that Router "condition" rules have a different
3141   interpretation, where general data can be used and only a few values
3142   map to FALSE.
3143   Note that readconf.c boolean matching, for boolean configuration options,
3144   only matches true/yes/false/no.
3145   The bool_lax{} condition matches the Router logic, which is much more
3146   liberal. */
3147   case ECOND_BOOL:
3148   case ECOND_BOOL_LAX:
3149     {
3150     uschar *sub_arg[1];
3151     uschar *t, *t2;
3152     uschar *ourname;
3153     size_t len;
3154     BOOL boolvalue = FALSE;
3155     while (isspace(*s)) s++;
3156     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
3157     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
3158     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
3159       {
3160       case 1: expand_string_message = string_sprintf(
3161                   "too few arguments or bracketing error for %s",
3162                   ourname);
3163       /*FALLTHROUGH*/
3164       case 2:
3165       case 3: return NULL;
3166       }
3167     t = sub_arg[0];
3168     while (isspace(*t)) t++;
3169     len = Ustrlen(t);
3170     if (len)
3171       {
3172       /* trailing whitespace: seems like a good idea to ignore it too */
3173       t2 = t + len - 1;
3174       while (isspace(*t2)) t2--;
3175       if (t2 != (t + len))
3176         {
3177         *++t2 = '\0';
3178         len = t2 - t;
3179         }
3180       }
3181     DEBUG(D_expand)
3182       debug_printf_indent("considering %s: %s\n", ourname, len ? t : US"<empty>");
3183     /* logic for the lax case from expand_check_condition(), which also does
3184     expands, and the logic is both short and stable enough that there should
3185     be no maintenance burden from replicating it. */
3186     if (len == 0)
3187       boolvalue = FALSE;
3188     else if (*t == '-'
3189              ? Ustrspn(t+1, "0123456789") == len-1
3190              : Ustrspn(t,   "0123456789") == len)
3191       {
3192       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
3193       /* expand_check_condition only does a literal string "0" check */
3194       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
3195         boolvalue = TRUE;
3196       }
3197     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
3198       boolvalue = TRUE;
3199     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
3200       boolvalue = FALSE;
3201     else if (cond_type == ECOND_BOOL_LAX)
3202       boolvalue = TRUE;
3203     else
3204       {
3205       expand_string_message = string_sprintf("unrecognised boolean "
3206        "value \"%s\"", t);
3207       return NULL;
3208       }
3209     DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", ourname,
3210         boolvalue? "true":"false");
3211     if (yield != NULL) *yield = (boolvalue == testfor);
3212     return s;
3213     }
3214
3215   /* Unknown condition */
3216
3217   default:
3218   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
3219   return NULL;
3220   }   /* End switch on condition type */
3221
3222 /* Missing braces at start and end of data */
3223
3224 COND_FAILED_CURLY_START:
3225 expand_string_message = string_sprintf("missing { after \"%s\"", name);
3226 return NULL;
3227
3228 COND_FAILED_CURLY_END:
3229 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
3230   name);
3231 return NULL;
3232
3233 /* A condition requires code that is not compiled */
3234
3235 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
3236     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
3237     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
3238 COND_FAILED_NOT_COMPILED:
3239 expand_string_message = string_sprintf("support for \"%s\" not compiled",
3240   name);
3241 return NULL;
3242 #endif
3243 }
3244
3245
3246
3247
3248 /*************************************************
3249 *          Save numerical variables              *
3250 *************************************************/
3251
3252 /* This function is called from items such as "if" that want to preserve and
3253 restore the numbered variables.
3254
3255 Arguments:
3256   save_expand_string    points to an array of pointers to set
3257   save_expand_nlength   points to an array of ints for the lengths
3258
3259 Returns:                the value of expand max to save
3260 */
3261
3262 static int
3263 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3264 {
3265 for (int i = 0; i <= expand_nmax; i++)
3266   {
3267   save_expand_nstring[i] = expand_nstring[i];
3268   save_expand_nlength[i] = expand_nlength[i];
3269   }
3270 return expand_nmax;
3271 }
3272
3273
3274
3275 /*************************************************
3276 *           Restore numerical variables          *
3277 *************************************************/
3278
3279 /* This function restored saved values of numerical strings.
3280
3281 Arguments:
3282   save_expand_nmax      the number of strings to restore
3283   save_expand_string    points to an array of pointers
3284   save_expand_nlength   points to an array of ints
3285
3286 Returns:                nothing
3287 */
3288
3289 static void
3290 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3291   int *save_expand_nlength)
3292 {
3293 expand_nmax = save_expand_nmax;
3294 for (int i = 0; i <= expand_nmax; i++)
3295   {
3296   expand_nstring[i] = save_expand_nstring[i];
3297   expand_nlength[i] = save_expand_nlength[i];
3298   }
3299 }
3300
3301
3302
3303
3304
3305 /*************************************************
3306 *            Handle yes/no substrings            *
3307 *************************************************/
3308
3309 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3310 alternative substrings that depend on whether or not the condition was true,
3311 or the lookup or extraction succeeded. The substrings always have to be
3312 expanded, to check their syntax, but "skipping" is set when the result is not
3313 needed - this avoids unnecessary nested lookups.
3314
3315 Arguments:
3316   skipping       TRUE if we were skipping when this item was reached
3317   yes            TRUE if the first string is to be used, else use the second
3318   save_lookup    a value to put back into lookup_value before the 2nd expansion
3319   sptr           points to the input string pointer
3320   yieldptr       points to the output growable-string pointer
3321   type           "lookup", "if", "extract", "run", "env", "listextract" or
3322                  "certextract" for error message
3323   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3324                 the store.
3325
3326 Returns:         0 OK; lookup_value has been reset to save_lookup
3327                  1 expansion failed
3328                  2 expansion failed because of bracketing error
3329 */
3330
3331 static int
3332 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr,
3333   gstring ** yieldptr, uschar *type, BOOL *resetok)
3334 {
3335 int rc = 0;
3336 const uschar *s = *sptr;    /* Local value */
3337 uschar *sub1, *sub2;
3338 const uschar * errwhere;
3339
3340 /* If there are no following strings, we substitute the contents of $value for
3341 lookups and for extractions in the success case. For the ${if item, the string
3342 "true" is substituted. In the fail case, nothing is substituted for all three
3343 items. */
3344
3345 while (isspace(*s)) s++;
3346 if (*s == '}')
3347   {
3348   if (type[0] == 'i')
3349     {
3350     if (yes && !skipping)
3351       *yieldptr = string_catn(*yieldptr, US"true", 4);
3352     }
3353   else
3354     {
3355     if (yes && lookup_value && !skipping)
3356       *yieldptr = string_cat(*yieldptr, lookup_value);
3357     lookup_value = save_lookup;
3358     }
3359   s++;
3360   goto RETURN;
3361   }
3362
3363 /* The first following string must be braced. */
3364
3365 if (*s++ != '{')
3366   {
3367   errwhere = US"'yes' part did not start with '{'";
3368   goto FAILED_CURLY;
3369   }
3370
3371 /* Expand the first substring. Forced failures are noticed only if we actually
3372 want this string. Set skipping in the call in the fail case (this will always
3373 be the case if we were already skipping). */
3374
3375 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3376 if (sub1 == NULL && (yes || !f.expand_string_forcedfail)) goto FAILED;
3377 f.expand_string_forcedfail = FALSE;
3378 if (*s++ != '}')
3379   {
3380   errwhere = US"'yes' part did not end with '}'";
3381   goto FAILED_CURLY;
3382   }
3383
3384 /* If we want the first string, add it to the output */
3385
3386 if (yes)
3387   *yieldptr = string_cat(*yieldptr, sub1);
3388
3389 /* If this is called from a lookup/env or a (cert)extract, we want to restore
3390 $value to what it was at the start of the item, so that it has this value
3391 during the second string expansion. For the call from "if" or "run" to this
3392 function, save_lookup is set to lookup_value, so that this statement does
3393 nothing. */
3394
3395 lookup_value = save_lookup;
3396
3397 /* There now follows either another substring, or "fail", or nothing. This
3398 time, forced failures are noticed only if we want the second string. We must
3399 set skipping in the nested call if we don't want this string, or if we were
3400 already skipping. */
3401
3402 while (isspace(*s)) s++;
3403 if (*s == '{')
3404   {
3405   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3406   if (sub2 == NULL && (!yes || !f.expand_string_forcedfail)) goto FAILED;
3407   f.expand_string_forcedfail = FALSE;
3408   if (*s++ != '}')
3409     {
3410     errwhere = US"'no' part did not start with '{'";
3411     goto FAILED_CURLY;
3412     }
3413
3414   /* If we want the second string, add it to the output */
3415
3416   if (!yes)
3417     *yieldptr = string_cat(*yieldptr, sub2);
3418   }
3419
3420 /* If there is no second string, but the word "fail" is present when the use of
3421 the second string is wanted, set a flag indicating it was a forced failure
3422 rather than a syntactic error. Swallow the terminating } in case this is nested
3423 inside another lookup or if or extract. */
3424
3425 else if (*s != '}')
3426   {
3427   uschar name[256];
3428   /* deconst cast ok here as source is s anyway */
3429   s = US read_name(name, sizeof(name), s, US"_");
3430   if (Ustrcmp(name, "fail") == 0)
3431     {
3432     if (!yes && !skipping)
3433       {
3434       while (isspace(*s)) s++;
3435       if (*s++ != '}')
3436         {
3437         errwhere = US"did not close with '}' after forcedfail";
3438         goto FAILED_CURLY;
3439         }
3440       expand_string_message =
3441         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3442       f.expand_string_forcedfail = TRUE;
3443       goto FAILED;
3444       }
3445     }
3446   else
3447     {
3448     expand_string_message =
3449       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3450     goto FAILED;
3451     }
3452   }
3453
3454 /* All we have to do now is to check on the final closing brace. */
3455
3456 while (isspace(*s)) s++;
3457 if (*s++ != '}')
3458   {
3459   errwhere = US"did not close with '}'";
3460   goto FAILED_CURLY;
3461   }
3462
3463
3464 RETURN:
3465 /* Update the input pointer value before returning */
3466 *sptr = s;
3467 return rc;
3468
3469 FAILED_CURLY:
3470   /* Get here if there is a bracketing failure */
3471   expand_string_message = string_sprintf(
3472     "curly-bracket problem in conditional yes/no parsing: %s\n"
3473     " remaining string is '%s'", errwhere, --s);
3474   rc = 2;
3475   goto RETURN;
3476
3477 FAILED:
3478   /* Get here for other failures */
3479   rc = 1;
3480   goto RETURN;
3481 }
3482
3483
3484
3485
3486 /*************************************************
3487 *    Handle MD5 or SHA-1 computation for HMAC    *
3488 *************************************************/
3489
3490 /* These are some wrapping functions that enable the HMAC code to be a bit
3491 cleaner. A good compiler will spot the tail recursion.
3492
3493 Arguments:
3494   type         HMAC_MD5 or HMAC_SHA1
3495   remaining    are as for the cryptographic hash functions
3496
3497 Returns:       nothing
3498 */
3499
3500 static void
3501 chash_start(int type, void *base)
3502 {
3503 if (type == HMAC_MD5)
3504   md5_start((md5 *)base);
3505 else
3506   sha1_start((hctx *)base);
3507 }
3508
3509 static void
3510 chash_mid(int type, void *base, uschar *string)
3511 {
3512 if (type == HMAC_MD5)
3513   md5_mid((md5 *)base, string);
3514 else
3515   sha1_mid((hctx *)base, string);
3516 }
3517
3518 static void
3519 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3520 {
3521 if (type == HMAC_MD5)
3522   md5_end((md5 *)base, string, length, digest);
3523 else
3524   sha1_end((hctx *)base, string, length, digest);
3525 }
3526
3527
3528
3529
3530
3531 /********************************************************
3532 * prvs: Get last three digits of days since Jan 1, 1970 *
3533 ********************************************************/
3534
3535 /* This is needed to implement the "prvs" BATV reverse
3536    path signing scheme
3537
3538 Argument: integer "days" offset to add or substract to
3539           or from the current number of days.
3540
3541 Returns:  pointer to string containing the last three
3542           digits of the number of days since Jan 1, 1970,
3543           modified by the offset argument, NULL if there
3544           was an error in the conversion.
3545
3546 */
3547
3548 static uschar *
3549 prvs_daystamp(int day_offset)
3550 {
3551 uschar *days = store_get(32);                /* Need at least 24 for cases */
3552 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3553   (time(NULL) + day_offset*86400)/86400);
3554 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3555 }
3556
3557
3558
3559 /********************************************************
3560 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3561 ********************************************************/
3562
3563 /* This is needed to implement the "prvs" BATV reverse
3564    path signing scheme
3565
3566 Arguments:
3567   address RFC2821 Address to use
3568       key The key to use (must be less than 64 characters
3569           in size)
3570   key_num Single-digit key number to use. Defaults to
3571           '0' when NULL.
3572
3573 Returns:  pointer to string containing the first three
3574           bytes of the final hash in hex format, NULL if
3575           there was an error in the process.
3576 */
3577
3578 static uschar *
3579 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3580 {
3581 gstring * hash_source;
3582 uschar * p;
3583 hctx h;
3584 uschar innerhash[20];
3585 uschar finalhash[20];
3586 uschar innerkey[64];
3587 uschar outerkey[64];
3588 uschar *finalhash_hex = store_get(40);
3589
3590 if (key_num == NULL)
3591   key_num = US"0";
3592
3593 if (Ustrlen(key) > 64)
3594   return NULL;
3595
3596 hash_source = string_catn(NULL, key_num, 1);
3597 hash_source = string_catn(hash_source, daystamp, 3);
3598 hash_source = string_cat(hash_source, address);
3599 (void) string_from_gstring(hash_source);
3600
3601 DEBUG(D_expand)
3602   debug_printf_indent("prvs: hash source is '%s'\n", hash_source->s);
3603
3604 memset(innerkey, 0x36, 64);
3605 memset(outerkey, 0x5c, 64);
3606
3607 for (int i = 0; i < Ustrlen(key); i++)
3608   {
3609   innerkey[i] ^= key[i];
3610   outerkey[i] ^= key[i];
3611   }
3612
3613 chash_start(HMAC_SHA1, &h);
3614 chash_mid(HMAC_SHA1, &h, innerkey);
3615 chash_end(HMAC_SHA1, &h, hash_source->s, hash_source->ptr, innerhash);
3616
3617 chash_start(HMAC_SHA1, &h);
3618 chash_mid(HMAC_SHA1, &h, outerkey);
3619 chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash);
3620
3621 p = finalhash_hex;
3622 for (int i = 0; i < 3; i++)
3623   {
3624   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3625   *p++ = hex_digits[finalhash[i] & 0x0f];
3626   }
3627 *p = '\0';
3628
3629 return finalhash_hex;
3630 }
3631
3632
3633
3634
3635 /*************************************************
3636 *        Join a file onto the output string      *
3637 *************************************************/
3638
3639 /* This is used for readfile/readsock and after a run expansion.
3640 It joins the contents of a file onto the output string, globally replacing
3641 newlines with a given string (optionally).
3642
3643 Arguments:
3644   f            the FILE
3645   yield        pointer to the expandable string struct
3646   eol          newline replacement string, or NULL
3647
3648 Returns:       new pointer for expandable string, terminated if non-null
3649 */
3650
3651 static gstring *
3652 cat_file(FILE *f, gstring *yield, uschar *eol)
3653 {
3654 uschar buffer[1024];
3655
3656 while (Ufgets(buffer, sizeof(buffer), f))
3657   {
3658   int len = Ustrlen(buffer);
3659   if (eol && buffer[len-1] == '\n') len--;
3660   yield = string_catn(yield, buffer, len);
3661   if (eol && buffer[len])
3662     yield = string_cat(yield, eol);
3663   }
3664
3665 (void) string_from_gstring(yield);
3666 return yield;
3667 }
3668
3669
3670 #ifndef DISABLE_TLS
3671 static gstring *
3672 cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol)
3673 {
3674 int rc;
3675 uschar buffer[1024];
3676
3677 /*XXX could we read direct into a pre-grown string? */
3678
3679 while ((rc = tls_read(tls_ctx, buffer, sizeof(buffer))) > 0)
3680   for (uschar * s = buffer; rc--; s++)
3681     yield = eol && *s == '\n'
3682       ? string_cat(yield, eol) : string_catn(yield, s, 1);
3683
3684 /* We assume that all errors, and any returns of zero bytes,
3685 are actually EOF. */
3686
3687 (void) string_from_gstring(yield);
3688 return yield;
3689 }
3690 #endif
3691
3692
3693 /*************************************************
3694 *          Evaluate numeric expression           *
3695 *************************************************/
3696
3697 /* This is a set of mutually recursive functions that evaluate an arithmetic
3698 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3699 these functions that is called from elsewhere is eval_expr, whose interface is:
3700
3701 Arguments:
3702   sptr        pointer to the pointer to the string - gets updated
3703   decimal     TRUE if numbers are to be assumed decimal
3704   error       pointer to where to put an error message - must be NULL on input
3705   endket      TRUE if ')' must terminate - FALSE for external call
3706
3707 Returns:      on success: the value of the expression, with *error still NULL
3708               on failure: an undefined value, with *error = a message
3709 */
3710
3711 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3712
3713
3714 static int_eximarith_t
3715 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3716 {
3717 uschar *s = *sptr;
3718 int_eximarith_t x = eval_op_or(&s, decimal, error);
3719
3720 if (!*error)
3721   if (endket)
3722     if (*s != ')')
3723       *error = US"expecting closing parenthesis";
3724     else
3725       while (isspace(*(++s)));
3726   else if (*s)
3727     *error = US"expecting operator";
3728 *sptr = s;
3729 return x;
3730 }
3731
3732
3733 static int_eximarith_t
3734 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3735 {
3736 int c;
3737 int_eximarith_t n;
3738 uschar *s = *sptr;
3739
3740 while (isspace(*s)) s++;
3741 if (isdigit((c = *s)))
3742   {
3743   int count;
3744   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3745   s += count;
3746   switch (tolower(*s))
3747     {
3748     default: break;
3749     case 'k': n *= 1024; s++; break;
3750     case 'm': n *= 1024*1024; s++; break;
3751     case 'g': n *= 1024*1024*1024; s++; break;
3752     }
3753   while (isspace (*s)) s++;
3754   }
3755 else if (c == '(')
3756   {
3757   s++;
3758   n = eval_expr(&s, decimal, error, 1);
3759   }
3760 else
3761   {
3762   *error = US"expecting number or opening parenthesis";
3763   n = 0;
3764   }
3765 *sptr = s;
3766 return n;
3767 }
3768
3769
3770 static int_eximarith_t
3771 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3772 {
3773 uschar *s = *sptr;
3774 int_eximarith_t x;
3775 while (isspace(*s)) s++;
3776 if (*s == '+' || *s == '-' || *s == '~')
3777   {
3778   int op = *s++;
3779   x = eval_op_unary(&s, decimal, error);
3780   if (op == '-') x = -x;
3781     else if (op == '~') x = ~x;
3782   }
3783 else
3784   x = eval_number(&s, decimal, error);
3785
3786 *sptr = s;
3787 return x;
3788 }
3789
3790
3791 static int_eximarith_t
3792 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3793 {
3794 uschar *s = *sptr;
3795 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3796 if (*error == NULL)
3797   {
3798   while (*s == '*' || *s == '/' || *s == '%')
3799     {
3800     int op = *s++;
3801     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3802     if (*error != NULL) break;
3803     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3804      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3805      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3806      * -N*M is INT_MIN will yield INT_MIN.
3807      * Since we don't support floating point, this is somewhat simpler.
3808      * Ideally, we'd return an error, but since we overflow for all other
3809      * arithmetic, consistency suggests otherwise, but what's the correct value
3810      * to use?  There is none.
3811      * The C standard guarantees overflow for unsigned arithmetic but signed
3812      * overflow invokes undefined behaviour; in practice, this is overflow
3813      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3814      * that long/longlong larger than int are available, or we could just work
3815      * with larger types.  We should consider whether to guarantee 32bit eval
3816      * and 64-bit working variables, with errors returned.  For now ...
3817      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3818      * can just let the other invalid results occur otherwise, as they have
3819      * until now.  For this one case, we can coerce.
3820      */
3821     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3822       {
3823       DEBUG(D_expand)
3824         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3825             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3826       x = EXIM_ARITH_MAX;
3827       continue;
3828       }
3829     if (op == '*')
3830       x *= y;
3831     else
3832       {
3833       if (y == 0)
3834         {
3835         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3836         x = 0;
3837         break;
3838         }
3839       if (op == '/')
3840         x /= y;
3841       else
3842         x %= y;
3843       }
3844     }
3845   }
3846 *sptr = s;
3847 return x;
3848 }
3849
3850
3851 static int_eximarith_t
3852 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3853 {
3854 uschar *s = *sptr;
3855 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3856 if (!*error)
3857   {
3858   while (*s == '+' || *s == '-')
3859     {
3860     int op = *s++;
3861     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3862     if (*error) break;
3863     if (  (x >=   EXIM_ARITH_MAX/2  && x >=   EXIM_ARITH_MAX/2)
3864        || (x <= -(EXIM_ARITH_MAX/2) && y <= -(EXIM_ARITH_MAX/2)))
3865       {                 /* over-conservative check */
3866       *error = op == '+'
3867         ? US"overflow in sum" : US"overflow in difference";
3868       break;
3869       }
3870     if (op == '+') x += y; else x -= y;
3871     }
3872   }
3873 *sptr = s;
3874 return x;
3875 }
3876
3877
3878 static int_eximarith_t
3879 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3880 {
3881 uschar *s = *sptr;
3882 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3883 if (*error == NULL)
3884   {
3885   while ((*s == '<' || *s == '>') && s[1] == s[0])
3886     {
3887     int_eximarith_t y;
3888     int op = *s++;
3889     s++;
3890     y = eval_op_sum(&s, decimal, error);
3891     if (*error != NULL) break;
3892     if (op == '<') x <<= y; else x >>= y;
3893     }
3894   }
3895 *sptr = s;
3896 return x;
3897 }
3898
3899
3900 static int_eximarith_t
3901 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3902 {
3903 uschar *s = *sptr;
3904 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3905 if (*error == NULL)
3906   {
3907   while (*s == '&')
3908     {
3909     int_eximarith_t y;
3910     s++;
3911     y = eval_op_shift(&s, decimal, error);
3912     if (*error != NULL) break;
3913     x &= y;
3914     }
3915   }
3916 *sptr = s;
3917 return x;
3918 }
3919
3920
3921 static int_eximarith_t
3922 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3923 {
3924 uschar *s = *sptr;
3925 int_eximarith_t x = eval_op_and(&s, decimal, error);
3926 if (*error == NULL)
3927   {
3928   while (*s == '^')
3929     {
3930     int_eximarith_t y;
3931     s++;
3932     y = eval_op_and(&s, decimal, error);
3933     if (*error != NULL) break;
3934     x ^= y;
3935     }
3936   }
3937 *sptr = s;
3938 return x;
3939 }
3940
3941
3942 static int_eximarith_t
3943 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3944 {
3945 uschar *s = *sptr;
3946 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3947 if (*error == NULL)
3948   {
3949   while (*s == '|')
3950     {
3951     int_eximarith_t y;
3952     s++;
3953     y = eval_op_xor(&s, decimal, error);
3954     if (*error != NULL) break;
3955     x |= y;
3956     }
3957   }
3958 *sptr = s;
3959 return x;
3960 }
3961
3962
3963
3964 /*************************************************
3965 *                 Expand string                  *
3966 *************************************************/
3967
3968 /* Returns either an unchanged string, or the expanded string in stacking pool
3969 store. Interpreted sequences are:
3970
3971    \...                    normal escaping rules
3972    $name                   substitutes the variable
3973    ${name}                 ditto
3974    ${op:string}            operates on the expanded string value
3975    ${item{arg1}{arg2}...}  expands the args and then does the business
3976                              some literal args are not enclosed in {}
3977
3978 There are now far too many operators and item types to make it worth listing
3979 them here in detail any more.
3980
3981 We use an internal routine recursively to handle embedded substrings. The
3982 external function follows. The yield is NULL if the expansion failed, and there
3983 are two cases: if something collapsed syntactically, or if "fail" was given
3984 as the action on a lookup failure. These can be distinguished by looking at the
3985 variable expand_string_forcedfail, which is TRUE in the latter case.
3986
3987 The skipping flag is set true when expanding a substring that isn't actually
3988 going to be used (after "if" or "lookup") and it prevents lookups from
3989 happening lower down.
3990
3991 Store usage: At start, a store block of the length of the input plus 64
3992 is obtained. This is expanded as necessary by string_cat(), which might have to
3993 get a new block, or might be able to expand the original. At the end of the
3994 function we can release any store above that portion of the yield block that
3995 was actually used. In many cases this will be optimal.
3996
3997 However: if the first item in the expansion is a variable name or header name,
3998 we reset the store before processing it; if the result is in fresh store, we
3999 use that without copying. This is helpful for expanding strings like
4000 $message_headers which can get very long.
4001
4002 There's a problem if a ${dlfunc item has side-effects that cause allocation,
4003 since resetting the store at the end of the expansion will free store that was
4004 allocated by the plugin code as well as the slop after the expanded string. So
4005 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
4006 and, given the acl condition, ${if }. This is an unfortunate consequence of
4007 string expansion becoming too powerful.
4008
4009 Arguments:
4010   string         the string to be expanded
4011   ket_ends       true if expansion is to stop at }
4012   left           if not NULL, a pointer to the first character after the
4013                  expansion is placed here (typically used with ket_ends)
4014   skipping       TRUE for recursive calls when the value isn't actually going
4015                  to be used (to allow for optimisation)
4016   honour_dollar  TRUE if $ is to be expanded,
4017                  FALSE if it's just another character
4018   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
4019                  the store.
4020
4021 Returns:         NULL if expansion fails:
4022                    expand_string_forcedfail is set TRUE if failure was forced
4023                    expand_string_message contains a textual error message
4024                  a pointer to the expanded string on success
4025 */
4026
4027 static uschar *
4028 expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left,
4029   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
4030 {
4031 gstring * yield = string_get(Ustrlen(string) + 64);
4032 int item_type;
4033 const uschar *s = string;
4034 uschar *save_expand_nstring[EXPAND_MAXN+1];
4035 int save_expand_nlength[EXPAND_MAXN+1];
4036 BOOL resetok = TRUE;
4037
4038 expand_level++;
4039 DEBUG(D_expand)
4040   DEBUG(D_noutf8)
4041     debug_printf_indent("/%s: %s\n",
4042       skipping ? "---scanning" : "considering", string);
4043   else
4044     debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n",
4045       skipping
4046       ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning"
4047       : "considering",
4048       string);
4049
4050 f.expand_string_forcedfail = FALSE;
4051 expand_string_message = US"";
4052
4053 while (*s != 0)
4054   {
4055   uschar *value;
4056   uschar name[256];
4057
4058   /* \ escapes the next character, which must exist, or else
4059   the expansion fails. There's a special escape, \N, which causes
4060   copying of the subject verbatim up to the next \N. Otherwise,
4061   the escapes are the standard set. */
4062
4063   if (*s == '\\')
4064     {
4065     if (s[1] == 0)
4066       {
4067       expand_string_message = US"\\ at end of string";
4068       goto EXPAND_FAILED;
4069       }
4070
4071     if (s[1] == 'N')
4072       {
4073       const uschar * t = s + 2;
4074       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
4075       yield = string_catn(yield, t, s - t);
4076       if (*s != 0) s += 2;
4077       }
4078
4079     else
4080       {
4081       uschar ch[1];
4082       ch[0] = string_interpret_escape(&s);
4083       s++;
4084       yield = string_catn(yield, ch, 1);
4085       }
4086
4087     continue;
4088     }
4089
4090   /*{*/
4091   /* Anything other than $ is just copied verbatim, unless we are
4092   looking for a terminating } character. */
4093
4094   /*{*/
4095   if (ket_ends && *s == '}') break;
4096
4097   if (*s != '$' || !honour_dollar)
4098     {
4099     yield = string_catn(yield, s++, 1);
4100     continue;
4101     }
4102
4103   /* No { after the $ - must be a plain name or a number for string
4104   match variable. There has to be a fudge for variables that are the
4105   names of header fields preceded by "$header_" because header field
4106   names can contain any printing characters except space and colon.
4107   For those that don't like typing this much, "$h_" is a synonym for
4108   "$header_". A non-existent header yields a NULL value; nothing is
4109   inserted. */  /*}*/
4110
4111   if (isalpha((*(++s))))
4112     {
4113     int len;
4114     int newsize = 0;
4115     gstring * g = NULL;
4116     uschar * t;
4117
4118     s = read_name(name, sizeof(name), s, US"_");
4119
4120     /* If this is the first thing to be expanded, release the pre-allocated
4121     buffer. */
4122
4123     if (!yield)
4124       g = store_get(sizeof(gstring));
4125     else if (yield->ptr == 0)
4126       {
4127       if (resetok) store_reset(yield);
4128       yield = NULL;
4129       g = store_get(sizeof(gstring));   /* alloc _before_ calling find_variable() */
4130       }
4131
4132     /* Header */
4133
4134     if (  ( *(t = name) == 'h'
4135           || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h'
4136           )
4137        && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0)
4138        )
4139       {
4140       unsigned flags = *name == 'r' ? FH_WANT_RAW
4141                       : *name == 'l' ? FH_WANT_RAW|FH_WANT_LIST
4142                       : 0;
4143       uschar * charset = *name == 'b' ? NULL : headers_charset;
4144
4145       s = read_header_name(name, sizeof(name), s);
4146       value = find_header(name, &newsize, flags, charset);
4147
4148       /* If we didn't find the header, and the header contains a closing brace
4149       character, this may be a user error where the terminating colon
4150       has been omitted. Set a flag to adjust the error message in this case.
4151       But there is no error here - nothing gets inserted. */
4152
4153       if (!value)
4154         {
4155         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
4156         continue;
4157         }
4158       }
4159
4160     /* Variable */
4161
4162     else if (!(value = find_variable(name, FALSE, skipping, &newsize)))
4163       {
4164       expand_string_message =
4165         string_sprintf("unknown variable name \"%s\"", name);
4166         check_variable_error_message(name);
4167       goto EXPAND_FAILED;
4168       }
4169
4170     /* If the data is known to be in a new buffer, newsize will be set to the
4171     size of that buffer. If this is the first thing in an expansion string,
4172     yield will be NULL; just point it at the new store instead of copying. Many
4173     expansion strings contain just one reference, so this is a useful
4174     optimization, especially for humungous headers.  We need to use a gstring
4175     structure that is not allocated after that new-buffer, else a later store
4176     reset in the middle of the buffer will make it inaccessible. */
4177
4178     len = Ustrlen(value);
4179     if (!yield && newsize != 0)
4180       {
4181       yield = g;
4182       yield->size = newsize;
4183       yield->ptr = len;
4184       yield->s = value;
4185       }
4186     else
4187       yield = string_catn(yield, value, len);
4188
4189     continue;
4190     }
4191
4192   if (isdigit(*s))
4193     {
4194     int n;
4195     s = read_cnumber(&n, s);
4196     if (n >= 0 && n <= expand_nmax)
4197       yield = string_catn(yield, expand_nstring[n], expand_nlength[n]);
4198     continue;
4199     }
4200
4201   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
4202
4203   if (*s != '{')                                                        /*}*/
4204     {
4205     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
4206     goto EXPAND_FAILED;
4207     }
4208
4209   /* After { there can be various things, but they all start with
4210   an initial word, except for a number for a string match variable. */
4211
4212   if (isdigit((*(++s))))
4213     {
4214     int n;
4215     s = read_cnumber(&n, s);            /*{*/
4216     if (*s++ != '}')
4217       {                                 /*{*/
4218       expand_string_message = US"} expected after number";
4219       goto EXPAND_FAILED;
4220       }
4221     if (n >= 0 && n <= expand_nmax)
4222       yield = string_catn(yield, expand_nstring[n], expand_nlength[n]);
4223     continue;
4224     }
4225
4226   if (!isalpha(*s))
4227     {
4228     expand_string_message = US"letter or digit expected after ${";      /*}*/
4229     goto EXPAND_FAILED;
4230     }
4231
4232   /* Allow "-" in names to cater for substrings with negative
4233   arguments. Since we are checking for known names after { this is
4234   OK. */
4235
4236   s = read_name(name, sizeof(name), s, US"_-");
4237   item_type = chop_match(name, item_table, nelem(item_table));
4238
4239   switch(item_type)
4240     {
4241     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
4242     If the ACL returns accept or reject we return content set by "message ="
4243     There is currently no limit on recursion; this would have us call
4244     acl_check_internal() directly and get a current level from somewhere.
4245     See also the acl expansion condition ECOND_ACL and the traditional
4246     acl modifier ACLC_ACL.
4247     Assume that the function has side-effects on the store that must be preserved.
4248     */
4249
4250     case EITEM_ACL:
4251       /* ${acl {name} {arg1}{arg2}...} */
4252       {
4253       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
4254       uschar *user_msg;
4255       int rc;
4256
4257       switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, US"acl",
4258                       &resetok))
4259         {
4260         case 1: goto EXPAND_FAILED_CURLY;
4261         case 2:
4262         case 3: goto EXPAND_FAILED;
4263         }
4264       if (skipping) continue;
4265
4266       resetok = FALSE;
4267       switch(rc = eval_acl(sub, nelem(sub), &user_msg))
4268         {
4269         case OK:
4270         case FAIL:
4271           DEBUG(D_expand)
4272             debug_printf_indent("acl expansion yield: %s\n", user_msg);
4273           if (user_msg)
4274             yield = string_cat(yield, user_msg);
4275           continue;
4276
4277         case DEFER:
4278           f.expand_string_forcedfail = TRUE;
4279           /*FALLTHROUGH*/
4280         default:
4281           expand_string_message = string_sprintf("%s from acl \"%s\"",
4282             rc_names[rc], sub[0]);
4283           goto EXPAND_FAILED;
4284         }
4285       }
4286
4287     case EITEM_AUTHRESULTS:
4288       /* ${authresults {mysystemname}} */
4289       {
4290       uschar *sub_arg[1];
4291
4292       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4293                       &resetok))
4294         {
4295         case 1: goto EXPAND_FAILED_CURLY;
4296         case 2:
4297         case 3: goto EXPAND_FAILED;
4298         }
4299
4300       yield = string_append(yield, 3,
4301                         US"Authentication-Results: ", sub_arg[0], US"; none");
4302       yield->ptr -= 6;
4303
4304       yield = authres_local(yield, sub_arg[0]);
4305       yield = authres_iprev(yield);
4306       yield = authres_smtpauth(yield);
4307 #ifdef SUPPORT_SPF
4308       yield = authres_spf(yield);
4309 #endif
4310 #ifndef DISABLE_DKIM
4311       yield = authres_dkim(yield);
4312 #endif
4313 #ifdef EXPERIMENTAL_DMARC
4314       yield = authres_dmarc(yield);
4315 #endif
4316 #ifdef EXPERIMENTAL_ARC
4317       yield = authres_arc(yield);
4318 #endif
4319       continue;
4320       }
4321
4322     /* Handle conditionals - preserve the values of the numerical expansion
4323     variables in case they get changed by a regular expression match in the
4324     condition. If not, they retain their external settings. At the end
4325     of this "if" section, they get restored to their previous values. */
4326
4327     case EITEM_IF:
4328       {
4329       BOOL cond = FALSE;
4330       const uschar *next_s;
4331       int save_expand_nmax =
4332         save_expand_strings(save_expand_nstring, save_expand_nlength);
4333
4334       while (isspace(*s)) s++;
4335       next_s = eval_condition(s, &resetok, skipping ? NULL : &cond);
4336       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
4337
4338       DEBUG(D_expand)
4339         DEBUG(D_noutf8)
4340           {
4341           debug_printf_indent("|--condition: %.*s\n", (int)(next_s - s), s);
4342           debug_printf_indent("|-----result: %s\n", cond ? "true" : "false");
4343           }
4344         else
4345           {
4346           debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4347             "condition: %.*s\n",
4348             (int)(next_s - s), s);
4349           debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
4350             UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
4351             "result: %s\n",
4352             cond ? "true" : "false");
4353           }
4354
4355       s = next_s;
4356
4357       /* The handling of "yes" and "no" result strings is now in a separate
4358       function that is also used by ${lookup} and ${extract} and ${run}. */
4359
4360       switch(process_yesno(
4361                skipping,                     /* were previously skipping */
4362                cond,                         /* success/failure indicator */
4363                lookup_value,                 /* value to reset for string2 */
4364                &s,                           /* input pointer */
4365                &yield,                       /* output pointer */
4366                US"if",                       /* condition type */
4367                &resetok))
4368         {
4369         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4370         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4371         }
4372
4373       /* Restore external setting of expansion variables for continuation
4374       at this level. */
4375
4376       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4377         save_expand_nlength);
4378       continue;
4379       }
4380
4381 #ifdef SUPPORT_I18N
4382     case EITEM_IMAPFOLDER:
4383       {                         /* ${imapfolder {name}{sep]{specials}} */
4384       uschar *sub_arg[3];
4385       uschar *encoded;
4386
4387       switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name,
4388                       &resetok))
4389         {
4390         case 1: goto EXPAND_FAILED_CURLY;
4391         case 2:
4392         case 3: goto EXPAND_FAILED;
4393         }
4394
4395       if (sub_arg[1] == NULL)           /* One argument */
4396         {
4397         sub_arg[1] = US"/";             /* default separator */
4398         sub_arg[2] = NULL;
4399         }
4400       else if (Ustrlen(sub_arg[1]) != 1)
4401         {
4402         expand_string_message =
4403           string_sprintf(
4404                 "IMAP folder separator must be one character, found \"%s\"",
4405                 sub_arg[1]);
4406         goto EXPAND_FAILED;
4407         }
4408
4409       if (!skipping)
4410         {
4411         if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
4412                             sub_arg[1][0], sub_arg[2], &expand_string_message)))
4413           goto EXPAND_FAILED;
4414         yield = string_cat(yield, encoded);
4415         }
4416       continue;
4417       }
4418 #endif
4419
4420     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4421     expanding an internal string that isn't actually going to be used. All we
4422     need to do is check the syntax, so don't do a lookup at all. Preserve the
4423     values of the numerical expansion variables in case they get changed by a
4424     partial lookup. If not, they retain their external settings. At the end
4425     of this "lookup" section, they get restored to their previous values. */
4426
4427     case EITEM_LOOKUP:
4428       {
4429       int stype, partial, affixlen, starflags;
4430       int expand_setup = 0;
4431       int nameptr = 0;
4432       uschar *key, *filename;
4433       const uschar *affix;
4434       uschar *save_lookup_value = lookup_value;
4435       int save_expand_nmax =
4436         save_expand_strings(save_expand_nstring, save_expand_nlength);
4437
4438       if ((expand_forbid & RDO_LOOKUP) != 0)
4439         {
4440         expand_string_message = US"lookup expansions are not permitted";
4441         goto EXPAND_FAILED;
4442         }
4443
4444       /* Get the key we are to look up for single-key+file style lookups.
4445       Otherwise set the key NULL pro-tem. */
4446
4447       while (isspace(*s)) s++;
4448       if (*s == '{')                                    /*}*/
4449         {
4450         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4451         if (!key) goto EXPAND_FAILED;                   /*{{*/
4452         if (*s++ != '}')
4453           {
4454           expand_string_message = US"missing '}' after lookup key";
4455           goto EXPAND_FAILED_CURLY;
4456           }
4457         while (isspace(*s)) s++;
4458         }
4459       else key = NULL;
4460
4461       /* Find out the type of database */
4462
4463       if (!isalpha(*s))
4464         {
4465         expand_string_message = US"missing lookup type";
4466         goto EXPAND_FAILED;
4467         }
4468
4469       /* The type is a string that may contain special characters of various
4470       kinds. Allow everything except space or { to appear; the actual content
4471       is checked by search_findtype_partial. */         /*}*/
4472
4473       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4474         {
4475         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4476         s++;
4477         }
4478       name[nameptr] = 0;
4479       while (isspace(*s)) s++;
4480
4481       /* Now check for the individual search type and any partial or default
4482       options. Only those types that are actually in the binary are valid. */
4483
4484       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4485         &starflags);
4486       if (stype < 0)
4487         {
4488         expand_string_message = search_error_message;
4489         goto EXPAND_FAILED;
4490         }
4491
4492       /* Check that a key was provided for those lookup types that need it,
4493       and was not supplied for those that use the query style. */
4494
4495       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4496         {
4497         if (key == NULL)
4498           {
4499           expand_string_message = string_sprintf("missing {key} for single-"
4500             "key \"%s\" lookup", name);
4501           goto EXPAND_FAILED;
4502           }
4503         }
4504       else
4505         {
4506         if (key != NULL)
4507           {
4508           expand_string_message = string_sprintf("a single key was given for "
4509             "lookup type \"%s\", which is not a single-key lookup type", name);
4510           goto EXPAND_FAILED;
4511           }
4512         }
4513
4514       /* Get the next string in brackets and expand it. It is the file name for
4515       single-key+file lookups, and the whole query otherwise. In the case of
4516       queries that also require a file name (e.g. sqlite), the file name comes
4517       first. */
4518
4519       if (*s != '{')
4520         {
4521         expand_string_message = US"missing '{' for lookup file-or-query arg";
4522         goto EXPAND_FAILED_CURLY;
4523         }
4524       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4525       if (filename == NULL) goto EXPAND_FAILED;
4526       if (*s++ != '}')
4527         {
4528         expand_string_message = US"missing '}' closing lookup file-or-query arg";
4529         goto EXPAND_FAILED_CURLY;
4530         }
4531       while (isspace(*s)) s++;
4532
4533       /* If this isn't a single-key+file lookup, re-arrange the variables
4534       to be appropriate for the search_ functions. For query-style lookups,
4535       there is just a "key", and no file name. For the special query-style +
4536       file types, the query (i.e. "key") starts with a file name. */
4537
4538       if (!key)
4539         {
4540         while (isspace(*filename)) filename++;
4541         key = filename;
4542
4543         if (mac_islookup(stype, lookup_querystyle))
4544           filename = NULL;
4545         else
4546           {
4547           if (*filename != '/')
4548             {
4549             expand_string_message = string_sprintf(
4550               "absolute file name expected for \"%s\" lookup", name);
4551             goto EXPAND_FAILED;
4552             }
4553           while (*key != 0 && !isspace(*key)) key++;
4554           if (*key != 0) *key++ = 0;
4555           }
4556         }
4557
4558       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4559       the entry was not found. Note that there is no search_close() function.
4560       Files are left open in case of re-use. At suitable places in higher logic,
4561       search_tidyup() is called to tidy all open files. This can save opening
4562       the same file several times. However, files may also get closed when
4563       others are opened, if too many are open at once. The rule is that a
4564       handle should not be used after a second search_open().
4565
4566       Request that a partial search sets up $1 and maybe $2 by passing
4567       expand_setup containing zero. If its value changes, reset expand_nmax,
4568       since new variables will have been set. Note that at the end of this
4569       "lookup" section, the old numeric variables are restored. */
4570
4571       if (skipping)
4572         lookup_value = NULL;
4573       else
4574         {
4575         void *handle = search_open(filename, stype, 0, NULL, NULL);
4576         if (handle == NULL)
4577           {
4578           expand_string_message = search_error_message;
4579           goto EXPAND_FAILED;
4580           }
4581         lookup_value = search_find(handle, filename, key, partial, affix,
4582           affixlen, starflags, &expand_setup);
4583         if (f.search_find_defer)
4584           {
4585           expand_string_message =
4586             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4587               string_printing2(key, FALSE), search_error_message);
4588           goto EXPAND_FAILED;
4589           }
4590         if (expand_setup > 0) expand_nmax = expand_setup;
4591         }
4592
4593       /* The handling of "yes" and "no" result strings is now in a separate
4594       function that is also used by ${if} and ${extract}. */
4595
4596       switch(process_yesno(
4597                skipping,                     /* were previously skipping */
4598                lookup_value != NULL,         /* success/failure indicator */
4599                save_lookup_value,            /* value to reset for string2 */
4600                &s,                           /* input pointer */
4601                &yield,                       /* output pointer */
4602                US"lookup",                   /* condition type */
4603                &resetok))
4604         {
4605         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4606         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4607         }
4608
4609       /* Restore external setting of expansion variables for carrying on
4610       at this level, and continue. */
4611
4612       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4613         save_expand_nlength);
4614       continue;
4615       }
4616
4617     /* If Perl support is configured, handle calling embedded perl subroutines,
4618     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4619     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4620     arguments (defined below). */
4621
4622     #define EXIM_PERL_MAX_ARGS 8
4623
4624     case EITEM_PERL:
4625     #ifndef EXIM_PERL
4626     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4627       "is not included in this binary";
4628     goto EXPAND_FAILED;
4629
4630     #else   /* EXIM_PERL */
4631       {
4632       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4633       gstring *new_yield;
4634
4635       if ((expand_forbid & RDO_PERL) != 0)
4636         {
4637         expand_string_message = US"Perl calls are not permitted";
4638         goto EXPAND_FAILED;
4639         }
4640
4641       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4642            US"perl", &resetok))
4643         {
4644         case 1: goto EXPAND_FAILED_CURLY;
4645         case 2:
4646         case 3: goto EXPAND_FAILED;
4647         }
4648
4649       /* If skipping, we don't actually do anything */
4650
4651       if (skipping) continue;
4652
4653       /* Start the interpreter if necessary */
4654
4655       if (!opt_perl_started)
4656         {
4657         uschar *initerror;
4658         if (opt_perl_startup == NULL)
4659           {
4660           expand_string_message = US"A setting of perl_startup is needed when "
4661             "using the Perl interpreter";
4662           goto EXPAND_FAILED;
4663           }
4664         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4665         initerror = init_perl(opt_perl_startup);
4666         if (initerror != NULL)
4667           {
4668           expand_string_message =
4669             string_sprintf("error in perl_startup code: %s\n", initerror);
4670           goto EXPAND_FAILED;
4671           }
4672         opt_perl_started = TRUE;
4673         }
4674
4675       /* Call the function */
4676
4677       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4678       new_yield = call_perl_cat(yield, &expand_string_message,
4679         sub_arg[0], sub_arg + 1);
4680
4681       /* NULL yield indicates failure; if the message pointer has been set to
4682       NULL, the yield was undef, indicating a forced failure. Otherwise the
4683       message will indicate some kind of Perl error. */
4684
4685       if (new_yield == NULL)
4686         {
4687         if (expand_string_message == NULL)
4688           {
4689           expand_string_message =
4690             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4691               "failure", sub_arg[0]);
4692           f.expand_string_forcedfail = TRUE;
4693           }
4694         goto EXPAND_FAILED;
4695         }
4696
4697       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4698       set during a callback from Perl. */
4699
4700       f.expand_string_forcedfail = FALSE;
4701       yield = new_yield;
4702       continue;
4703       }
4704     #endif /* EXIM_PERL */
4705
4706     /* Transform email address to "prvs" scheme to use
4707        as BATV-signed return path */
4708
4709     case EITEM_PRVS:
4710       {
4711       uschar *sub_arg[3];
4712       uschar *p,*domain;
4713
4714       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4715         {
4716         case 1: goto EXPAND_FAILED_CURLY;
4717         case 2:
4718         case 3: goto EXPAND_FAILED;
4719         }
4720
4721       /* If skipping, we don't actually do anything */
4722       if (skipping) continue;
4723
4724       /* sub_arg[0] is the address */
4725       if (  !(domain = Ustrrchr(sub_arg[0],'@'))
4726          || domain == sub_arg[0] || Ustrlen(domain) == 1)
4727         {
4728         expand_string_message = US"prvs first argument must be a qualified email address";
4729         goto EXPAND_FAILED;
4730         }
4731
4732       /* Calculate the hash. The third argument must be a single-digit
4733       key number, or unset. */
4734
4735       if (  sub_arg[2]
4736          && (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4737         {
4738         expand_string_message = US"prvs third argument must be a single digit";
4739         goto EXPAND_FAILED;
4740         }
4741
4742       p = prvs_hmac_sha1(sub_arg[0], sub_arg[1], sub_arg[2], prvs_daystamp(7));
4743       if (!p)
4744         {
4745         expand_string_message = US"prvs hmac-sha1 conversion failed";
4746         goto EXPAND_FAILED;
4747         }
4748
4749       /* Now separate the domain from the local part */
4750       *domain++ = '\0';
4751
4752       yield = string_catn(yield, US"prvs=", 5);
4753       yield = string_catn(yield, sub_arg[2] ? sub_arg[2] : US"0", 1);
4754       yield = string_catn(yield, prvs_daystamp(7), 3);
4755       yield = string_catn(yield, p, 6);
4756       yield = string_catn(yield, US"=", 1);
4757       yield = string_cat (yield, sub_arg[0]);
4758       yield = string_catn(yield, US"@", 1);
4759       yield = string_cat (yield, domain);
4760
4761       continue;
4762       }
4763
4764     /* Check a prvs-encoded address for validity */
4765
4766     case EITEM_PRVSCHECK:
4767       {
4768       uschar *sub_arg[3];
4769       gstring * g;
4770       const pcre *re;
4771       uschar *p;
4772
4773       /* TF: Ugliness: We want to expand parameter 1 first, then set
4774          up expansion variables that are used in the expansion of
4775          parameter 2. So we clone the string for the first
4776          expansion, where we only expand parameter 1.
4777
4778          PH: Actually, that isn't necessary. The read_subs() function is
4779          designed to work this way for the ${if and ${lookup expansions. I've
4780          tidied the code.
4781       */
4782
4783       /* Reset expansion variables */
4784       prvscheck_result = NULL;
4785       prvscheck_address = NULL;
4786       prvscheck_keynum = NULL;
4787
4788       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4789         {
4790         case 1: goto EXPAND_FAILED_CURLY;
4791         case 2:
4792         case 3: goto EXPAND_FAILED;
4793         }
4794
4795       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4796                               TRUE,FALSE);
4797
4798       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4799         {
4800         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4801         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4802         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4803         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4804         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4805
4806         DEBUG(D_expand) debug_printf_indent("prvscheck localpart: %s\n", local_part);
4807         DEBUG(D_expand) debug_printf_indent("prvscheck key number: %s\n", key_num);
4808         DEBUG(D_expand) debug_printf_indent("prvscheck daystamp: %s\n", daystamp);
4809         DEBUG(D_expand) debug_printf_indent("prvscheck hash: %s\n", hash);
4810         DEBUG(D_expand) debug_printf_indent("prvscheck domain: %s\n", domain);
4811
4812         /* Set up expansion variables */
4813         g = string_cat (NULL, local_part);
4814         g = string_catn(g, US"@", 1);
4815         g = string_cat (g, domain);
4816         prvscheck_address = string_from_gstring(g);
4817         prvscheck_keynum = string_copy(key_num);
4818
4819         /* Now expand the second argument */
4820         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4821           {
4822           case 1: goto EXPAND_FAILED_CURLY;
4823           case 2:
4824           case 3: goto EXPAND_FAILED;
4825           }
4826
4827         /* Now we have the key and can check the address. */
4828
4829         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4830           daystamp);
4831
4832         if (!p)
4833           {
4834           expand_string_message = US"hmac-sha1 conversion failed";
4835           goto EXPAND_FAILED;
4836           }
4837
4838         DEBUG(D_expand) debug_printf_indent("prvscheck: received hash is %s\n", hash);
4839         DEBUG(D_expand) debug_printf_indent("prvscheck:      own hash is %s\n", p);
4840
4841         if (Ustrcmp(p,hash) == 0)
4842           {
4843           /* Success, valid BATV address. Now check the expiry date. */
4844           uschar *now = prvs_daystamp(0);
4845           unsigned int inow = 0,iexpire = 1;
4846
4847           (void)sscanf(CS now,"%u",&inow);
4848           (void)sscanf(CS daystamp,"%u",&iexpire);
4849
4850           /* When "iexpire" is < 7, a "flip" has occurred.
4851              Adjust "inow" accordingly. */
4852           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4853
4854           if (iexpire >= inow)
4855             {
4856             prvscheck_result = US"1";
4857             DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n");
4858             }
4859           else
4860             {
4861             prvscheck_result = NULL;
4862             DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n");
4863             }
4864           }
4865         else
4866           {
4867           prvscheck_result = NULL;
4868           DEBUG(D_expand) debug_printf_indent("prvscheck: hash failure, $pvrs_result unset\n");
4869           }
4870
4871         /* Now expand the final argument. We leave this till now so that
4872         it can include $prvscheck_result. */
4873
4874         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4875           {
4876           case 1: goto EXPAND_FAILED_CURLY;
4877           case 2:
4878           case 3: goto EXPAND_FAILED;
4879           }
4880
4881         yield = string_cat(yield,
4882           !sub_arg[0] || !*sub_arg[0] ? prvscheck_address : sub_arg[0]);
4883
4884         /* Reset the "internal" variables afterwards, because they are in
4885         dynamic store that will be reclaimed if the expansion succeeded. */
4886
4887         prvscheck_address = NULL;
4888         prvscheck_keynum = NULL;
4889         }
4890       else
4891         /* Does not look like a prvs encoded address, return the empty string.
4892            We need to make sure all subs are expanded first, so as to skip over
4893            the entire item. */
4894
4895         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4896           {
4897           case 1: goto EXPAND_FAILED_CURLY;
4898           case 2:
4899           case 3: goto EXPAND_FAILED;
4900           }
4901
4902       continue;
4903       }
4904
4905     /* Handle "readfile" to insert an entire file */
4906
4907     case EITEM_READFILE:
4908       {
4909       FILE *f;
4910       uschar *sub_arg[2];
4911
4912       if ((expand_forbid & RDO_READFILE) != 0)
4913         {
4914         expand_string_message = US"file insertions are not permitted";
4915         goto EXPAND_FAILED;
4916         }
4917
4918       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4919         {
4920         case 1: goto EXPAND_FAILED_CURLY;
4921         case 2:
4922         case 3: goto EXPAND_FAILED;
4923         }
4924
4925       /* If skipping, we don't actually do anything */
4926
4927       if (skipping) continue;
4928
4929       /* Open the file and read it */
4930
4931       if (!(f = Ufopen(sub_arg[0], "rb")))
4932         {
4933         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4934         goto EXPAND_FAILED;
4935         }
4936
4937       yield = cat_file(f, yield, sub_arg[1]);
4938       (void)fclose(f);
4939       continue;
4940       }
4941
4942     /* Handle "readsocket" to insert data from a socket, either
4943     Inet or Unix domain */
4944
4945     case EITEM_READSOCK:
4946       {
4947       client_conn_ctx cctx;
4948       int timeout = 5;
4949       int save_ptr = yield->ptr;
4950       FILE * fp = NULL;
4951       uschar * arg;
4952       uschar * sub_arg[4];
4953       uschar * server_name = NULL;
4954       host_item host;
4955       BOOL do_shutdown = TRUE;
4956       BOOL do_tls = FALSE;      /* Only set under ! DISABLE_TLS */
4957       blob reqstr;
4958
4959       if (expand_forbid & RDO_READSOCK)
4960         {
4961         expand_string_message = US"socket insertions are not permitted";
4962         goto EXPAND_FAILED;
4963         }
4964
4965       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4966       because there may be a string for expansion on failure. */
4967
4968       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4969         {
4970         case 1: goto EXPAND_FAILED_CURLY;
4971         case 2:                             /* Won't occur: no end check */
4972         case 3: goto EXPAND_FAILED;
4973         }
4974
4975       /* Grab the request string, if any */
4976
4977       reqstr.data = sub_arg[1];
4978       reqstr.len = Ustrlen(sub_arg[1]);
4979
4980       /* Sort out timeout, if given.  The second arg is a list with the first element
4981       being a time value.  Any more are options of form "name=value".  Currently the
4982       only option recognised is "shutdown". */
4983
4984       if (sub_arg[2])
4985         {
4986         const uschar * list = sub_arg[2];
4987         uschar * item;
4988         int sep = 0;
4989
4990         item = string_nextinlist(&list, &sep, NULL, 0);
4991         if ((timeout = readconf_readtime(item, 0, FALSE)) < 0)
4992           {
4993           expand_string_message = string_sprintf("bad time value %s", item);
4994           goto EXPAND_FAILED;
4995           }
4996
4997         while ((item = string_nextinlist(&list, &sep, NULL, 0)))
4998           if (Ustrncmp(item, US"shutdown=", 9) == 0)
4999             { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; }
5000 #ifndef DISABLE_TLS
5001           else if (Ustrncmp(item, US"tls=", 4) == 0)
5002             { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; }
5003 #endif
5004         }
5005       else
5006         sub_arg[3] = NULL;                     /* No eol if no timeout */
5007
5008       /* If skipping, we don't actually do anything. Otherwise, arrange to
5009       connect to either an IP or a Unix socket. */
5010
5011       if (!skipping)
5012         {
5013         /* Handle an IP (internet) domain */
5014
5015         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
5016           {
5017           int port;
5018           uschar * port_name;
5019
5020           server_name = sub_arg[0] + 5;
5021           port_name = Ustrrchr(server_name, ':');
5022
5023           /* Sort out the port */
5024
5025           if (!port_name)
5026             {
5027             expand_string_message =
5028               string_sprintf("missing port for readsocket %s", sub_arg[0]);
5029             goto EXPAND_FAILED;
5030             }
5031           *port_name++ = 0;           /* Terminate server name */
5032
5033           if (isdigit(*port_name))
5034             {
5035             uschar *end;
5036             port = Ustrtol(port_name, &end, 0);
5037             if (end != port_name + Ustrlen(port_name))
5038               {
5039               expand_string_message =
5040                 string_sprintf("invalid port number %s", port_name);
5041               goto EXPAND_FAILED;
5042               }
5043             }
5044           else
5045             {
5046             struct servent *service_info = getservbyname(CS port_name, "tcp");
5047             if (!service_info)
5048               {
5049               expand_string_message = string_sprintf("unknown port \"%s\"",
5050                 port_name);
5051               goto EXPAND_FAILED;
5052               }
5053             port = ntohs(service_info->s_port);
5054             }
5055
5056           /*XXX we trust that the request is idempotent for TFO.  Hmm. */
5057           cctx.sock = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
5058                   timeout, &host, &expand_string_message,
5059                   do_tls ? NULL : &reqstr);
5060           callout_address = NULL;
5061           if (cctx.sock < 0)
5062             goto SOCK_FAIL;
5063           if (!do_tls)
5064             reqstr.len = 0;
5065           }
5066
5067         /* Handle a Unix domain socket */
5068
5069         else
5070           {
5071           struct sockaddr_un sockun;         /* don't call this "sun" ! */
5072           int rc;
5073
5074           if ((cctx.sock = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
5075             {
5076             expand_string_message = string_sprintf("failed to create socket: %s",
5077               strerror(errno));
5078             goto SOCK_FAIL;
5079             }
5080
5081           sockun.sun_family = AF_UNIX;
5082           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
5083             sub_arg[0]);
5084           server_name = US sockun.sun_path;
5085
5086           sigalrm_seen = FALSE;
5087           ALARM(timeout);
5088           rc = connect(cctx.sock, (struct sockaddr *)(&sockun), sizeof(sockun));
5089           ALARM_CLR(0);
5090           if (sigalrm_seen)
5091             {
5092             expand_string_message = US "socket connect timed out";
5093             goto SOCK_FAIL;
5094             }
5095           if (rc < 0)
5096             {
5097             expand_string_message = string_sprintf("failed to connect to socket "
5098               "%s: %s", sub_arg[0], strerror(errno));
5099             goto SOCK_FAIL;
5100             }
5101           host.name = server_name;
5102           host.address = US"";
5103           }
5104
5105         DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]);
5106
5107 #ifndef DISABLE_TLS
5108         if (do_tls)
5109           {
5110           smtp_connect_args conn_args = {.host = &host };
5111           tls_support tls_dummy = {.sni=NULL};
5112           uschar * errstr;
5113
5114           if (!tls_client_start(&cctx, &conn_args, NULL, &tls_dummy, &errstr))
5115             {
5116             expand_string_message = string_sprintf("TLS connect failed: %s", errstr);
5117             goto SOCK_FAIL;
5118             }
5119           }
5120 #endif
5121
5122         /* Allow sequencing of test actions */
5123         if (f.running_in_test_harness) millisleep(100);
5124
5125         /* Write the request string, if not empty or already done */
5126
5127         if (reqstr.len)
5128           {
5129           DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n",
5130             reqstr.data);
5131           if ( (
5132 #ifndef DISABLE_TLS
5133               do_tls ? tls_write(cctx.tls_ctx, reqstr.data, reqstr.len, FALSE) :
5134 #endif
5135                         write(cctx.sock, reqstr.data, reqstr.len)) != reqstr.len)
5136             {
5137             expand_string_message = string_sprintf("request write to socket "
5138               "failed: %s", strerror(errno));
5139             goto SOCK_FAIL;
5140             }
5141           }
5142
5143         /* Shut down the sending side of the socket. This helps some servers to
5144         recognise that it is their turn to do some work. Just in case some
5145         system doesn't have this function, make it conditional. */
5146
5147 #ifdef SHUT_WR
5148         if (!do_tls && do_shutdown) shutdown(cctx.sock, SHUT_WR);
5149 #endif
5150
5151         if (f.running_in_test_harness) millisleep(100);
5152
5153         /* Now we need to read from the socket, under a timeout. The function
5154         that reads a file can be used. */
5155
5156         if (!do_tls)
5157           fp = fdopen(cctx.sock, "rb");
5158         sigalrm_seen = FALSE;
5159         ALARM(timeout);
5160         yield =
5161 #ifndef DISABLE_TLS
5162           do_tls ? cat_file_tls(cctx.tls_ctx, yield, sub_arg[3]) :
5163 #endif
5164                     cat_file(fp, yield, sub_arg[3]);
5165         ALARM_CLR(0);
5166
5167 #ifndef DISABLE_TLS
5168         if (do_tls)
5169           {
5170           tls_close(cctx.tls_ctx, TRUE);
5171           close(cctx.sock);
5172           }
5173         else
5174 #endif
5175           (void)fclose(fp);
5176
5177         /* After a timeout, we restore the pointer in the result, that is,
5178         make sure we add nothing from the socket. */
5179
5180         if (sigalrm_seen)
5181           {
5182           yield->ptr = save_ptr;
5183           expand_string_message = US "socket read timed out";
5184           goto SOCK_FAIL;
5185           }
5186         }
5187
5188       /* The whole thing has worked (or we were skipping). If there is a
5189       failure string following, we need to skip it. */
5190
5191       if (*s == '{')
5192         {
5193         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
5194           goto EXPAND_FAILED;
5195         if (*s++ != '}')
5196           {
5197           expand_string_message = US"missing '}' closing failstring for readsocket";
5198           goto EXPAND_FAILED_CURLY;
5199           }
5200         while (isspace(*s)) s++;
5201         }
5202
5203     READSOCK_DONE:
5204       if (*s++ != '}')
5205         {
5206         expand_string_message = US"missing '}' closing readsocket";
5207         goto EXPAND_FAILED_CURLY;
5208         }
5209       continue;
5210
5211       /* Come here on failure to create socket, connect socket, write to the
5212       socket, or timeout on reading. If another substring follows, expand and
5213       use it. Otherwise, those conditions give expand errors. */
5214
5215     SOCK_FAIL:
5216       if (*s != '{') goto EXPAND_FAILED;
5217       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
5218       if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok)))
5219         goto EXPAND_FAILED;
5220       yield = string_cat(yield, arg);
5221       if (*s++ != '}')
5222         {
5223         expand_string_message = US"missing '}' closing failstring for readsocket";
5224         goto EXPAND_FAILED_CURLY;
5225         }
5226       while (isspace(*s)) s++;
5227       goto READSOCK_DONE;
5228       }
5229
5230     /* Handle "run" to execute a program. */
5231
5232     case EITEM_RUN:
5233       {
5234       FILE *f;
5235       uschar *arg;
5236       const uschar **argv;
5237       pid_t pid;
5238       int fd_in, fd_out;
5239
5240       if ((expand_forbid & RDO_RUN) != 0)
5241         {
5242         expand_string_message = US"running a command is not permitted";
5243         goto EXPAND_FAILED;
5244         }
5245
5246       while (isspace(*s)) s++;
5247       if (*s != '{')
5248         {
5249         expand_string_message = US"missing '{' for command arg of run";
5250         goto EXPAND_FAILED_CURLY;
5251         }
5252       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5253       if (arg == NULL) goto EXPAND_FAILED;
5254       while (isspace(*s)) s++;
5255       if (*s++ != '}')
5256         {
5257         expand_string_message = US"missing '}' closing command arg of run";
5258         goto EXPAND_FAILED_CURLY;
5259         }
5260
5261       if (skipping)   /* Just pretend it worked when we're skipping */
5262         {
5263         runrc = 0;
5264         lookup_value = NULL;
5265         }
5266       else
5267         {
5268         if (!transport_set_up_command(&argv,    /* anchor for arg list */
5269             arg,                                /* raw command */
5270             FALSE,                              /* don't expand the arguments */
5271             0,                                  /* not relevant when... */
5272             NULL,                               /* no transporting address */
5273             US"${run} expansion",               /* for error messages */
5274             &expand_string_message))            /* where to put error message */
5275           goto EXPAND_FAILED;
5276
5277         /* Create the child process, making it a group leader. */
5278
5279         if ((pid = child_open(USS argv, NULL, 0077, &fd_in, &fd_out, TRUE)) < 0)
5280           {
5281           expand_string_message =
5282             string_sprintf("couldn't create child process: %s", strerror(errno));
5283           goto EXPAND_FAILED;
5284           }
5285
5286         /* Nothing is written to the standard input. */
5287
5288         (void)close(fd_in);
5289
5290         /* Read the pipe to get the command's output into $value (which is kept
5291         in lookup_value). Read during execution, so that if the output exceeds
5292         the OS pipe buffer limit, we don't block forever. Remember to not release
5293         memory just allocated for $value. */
5294
5295         resetok = FALSE;
5296         f = fdopen(fd_out, "rb");
5297         sigalrm_seen = FALSE;
5298         ALARM(60);
5299         lookup_value = string_from_gstring(cat_file(f, NULL, NULL));
5300         ALARM_CLR(0);
5301         (void)fclose(f);
5302
5303         /* Wait for the process to finish, applying the timeout, and inspect its
5304         return code for serious disasters. Simple non-zero returns are passed on.
5305         */
5306
5307         if (sigalrm_seen || (runrc = child_close(pid, 30)) < 0)
5308           {
5309           if (sigalrm_seen || runrc == -256)
5310             {
5311             expand_string_message = string_sprintf("command timed out");
5312             killpg(pid, SIGKILL);       /* Kill the whole process group */
5313             }
5314
5315           else if (runrc == -257)
5316             expand_string_message = string_sprintf("wait() failed: %s",
5317               strerror(errno));
5318
5319           else
5320             expand_string_message = string_sprintf("command killed by signal %d",
5321               -runrc);
5322
5323           goto EXPAND_FAILED;
5324           }
5325         }
5326
5327       /* Process the yes/no strings; $value may be useful in both cases */
5328
5329       switch(process_yesno(
5330                skipping,                     /* were previously skipping */
5331                runrc == 0,                   /* success/failure indicator */
5332                lookup_value,                 /* value to reset for string2 */
5333                &s,                           /* input pointer */
5334                &yield,                       /* output pointer */
5335                US"run",                      /* condition type */
5336                &resetok))
5337         {
5338         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5339         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5340         }
5341
5342       continue;
5343       }
5344
5345     /* Handle character translation for "tr" */
5346
5347     case EITEM_TR:
5348       {
5349       int oldptr = yield->ptr;
5350       int o2m;
5351       uschar *sub[3];
5352
5353       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
5354         {
5355         case 1: goto EXPAND_FAILED_CURLY;
5356         case 2:
5357         case 3: goto EXPAND_FAILED;
5358         }
5359
5360       yield = string_cat(yield, sub[0]);
5361       o2m = Ustrlen(sub[2]) - 1;
5362
5363       if (o2m >= 0) for (; oldptr < yield->ptr; oldptr++)
5364         {
5365         uschar *m = Ustrrchr(sub[1], yield->s[oldptr]);
5366         if (m != NULL)
5367           {
5368           int o = m - sub[1];
5369           yield->s[oldptr] = sub[2][(o < o2m)? o : o2m];
5370           }
5371         }
5372
5373       continue;
5374       }
5375
5376     /* Handle "hash", "length", "nhash", and "substr" when they are given with
5377     expanded arguments. */
5378
5379     case EITEM_HASH:
5380     case EITEM_LENGTH:
5381     case EITEM_NHASH:
5382     case EITEM_SUBSTR:
5383       {
5384       int len;
5385       uschar *ret;
5386       int val[2] = { 0, -1 };
5387       uschar *sub[3];
5388
5389       /* "length" takes only 2 arguments whereas the others take 2 or 3.
5390       Ensure that sub[2] is set in the ${length } case. */
5391
5392       sub[2] = NULL;
5393       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
5394              TRUE, name, &resetok))
5395         {
5396         case 1: goto EXPAND_FAILED_CURLY;
5397         case 2:
5398         case 3: goto EXPAND_FAILED;
5399         }
5400
5401       /* Juggle the arguments if there are only two of them: always move the
5402       string to the last position and make ${length{n}{str}} equivalent to
5403       ${substr{0}{n}{str}}. See the defaults for val[] above. */
5404
5405       if (sub[2] == NULL)
5406         {
5407         sub[2] = sub[1];
5408         sub[1] = NULL;
5409         if (item_type == EITEM_LENGTH)
5410           {
5411           sub[1] = sub[0];
5412           sub[0] = NULL;
5413           }
5414         }
5415
5416       for (int i = 0; i < 2; i++) if (sub[i])
5417         {
5418         val[i] = (int)Ustrtol(sub[i], &ret, 10);
5419         if (*ret != 0 || (i != 0 && val[i] < 0))
5420           {
5421           expand_string_message = string_sprintf("\"%s\" is not a%s number "
5422             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
5423           goto EXPAND_FAILED;
5424           }
5425         }
5426
5427       ret =
5428         (item_type == EITEM_HASH)?
5429           compute_hash(sub[2], val[0], val[1], &len) :
5430         (item_type == EITEM_NHASH)?
5431           compute_nhash(sub[2], val[0], val[1], &len) :
5432           extract_substr(sub[2], val[0], val[1], &len);
5433
5434       if (ret == NULL) goto EXPAND_FAILED;
5435       yield = string_catn(yield, ret, len);
5436       continue;
5437       }
5438
5439     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
5440     This code originally contributed by Steve Haslam. It currently supports
5441     the use of MD5 and SHA-1 hashes.
5442
5443     We need some workspace that is large enough to handle all the supported
5444     hash types. Use macros to set the sizes rather than be too elaborate. */
5445
5446     #define MAX_HASHLEN      20
5447     #define MAX_HASHBLOCKLEN 64
5448
5449     case EITEM_HMAC:
5450       {
5451       uschar *sub[3];
5452       md5 md5_base;
5453       hctx sha1_ctx;
5454       void *use_base;
5455       int type;
5456       int hashlen;      /* Number of octets for the hash algorithm's output */
5457       int hashblocklen; /* Number of octets the hash algorithm processes */
5458       uschar *keyptr, *p;
5459       unsigned int keylen;
5460
5461       uschar keyhash[MAX_HASHLEN];
5462       uschar innerhash[MAX_HASHLEN];
5463       uschar finalhash[MAX_HASHLEN];
5464       uschar finalhash_hex[2*MAX_HASHLEN];
5465       uschar innerkey[MAX_HASHBLOCKLEN];
5466       uschar outerkey[MAX_HASHBLOCKLEN];
5467
5468       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
5469         {
5470         case 1: goto EXPAND_FAILED_CURLY;
5471         case 2:
5472         case 3: goto EXPAND_FAILED;
5473         }
5474
5475       if (!skipping)
5476         {
5477         if (Ustrcmp(sub[0], "md5") == 0)
5478           {
5479           type = HMAC_MD5;
5480           use_base = &md5_base;
5481           hashlen = 16;
5482           hashblocklen = 64;
5483           }
5484         else if (Ustrcmp(sub[0], "sha1") == 0)
5485           {
5486           type = HMAC_SHA1;
5487           use_base = &sha1_ctx;
5488           hashlen = 20;
5489           hashblocklen = 64;
5490           }
5491         else
5492           {
5493           expand_string_message =
5494             string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
5495           goto EXPAND_FAILED;
5496           }
5497
5498         keyptr = sub[1];
5499         keylen = Ustrlen(keyptr);
5500
5501         /* If the key is longer than the hash block length, then hash the key
5502         first */
5503
5504         if (keylen > hashblocklen)
5505           {
5506           chash_start(type, use_base);
5507           chash_end(type, use_base, keyptr, keylen, keyhash);
5508           keyptr = keyhash;
5509           keylen = hashlen;
5510           }
5511
5512         /* Now make the inner and outer key values */
5513
5514         memset(innerkey, 0x36, hashblocklen);
5515         memset(outerkey, 0x5c, hashblocklen);
5516
5517         for (int i = 0; i < keylen; i++)
5518           {
5519           innerkey[i] ^= keyptr[i];
5520           outerkey[i] ^= keyptr[i];
5521           }
5522
5523         /* Now do the hashes */
5524
5525         chash_start(type, use_base);
5526         chash_mid(type, use_base, innerkey);
5527         chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5528
5529         chash_start(type, use_base);
5530         chash_mid(type, use_base, outerkey);
5531         chash_end(type, use_base, innerhash, hashlen, finalhash);
5532
5533         /* Encode the final hash as a hex string */
5534
5535         p = finalhash_hex;
5536         for (int i = 0; i < hashlen; i++)
5537           {
5538           *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5539           *p++ = hex_digits[finalhash[i] & 0x0f];
5540           }
5541
5542         DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%s)=%.*s\n",
5543           sub[0], (int)keylen, keyptr, sub[2], hashlen*2, finalhash_hex);
5544
5545         yield = string_catn(yield, finalhash_hex, hashlen*2);
5546         }
5547       continue;
5548       }
5549
5550     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5551     We have to save the numerical variables and restore them afterwards. */
5552
5553     case EITEM_SG:
5554       {
5555       const pcre *re;
5556       int moffset, moffsetextra, slen;
5557       int roffset;
5558       int emptyopt;
5559       const uschar *rerror;
5560       uschar *subject;
5561       uschar *sub[3];
5562       int save_expand_nmax =
5563         save_expand_strings(save_expand_nstring, save_expand_nlength);
5564
5565       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5566         {
5567         case 1: goto EXPAND_FAILED_CURLY;
5568         case 2:
5569         case 3: goto EXPAND_FAILED;
5570         }
5571
5572       /* Compile the regular expression */
5573
5574       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5575         NULL);
5576
5577       if (re == NULL)
5578         {
5579         expand_string_message = string_sprintf("regular expression error in "
5580           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5581         goto EXPAND_FAILED;
5582         }
5583
5584       /* Now run a loop to do the substitutions as often as necessary. It ends
5585       when there are no more matches. Take care over matches of the null string;
5586       do the same thing as Perl does. */
5587
5588       subject = sub[0];
5589       slen = Ustrlen(sub[0]);
5590       moffset = moffsetextra = 0;
5591       emptyopt = 0;
5592
5593       for (;;)
5594         {
5595         int ovector[3*(EXPAND_MAXN+1)];
5596         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5597           PCRE_EOPT | emptyopt, ovector, nelem(ovector));
5598         uschar *insert;
5599
5600         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5601         is not necessarily the end. We want to repeat the match from one
5602         character further along, but leaving the basic offset the same (for
5603         copying below). We can't be at the end of the string - that was checked
5604         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5605         finished; copy the remaining string and end the loop. */
5606
5607         if (n < 0)
5608           {
5609           if (emptyopt != 0)
5610             {
5611             moffsetextra = 1;
5612             emptyopt = 0;
5613             continue;
5614             }
5615           yield = string_catn(yield, subject+moffset, slen-moffset);
5616           break;
5617           }
5618
5619         /* Match - set up for expanding the replacement. */
5620
5621         if (n == 0) n = EXPAND_MAXN + 1;
5622         expand_nmax = 0;
5623         for (int nn = 0; nn < n*2; nn += 2)
5624           {
5625           expand_nstring[expand_nmax] = subject + ovector[nn];
5626           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5627           }
5628         expand_nmax--;
5629
5630         /* Copy the characters before the match, plus the expanded insertion. */
5631
5632         yield = string_catn(yield, subject + moffset, ovector[0] - moffset);
5633         insert = expand_string(sub[2]);
5634         if (insert == NULL) goto EXPAND_FAILED;
5635         yield = string_cat(yield, insert);
5636
5637         moffset = ovector[1];
5638         moffsetextra = 0;
5639         emptyopt = 0;
5640
5641         /* If we have matched an empty string, first check to see if we are at
5642         the end of the subject. If so, the loop is over. Otherwise, mimic
5643         what Perl's /g options does. This turns out to be rather cunning. First
5644         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5645         string at the same point. If this fails (picked up above) we advance to
5646         the next character. */
5647
5648         if (ovector[0] == ovector[1])
5649           {
5650           if (ovector[0] == slen) break;
5651           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5652           }
5653         }
5654
5655       /* All done - restore numerical variables. */
5656
5657       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5658         save_expand_nlength);
5659       continue;
5660       }
5661
5662     /* Handle keyed and numbered substring extraction. If the first argument
5663     consists entirely of digits, then a numerical extraction is assumed. */
5664
5665     case EITEM_EXTRACT:
5666       {
5667       int field_number = 1;
5668       BOOL field_number_set = FALSE;
5669       uschar *save_lookup_value = lookup_value;
5670       uschar *sub[3];
5671       int save_expand_nmax =
5672         save_expand_strings(save_expand_nstring, save_expand_nlength);
5673
5674       /* On reflection the original behaviour of extract-json for a string
5675       result, leaving it quoted, was a mistake.  But it was already published,
5676       hence the addition of jsons.  In a future major version, make json
5677       work like josons, and withdraw jsons. */
5678
5679       enum {extract_basic, extract_json, extract_jsons} fmt = extract_basic;
5680
5681       while (isspace(*s)) s++;
5682
5683       /* Check for a format-variant specifier */
5684
5685       if (*s != '{')                                    /*}*/
5686         if (Ustrncmp(s, "json", 4) == 0)
5687           if (*(s += 4) == 's')
5688             {fmt = extract_jsons; s++;}
5689           else
5690             fmt = extract_json;
5691
5692       /* While skipping we cannot rely on the data for expansions being
5693       available (eg. $item) hence cannot decide on numeric vs. keyed.
5694       Read a maximum of 5 arguments (including the yes/no) */
5695
5696       if (skipping)
5697         {
5698         for (int j = 5; j > 0 && *s == '{'; j--)                /*'}'*/
5699           {
5700           if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))
5701             goto EXPAND_FAILED;                                 /*'{'*/
5702           if (*s++ != '}')
5703             {
5704             expand_string_message = US"missing '{' for arg of extract";
5705             goto EXPAND_FAILED_CURLY;
5706             }
5707           while (isspace(*s)) s++;
5708           }
5709         if (  Ustrncmp(s, "fail", 4) == 0                       /*'{'*/
5710            && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4])
5711            )
5712           {
5713           s += 4;
5714           while (isspace(*s)) s++;
5715           }                                                     /*'{'*/
5716         if (*s != '}')
5717           {
5718           expand_string_message = US"missing '}' closing extract";
5719           goto EXPAND_FAILED_CURLY;
5720           }
5721         }
5722
5723       else for (int i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */
5724         {
5725         while (isspace(*s)) s++;
5726         if (*s == '{')                                          /*'}'*/
5727           {
5728           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5729           if (sub[i] == NULL) goto EXPAND_FAILED;               /*'{'*/
5730           if (*s++ != '}')
5731             {
5732             expand_string_message = string_sprintf(
5733               "missing '}' closing arg %d of extract", i+1);
5734             goto EXPAND_FAILED_CURLY;
5735             }
5736
5737           /* After removal of leading and trailing white space, the first
5738           argument must not be empty; if it consists entirely of digits
5739           (optionally preceded by a minus sign), this is a numerical
5740           extraction, and we expect 3 arguments (normal) or 2 (json). */
5741
5742           if (i == 0)
5743             {
5744             int len;
5745             int x = 0;
5746             uschar *p = sub[0];
5747
5748             while (isspace(*p)) p++;
5749             sub[0] = p;
5750
5751             len = Ustrlen(p);
5752             while (len > 0 && isspace(p[len-1])) len--;
5753             p[len] = 0;
5754
5755             if (*p == 0)
5756               {
5757               expand_string_message = US"first argument of \"extract\" must "
5758                 "not be empty";
5759               goto EXPAND_FAILED;
5760               }
5761
5762             if (*p == '-')
5763               {
5764               field_number = -1;
5765               p++;
5766               }
5767             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5768             if (*p == 0)
5769               {
5770               field_number *= x;
5771               if (fmt == extract_basic) j = 3;               /* Need 3 args */
5772               field_number_set = TRUE;
5773               }
5774             }
5775           }
5776         else
5777           {
5778           expand_string_message = string_sprintf(
5779             "missing '{' for arg %d of extract", i+1);
5780           goto EXPAND_FAILED_CURLY;
5781           }
5782         }
5783
5784       /* Extract either the numbered or the keyed substring into $value. If
5785       skipping, just pretend the extraction failed. */
5786
5787       if (skipping)
5788         lookup_value = NULL;
5789       else switch (fmt)
5790         {
5791         case extract_basic:
5792           lookup_value = field_number_set
5793             ? expand_gettokened(field_number, sub[1], sub[2])
5794             : expand_getkeyed(sub[0], sub[1]);
5795           break;
5796
5797         case extract_json:
5798         case extract_jsons:
5799           {
5800           uschar * s, * item;
5801           const uschar * list;
5802
5803           /* Array: Bracket-enclosed and comma-separated.
5804           Object: Brace-enclosed, comma-sep list of name:value pairs */
5805
5806           if (!(s = dewrap(sub[1], field_number_set ? US"[]" : US"{}")))
5807             {
5808             expand_string_message =
5809               string_sprintf("%s wrapping %s for extract json",
5810                 expand_string_message,
5811                 field_number_set ? "array" : "object");
5812             goto EXPAND_FAILED_CURLY;
5813             }
5814
5815           list = s;
5816           if (field_number_set)
5817             {
5818             if (field_number <= 0)
5819               {
5820               expand_string_message = US"first argument of \"extract\" must "
5821                 "be greater than zero";
5822               goto EXPAND_FAILED;
5823               }
5824             while (field_number > 0 && (item = json_nextinlist(&list)))
5825               field_number--;
5826             if ((lookup_value = s = item))
5827               {
5828               while (*s) s++;
5829               while (--s >= lookup_value && isspace(*s)) *s = '\0';
5830               }
5831             }
5832           else
5833             {
5834             lookup_value = NULL;
5835             while ((item = json_nextinlist(&list)))
5836               {
5837               /* Item is:  string name-sep value.  string is quoted.
5838               Dequote the string and compare with the search key. */
5839
5840               if (!(item = dewrap(item, US"\"\"")))
5841                 {
5842                 expand_string_message =
5843                   string_sprintf("%s wrapping string key for extract json",
5844                     expand_string_message);
5845                 goto EXPAND_FAILED_CURLY;
5846                 }
5847               if (Ustrcmp(item, sub[0]) == 0)   /*XXX should be a UTF8-compare */
5848                 {
5849                 s = item + Ustrlen(item) + 1;
5850                 while (isspace(*s)) s++;
5851                 if (*s != ':')
5852                   {
5853                   expand_string_message = string_sprintf(
5854                     "missing object value-separator for extract json");
5855                   goto EXPAND_FAILED_CURLY;
5856                   }
5857                 s++;
5858                 while (isspace(*s)) s++;
5859                 lookup_value = s;
5860                 break;
5861                 }
5862               }
5863             }
5864           }
5865
5866           if (  fmt == extract_jsons
5867              && lookup_value
5868              && !(lookup_value = dewrap(lookup_value, US"\"\"")))
5869             {
5870             expand_string_message =
5871               string_sprintf("%s wrapping string result for extract jsons",
5872                 expand_string_message);
5873             goto EXPAND_FAILED_CURLY;
5874             }
5875           break;        /* json/s */
5876         }
5877
5878       /* If no string follows, $value gets substituted; otherwise there can
5879       be yes/no strings, as for lookup or if. */
5880
5881       switch(process_yesno(
5882                skipping,                     /* were previously skipping */
5883                lookup_value != NULL,         /* success/failure indicator */
5884                save_lookup_value,            /* value to reset for string2 */
5885                &s,                           /* input pointer */
5886                &yield,                       /* output pointer */
5887                US"extract",                  /* condition type */
5888                &resetok))
5889         {
5890         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5891         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5892         }
5893
5894       /* All done - restore numerical variables. */
5895
5896       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5897         save_expand_nlength);
5898
5899       continue;
5900       }
5901
5902     /* return the Nth item from a list */
5903
5904     case EITEM_LISTEXTRACT:
5905       {
5906       int field_number = 1;
5907       uschar *save_lookup_value = lookup_value;
5908       uschar *sub[2];
5909       int save_expand_nmax =
5910         save_expand_strings(save_expand_nstring, save_expand_nlength);
5911
5912       /* Read the field & list arguments */
5913
5914       for (int i = 0; i < 2; i++)
5915         {
5916         while (isspace(*s)) s++;
5917         if (*s != '{')                                  /*'}'*/
5918           {
5919           expand_string_message = string_sprintf(
5920             "missing '{' for arg %d of listextract", i+1);
5921           goto EXPAND_FAILED_CURLY;
5922           }
5923
5924         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5925         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5926         if (*s++ != '}')
5927           {
5928           expand_string_message = string_sprintf(
5929             "missing '}' closing arg %d of listextract", i+1);
5930           goto EXPAND_FAILED_CURLY;
5931           }
5932
5933         /* After removal of leading and trailing white space, the first
5934         argument must be numeric and nonempty. */
5935
5936         if (i == 0)
5937           {
5938           int len;
5939           int x = 0;
5940           uschar *p = sub[0];
5941
5942           while (isspace(*p)) p++;
5943           sub[0] = p;
5944
5945           len = Ustrlen(p);
5946           while (len > 0 && isspace(p[len-1])) len--;
5947           p[len] = 0;
5948
5949           if (!*p && !skipping)
5950             {
5951             expand_string_message = US"first argument of \"listextract\" must "
5952               "not be empty";
5953             goto EXPAND_FAILED;
5954             }
5955
5956           if (*p == '-')
5957             {
5958             field_number = -1;
5959             p++;
5960             }
5961           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5962           if (*p)
5963             {
5964             expand_string_message = US"first argument of \"listextract\" must "
5965               "be numeric";
5966             goto EXPAND_FAILED;
5967             }
5968           field_number *= x;
5969           }
5970         }
5971
5972       /* Extract the numbered element into $value. If
5973       skipping, just pretend the extraction failed. */
5974
5975       lookup_value = skipping ? NULL : expand_getlistele(field_number, sub[1]);
5976
5977       /* If no string follows, $value gets substituted; otherwise there can
5978       be yes/no strings, as for lookup or if. */
5979
5980       switch(process_yesno(
5981                skipping,                     /* were previously skipping */
5982                lookup_value != NULL,         /* success/failure indicator */
5983                save_lookup_value,            /* value to reset for string2 */
5984                &s,                           /* input pointer */
5985                &yield,                       /* output pointer */
5986                US"listextract",              /* condition type */
5987                &resetok))
5988         {
5989         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5990         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5991         }
5992
5993       /* All done - restore numerical variables. */
5994
5995       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5996         save_expand_nlength);
5997
5998       continue;
5999       }
6000
6001 #ifndef DISABLE_TLS
6002     case EITEM_CERTEXTRACT:
6003       {
6004       uschar *save_lookup_value = lookup_value;
6005       uschar *sub[2];
6006       int save_expand_nmax =
6007         save_expand_strings(save_expand_nstring, save_expand_nlength);
6008
6009       /* Read the field argument */
6010       while (isspace(*s)) s++;
6011       if (*s != '{')                                    /*}*/
6012         {
6013         expand_string_message = US"missing '{' for field arg of certextract";
6014         goto EXPAND_FAILED_CURLY;
6015         }
6016       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6017       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
6018       if (*s++ != '}')
6019         {
6020         expand_string_message = US"missing '}' closing field arg of certextract";
6021         goto EXPAND_FAILED_CURLY;
6022         }
6023       /* strip spaces fore & aft */
6024       {
6025       int len;
6026       uschar *p = sub[0];
6027
6028       while (isspace(*p)) p++;
6029       sub[0] = p;
6030
6031       len = Ustrlen(p);
6032       while (len > 0 && isspace(p[len-1])) len--;
6033       p[len] = 0;
6034       }
6035
6036       /* inspect the cert argument */
6037       while (isspace(*s)) s++;
6038       if (*s != '{')                                    /*}*/
6039         {
6040         expand_string_message = US"missing '{' for cert variable arg of certextract";
6041         goto EXPAND_FAILED_CURLY;
6042         }
6043       if (*++s != '$')
6044         {
6045         expand_string_message = US"second argument of \"certextract\" must "
6046           "be a certificate variable";
6047         goto EXPAND_FAILED;
6048         }
6049       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
6050       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
6051       if (*s++ != '}')
6052         {
6053         expand_string_message = US"missing '}' closing cert variable arg of certextract";
6054         goto EXPAND_FAILED_CURLY;
6055         }
6056
6057       if (skipping)
6058         lookup_value = NULL;
6059       else
6060         {
6061         lookup_value = expand_getcertele(sub[0], sub[1]);
6062         if (*expand_string_message) goto EXPAND_FAILED;
6063         }
6064       switch(process_yesno(
6065                skipping,                     /* were previously skipping */
6066                lookup_value != NULL,         /* success/failure indicator */
6067                save_lookup_value,            /* value to reset for string2 */
6068                &s,                           /* input pointer */
6069                &yield,                       /* output pointer */
6070                US"certextract",              /* condition type */
6071                &resetok))
6072         {
6073         case 1: goto EXPAND_FAILED;          /* when all is well, the */
6074         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
6075         }
6076
6077       restore_expand_strings(save_expand_nmax, save_expand_nstring,
6078         save_expand_nlength);
6079       continue;
6080       }
6081 #endif  /*DISABLE_TLS*/
6082
6083     /* Handle list operations */
6084
6085     case EITEM_FILTER:
6086     case EITEM_MAP:
6087     case EITEM_REDUCE:
6088       {
6089       int sep = 0;
6090       int save_ptr = yield->ptr;
6091       uschar outsep[2] = { '\0', '\0' };
6092       const uschar *list, *expr, *temp;
6093       uschar *save_iterate_item = iterate_item;
6094       uschar *save_lookup_value = lookup_value;
6095
6096       while (isspace(*s)) s++;
6097       if (*s++ != '{')
6098         {
6099         expand_string_message =
6100           string_sprintf("missing '{' for first arg of %s", name);
6101         goto EXPAND_FAILED_CURLY;
6102         }
6103
6104       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
6105       if (list == NULL) goto EXPAND_FAILED;
6106       if (*s++ != '}')
6107         {
6108         expand_string_message =
6109           string_sprintf("missing '}' closing first arg of %s", name);
6110         goto EXPAND_FAILED_CURLY;
6111         }
6112
6113       if (item_type == EITEM_REDUCE)
6114         {
6115         uschar * t;
6116         while (isspace(*s)) s++;
6117         if (*s++ != '{')
6118           {
6119           expand_string_message = US"missing '{' for second arg of reduce";
6120           goto EXPAND_FAILED_CURLY;
6121           }
6122         t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
6123         if (!t) goto EXPAND_FAILED;
6124         lookup_value = t;
6125         if (*s++ != '}')
6126           {
6127           expand_string_message = US"missing '}' closing second arg of reduce";
6128           goto EXPAND_FAILED_CURLY;
6129           }
6130         }
6131
6132       while (isspace(*s)) s++;
6133       if (*s++ != '{')
6134         {
6135         expand_string_message =
6136           string_sprintf("missing '{' for last arg of %s", name);
6137         goto EXPAND_FAILED_CURLY;
6138         }
6139
6140       expr = s;
6141
6142       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
6143       if scanning a "false" part). This allows us to find the end of the
6144       condition, because if the list is empty, we won't actually evaluate the
6145       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
6146       the normal internal expansion function. */
6147
6148       if (item_type == EITEM_FILTER)
6149         {
6150         temp = eval_condition(expr, &resetok, NULL);
6151         if (temp != NULL) s = temp;
6152         }
6153       else
6154         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
6155
6156       if (temp == NULL)
6157         {
6158         expand_string_message = string_sprintf("%s inside \"%s\" item",
6159           expand_string_message, name);
6160         goto EXPAND_FAILED;
6161         }
6162
6163       while (isspace(*s)) s++;
6164       if (*s++ != '}')
6165         {                                               /*{*/
6166         expand_string_message = string_sprintf("missing } at end of condition "
6167           "or expression inside \"%s\"; could be an unquoted } in the content",
6168           name);
6169         goto EXPAND_FAILED;
6170         }
6171
6172       while (isspace(*s)) s++;                          /*{*/
6173       if (*s++ != '}')
6174         {                                               /*{*/
6175         expand_string_message = string_sprintf("missing } at end of \"%s\"",
6176           name);
6177         goto EXPAND_FAILED;
6178         }
6179
6180       /* If we are skipping, we can now just move on to the next item. When
6181       processing for real, we perform the iteration. */
6182
6183       if (skipping) continue;
6184       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)))
6185         {
6186         *outsep = (uschar)sep;      /* Separator as a string */
6187
6188         DEBUG(D_expand) debug_printf_indent("%s: $item = '%s'  $value = '%s'\n",
6189                           name, iterate_item, lookup_value);
6190
6191         if (item_type == EITEM_FILTER)
6192           {
6193           BOOL condresult;
6194           if (eval_condition(expr, &resetok, &condresult) == NULL)
6195             {
6196             iterate_item = save_iterate_item;
6197             lookup_value = save_lookup_value;
6198             expand_string_message = string_sprintf("%s inside \"%s\" condition",
6199               expand_string_message, name);
6200             goto EXPAND_FAILED;
6201             }
6202           DEBUG(D_expand) debug_printf_indent("%s: condition is %s\n", name,
6203             condresult? "true":"false");
6204           if (condresult)
6205             temp = iterate_item;    /* TRUE => include this item */
6206           else
6207             continue;               /* FALSE => skip this item */
6208           }
6209
6210         /* EITEM_MAP and EITEM_REDUCE */
6211
6212         else
6213           {
6214           uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
6215           temp = t;
6216           if (temp == NULL)
6217             {
6218             iterate_item = save_iterate_item;
6219             expand_string_message = string_sprintf("%s inside \"%s\" item",
6220               expand_string_message, name);
6221             goto EXPAND_FAILED;
6222             }
6223           if (item_type == EITEM_REDUCE)
6224             {
6225             lookup_value = t;         /* Update the value of $value */
6226             continue;                 /* and continue the iteration */
6227             }
6228           }
6229
6230         /* We reach here for FILTER if the condition is true, always for MAP,
6231         and never for REDUCE. The value in "temp" is to be added to the output
6232         list that is being created, ensuring that any occurrences of the
6233         separator character are doubled. Unless we are dealing with the first
6234         item of the output list, add in a space if the new item begins with the
6235         separator character, or is an empty string. */
6236
6237         if (yield->ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
6238           yield = string_catn(yield, US" ", 1);
6239
6240         /* Add the string in "temp" to the output list that we are building,
6241         This is done in chunks by searching for the separator character. */
6242
6243         for (;;)
6244           {
6245           size_t seglen = Ustrcspn(temp, outsep);
6246
6247           yield = string_catn(yield, temp, seglen + 1);
6248
6249           /* If we got to the end of the string we output one character
6250           too many; backup and end the loop. Otherwise arrange to double the
6251           separator. */
6252
6253           if (temp[seglen] == '\0') { yield->ptr--; break; }
6254           yield = string_catn(yield, outsep, 1);
6255           temp += seglen + 1;
6256           }
6257
6258         /* Output a separator after the string: we will remove the redundant
6259         final one at the end. */
6260
6261         yield = string_catn(yield, outsep, 1);
6262         }   /* End of iteration over the list loop */
6263
6264       /* REDUCE has generated no output above: output the final value of
6265       $value. */
6266
6267       if (item_type == EITEM_REDUCE)
6268         {
6269         yield = string_cat(yield, lookup_value);
6270         lookup_value = save_lookup_value;  /* Restore $value */
6271         }
6272
6273       /* FILTER and MAP generate lists: if they have generated anything, remove
6274       the redundant final separator. Even though an empty item at the end of a
6275       list does not count, this is tidier. */
6276
6277       else if (yield->ptr != save_ptr) yield->ptr--;
6278
6279       /* Restore preserved $item */
6280
6281       iterate_item = save_iterate_item;
6282       continue;
6283       }
6284
6285     case EITEM_SORT:
6286       {
6287       int sep = 0;
6288       const uschar *srclist, *cmp, *xtract;
6289       uschar *srcitem;
6290       const uschar *dstlist = NULL, *dstkeylist = NULL;
6291       uschar * tmp;
6292       uschar *save_iterate_item = iterate_item;
6293
6294       while (isspace(*s)) s++;
6295       if (*s++ != '{')
6296         {
6297         expand_string_message = US"missing '{' for list arg of sort";
6298         goto EXPAND_FAILED_CURLY;
6299         }
6300
6301       srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
6302       if (!srclist) goto EXPAND_FAILED;
6303       if (*s++ != '}')
6304         {
6305         expand_string_message = US"missing '}' closing list arg of sort";
6306         goto EXPAND_FAILED_CURLY;
6307         }
6308
6309       while (isspace(*s)) s++;
6310       if (*s++ != '{')
6311         {
6312         expand_string_message = US"missing '{' for comparator arg of sort";
6313         goto EXPAND_FAILED_CURLY;
6314         }
6315
6316       cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok);
6317       if (!cmp) goto EXPAND_FAILED;
6318       if (*s++ != '}')
6319         {
6320         expand_string_message = US"missing '}' closing comparator arg of sort";
6321         goto EXPAND_FAILED_CURLY;
6322         }
6323
6324       while (isspace(*s)) s++;
6325       if (*s++ != '{')
6326         {
6327         expand_string_message = US"missing '{' for extractor arg of sort";
6328         goto EXPAND_FAILED_CURLY;
6329         }
6330
6331       xtract = s;
6332       tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
6333       if (!tmp) goto EXPAND_FAILED;
6334       xtract = string_copyn(xtract, s - xtract);
6335
6336       if (*s++ != '}')
6337         {
6338         expand_string_message = US"missing '}' closing extractor arg of sort";
6339         goto EXPAND_FAILED_CURLY;
6340         }
6341                                                         /*{*/
6342       if (*s++ != '}')
6343         {                                               /*{*/
6344         expand_string_message = US"missing } at end of \"sort\"";
6345         goto EXPAND_FAILED;
6346         }
6347
6348       if (skipping) continue;
6349
6350       while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0)))
6351         {
6352         uschar * dstitem;
6353         gstring * newlist = NULL;
6354         gstring * newkeylist = NULL;
6355         uschar * srcfield;
6356
6357         DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, srcitem);
6358
6359         /* extract field for comparisons */
6360         iterate_item = srcitem;
6361         if (  !(srcfield = expand_string_internal(xtract, FALSE, NULL, FALSE,
6362                                           TRUE, &resetok))
6363            || !*srcfield)
6364           {
6365           expand_string_message = string_sprintf(
6366               "field-extract in sort: \"%s\"", xtract);
6367           goto EXPAND_FAILED;
6368           }
6369
6370         /* Insertion sort */
6371
6372         /* copy output list until new-item < list-item */
6373         while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6374           {
6375           uschar * dstfield;
6376           uschar * expr;
6377           BOOL before;
6378
6379           /* field for comparison */
6380           if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6381             goto sort_mismatch;
6382
6383           /* build and run condition string */
6384           expr = string_sprintf("%s{%s}{%s}", cmp, srcfield, dstfield);
6385
6386           DEBUG(D_expand) debug_printf_indent("%s: cond = \"%s\"\n", name, expr);
6387           if (!eval_condition(expr, &resetok, &before))
6388             {
6389             expand_string_message = string_sprintf("comparison in sort: %s",
6390                 expr);
6391             goto EXPAND_FAILED;
6392             }
6393
6394           if (before)
6395             {
6396             /* New-item sorts before this dst-item.  Append new-item,
6397             then dst-item, then remainder of dst list. */
6398
6399             newlist = string_append_listele(newlist, sep, srcitem);
6400             newkeylist = string_append_listele(newkeylist, sep, srcfield);
6401             srcitem = NULL;
6402
6403             newlist = string_append_listele(newlist, sep, dstitem);
6404             newkeylist = string_append_listele(newkeylist, sep, dstfield);
6405
6406             while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0)))
6407               {
6408               if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0)))
6409                 goto sort_mismatch;
6410               newlist = string_append_listele(newlist, sep, dstitem);
6411               newkeylist = string_append_listele(newkeylist, sep, dstfield);
6412               }
6413
6414             break;
6415             }
6416
6417           newlist = string_append_listele(newlist, sep, dstitem);
6418           newkeylist = string_append_listele(newkeylist, sep, dstfield);
6419           }
6420
6421         /* If we ran out of dstlist without consuming srcitem, append it */
6422         if (srcitem)
6423           {
6424           newlist = string_append_listele(newlist, sep, srcitem);
6425           newkeylist = string_append_listele(newkeylist, sep, srcfield);
6426           }
6427
6428         dstlist = newlist->s;
6429         dstkeylist = newkeylist->s;
6430
6431         DEBUG(D_expand) debug_printf_indent("%s: dstlist = \"%s\"\n", name, dstlist);
6432         DEBUG(D_expand) debug_printf_indent("%s: dstkeylist = \"%s\"\n", name, dstkeylist);
6433         }
6434
6435       if (dstlist)
6436         yield = string_cat(yield, dstlist);
6437
6438       /* Restore preserved $item */
6439       iterate_item = save_iterate_item;
6440       continue;
6441
6442       sort_mismatch:
6443         expand_string_message = US"Internal error in sort (list mismatch)";
6444         goto EXPAND_FAILED;
6445       }
6446
6447
6448     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
6449     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
6450     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
6451     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
6452
6453     #define EXPAND_DLFUNC_MAX_ARGS 8
6454
6455     case EITEM_DLFUNC:
6456 #ifndef EXPAND_DLFUNC
6457       expand_string_message = US"\"${dlfunc\" encountered, but this facility "  /*}*/
6458         "is not included in this binary";
6459       goto EXPAND_FAILED;
6460
6461 #else   /* EXPAND_DLFUNC */
6462       {
6463       tree_node *t;
6464       exim_dlfunc_t *func;
6465       uschar *result;
6466       int status, argc;
6467       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
6468
6469       if ((expand_forbid & RDO_DLFUNC) != 0)
6470         {
6471         expand_string_message =
6472           US"dynamically-loaded functions are not permitted";
6473         goto EXPAND_FAILED;
6474         }
6475
6476       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
6477            TRUE, US"dlfunc", &resetok))
6478         {
6479         case 1: goto EXPAND_FAILED_CURLY;
6480         case 2:
6481         case 3: goto EXPAND_FAILED;
6482         }
6483
6484       /* If skipping, we don't actually do anything */
6485
6486       if (skipping) continue;
6487
6488       /* Look up the dynamically loaded object handle in the tree. If it isn't
6489       found, dlopen() the file and put the handle in the tree for next time. */
6490
6491       if (!(t = tree_search(dlobj_anchor, argv[0])))
6492         {
6493         void *handle = dlopen(CS argv[0], RTLD_LAZY);
6494         if (handle == NULL)
6495           {
6496           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
6497             argv[0], dlerror());
6498           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6499           goto EXPAND_FAILED;
6500           }
6501         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
6502         Ustrcpy(t->name, argv[0]);
6503         t->data.ptr = handle;
6504         (void)tree_insertnode(&dlobj_anchor, t);
6505         }
6506
6507       /* Having obtained the dynamically loaded object handle, look up the
6508       function pointer. */
6509
6510       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
6511       if (func == NULL)
6512         {
6513         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
6514           "%s", argv[1], argv[0], dlerror());
6515         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
6516         goto EXPAND_FAILED;
6517         }
6518
6519       /* Call the function and work out what to do with the result. If it
6520       returns OK, we have a replacement string; if it returns DEFER then
6521       expansion has failed in a non-forced manner; if it returns FAIL then
6522       failure was forced; if it returns ERROR or any other value there's a
6523       problem, so panic slightly. In any case, assume that the function has
6524       side-effects on the store that must be preserved. */
6525
6526       resetok = FALSE;
6527       result = NULL;
6528       for (argc = 0; argv[argc] != NULL; argc++);
6529       status = func(&result, argc - 2, &argv[2]);
6530       if(status == OK)
6531         {
6532         if (result == NULL) result = US"";
6533         yield = string_cat(yield, result);
6534         continue;
6535         }
6536       else
6537         {
6538         expand_string_message = result == NULL ? US"(no message)" : result;
6539         if(status == FAIL_FORCED) f.expand_string_forcedfail = TRUE;
6540           else if(status != FAIL)
6541             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
6542               argv[0], argv[1], status, expand_string_message);
6543         goto EXPAND_FAILED;
6544         }
6545       }
6546 #endif /* EXPAND_DLFUNC */
6547
6548     case EITEM_ENV:     /* ${env {name} {val_if_found} {val_if_unfound}} */
6549       {
6550       uschar * key;
6551       uschar *save_lookup_value = lookup_value;
6552
6553       while (isspace(*s)) s++;
6554       if (*s != '{')                                    /*}*/
6555         goto EXPAND_FAILED;
6556
6557       key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6558       if (!key) goto EXPAND_FAILED;                     /*{*/
6559       if (*s++ != '}')
6560         {
6561         expand_string_message = US"missing '{' for name arg of env";
6562         goto EXPAND_FAILED_CURLY;
6563         }
6564
6565       lookup_value = US getenv(CS key);
6566
6567       switch(process_yesno(
6568                skipping,                     /* were previously skipping */
6569                lookup_value != NULL,         /* success/failure indicator */
6570                save_lookup_value,            /* value to reset for string2 */
6571                &s,                           /* input pointer */
6572                &yield,                       /* output pointer */
6573                US"env",                      /* condition type */
6574                &resetok))
6575         {
6576         case 1: goto EXPAND_FAILED;          /* when all is well, the */
6577         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
6578         }
6579       continue;
6580       }
6581     }   /* EITEM_* switch */
6582
6583   /* Control reaches here if the name is not recognized as one of the more
6584   complicated expansion items. Check for the "operator" syntax (name terminated
6585   by a colon). Some of the operators have arguments, separated by _ from the
6586   name. */
6587
6588   if (*s == ':')
6589     {
6590     int c;
6591     uschar *arg = NULL;
6592     uschar *sub;
6593 #ifndef DISABLE_TLS
6594     var_entry *vp = NULL;
6595 #endif
6596
6597     /* Owing to an historical mis-design, an underscore may be part of the
6598     operator name, or it may introduce arguments.  We therefore first scan the
6599     table of names that contain underscores. If there is no match, we cut off
6600     the arguments and then scan the main table. */
6601
6602     if ((c = chop_match(name, op_table_underscore,
6603                         nelem(op_table_underscore))) < 0)
6604       {
6605       arg = Ustrchr(name, '_');
6606       if (arg != NULL) *arg = 0;
6607       c = chop_match(name, op_table_main, nelem(op_table_main));
6608       if (c >= 0) c += nelem(op_table_underscore);
6609       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
6610       }
6611
6612     /* Deal specially with operators that might take a certificate variable
6613     as we do not want to do the usual expansion. For most, expand the string.*/
6614     switch(c)
6615       {
6616 #ifndef DISABLE_TLS
6617       case EOP_MD5:
6618       case EOP_SHA1:
6619       case EOP_SHA256:
6620       case EOP_BASE64:
6621         if (s[1] == '$')
6622           {
6623           const uschar * s1 = s;
6624           sub = expand_string_internal(s+2, TRUE, &s1, skipping,
6625                   FALSE, &resetok);
6626           if (!sub)       goto EXPAND_FAILED;           /*{*/
6627           if (*s1 != '}')
6628             {
6629             expand_string_message =
6630               string_sprintf("missing '}' closing cert arg of %s", name);
6631             goto EXPAND_FAILED_CURLY;
6632             }
6633           if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
6634             {
6635             s = s1+1;
6636             break;
6637             }
6638           vp = NULL;
6639           }
6640         /*FALLTHROUGH*/
6641 #endif
6642       default:
6643         sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
6644         if (!sub) goto EXPAND_FAILED;
6645         s++;
6646         break;
6647       }
6648
6649     /* If we are skipping, we don't need to perform the operation at all.
6650     This matters for operations like "mask", because the data may not be
6651     in the correct format when skipping. For example, the expression may test
6652     for the existence of $sender_host_address before trying to mask it. For
6653     other operations, doing them may not fail, but it is a waste of time. */
6654
6655     if (skipping && c >= 0) continue;
6656
6657     /* Otherwise, switch on the operator type */
6658
6659     switch(c)
6660       {
6661       case EOP_BASE32:
6662         {
6663         uschar *t;
6664         unsigned long int n = Ustrtoul(sub, &t, 10);
6665         gstring * g = NULL;
6666
6667         if (*t != 0)
6668           {
6669           expand_string_message = string_sprintf("argument for base32 "
6670             "operator is \"%s\", which is not a decimal number", sub);
6671           goto EXPAND_FAILED;
6672           }
6673         for ( ; n; n >>= 5)
6674           g = string_catn(g, &base32_chars[n & 0x1f], 1);
6675
6676         if (g) while (g->ptr > 0) yield = string_catn(yield, &g->s[--g->ptr], 1);
6677         continue;
6678         }
6679
6680       case EOP_BASE32D:
6681         {
6682         uschar *tt = sub;
6683         unsigned long int n = 0;
6684         uschar * s;
6685         while (*tt)
6686           {
6687           uschar * t = Ustrchr(base32_chars, *tt++);
6688           if (t == NULL)
6689             {
6690             expand_string_message = string_sprintf("argument for base32d "
6691               "operator is \"%s\", which is not a base 32 number", sub);
6692             goto EXPAND_FAILED;
6693             }
6694           n = n * 32 + (t - base32_chars);
6695           }
6696         s = string_sprintf("%ld", n);
6697         yield = string_cat(yield, s);
6698         continue;
6699         }
6700
6701       case EOP_BASE62:
6702         {
6703         uschar *t;
6704         unsigned long int n = Ustrtoul(sub, &t, 10);
6705         if (*t != 0)
6706           {
6707           expand_string_message = string_sprintf("argument for base62 "
6708             "operator is \"%s\", which is not a decimal number", sub);
6709           goto EXPAND_FAILED;
6710           }
6711         t = string_base62(n);
6712         yield = string_cat(yield, t);
6713         continue;
6714         }
6715
6716       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
6717
6718       case EOP_BASE62D:
6719         {
6720         uschar *tt = sub;
6721         unsigned long int n = 0;
6722         while (*tt != 0)
6723           {
6724           uschar *t = Ustrchr(base62_chars, *tt++);
6725           if (t == NULL)
6726             {
6727             expand_string_message = string_sprintf("argument for base62d "
6728               "operator is \"%s\", which is not a base %d number", sub,
6729               BASE_62);
6730             goto EXPAND_FAILED;
6731             }
6732           n = n * BASE_62 + (t - base62_chars);
6733           }
6734         yield = string_fmt_append(yield, "%ld", n);
6735         continue;
6736         }
6737
6738       case EOP_EXPAND:
6739         {
6740         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
6741         if (expanded == NULL)
6742           {
6743           expand_string_message =
6744             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
6745               expand_string_message);
6746           goto EXPAND_FAILED;
6747           }
6748         yield = string_cat(yield, expanded);
6749         continue;
6750         }
6751
6752       case EOP_LC:
6753         {
6754         int count = 0;
6755         uschar *t = sub - 1;
6756         while (*(++t) != 0) { *t = tolower(*t); count++; }
6757         yield = string_catn(yield, sub, count);
6758         continue;
6759         }
6760
6761       case EOP_UC:
6762         {
6763         int count = 0;
6764         uschar *t = sub - 1;
6765         while (*(++t) != 0) { *t = toupper(*t); count++; }
6766         yield = string_catn(yield, sub, count);
6767         continue;
6768         }
6769
6770       case EOP_MD5:
6771 #ifndef DISABLE_TLS
6772         if (vp && *(void **)vp->value)
6773           {
6774           uschar * cp = tls_cert_fprt_md5(*(void **)vp->value);
6775           yield = string_cat(yield, cp);
6776           }
6777         else
6778 #endif
6779           {
6780           md5 base;
6781           uschar digest[16];
6782           md5_start(&base);
6783           md5_end(&base, sub, Ustrlen(sub), digest);
6784           for (int j = 0; j < 16; j++)
6785             yield = string_fmt_append(yield, "%02x", digest[j]);
6786           }
6787         continue;
6788
6789       case EOP_SHA1:
6790 #ifndef DISABLE_TLS
6791         if (vp && *(void **)vp->value)
6792           {
6793           uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value);
6794           yield = string_cat(yield, cp);
6795           }
6796         else
6797 #endif
6798           {
6799           hctx h;
6800           uschar digest[20];
6801           sha1_start(&h);
6802           sha1_end(&h, sub, Ustrlen(sub), digest);
6803           for (int j = 0; j < 20; j++)
6804             yield = string_fmt_append(yield, "%02X", digest[j]);
6805           }
6806         continue;
6807
6808       case EOP_SHA2:
6809       case EOP_SHA256:
6810 #ifdef EXIM_HAVE_SHA2
6811         if (vp && *(void **)vp->value)
6812           if (c == EOP_SHA256)
6813             yield = string_cat(yield, tls_cert_fprt_sha256(*(void **)vp->value));
6814           else
6815             expand_string_message = US"sha2_N not supported with certificates";
6816         else
6817           {
6818           hctx h;
6819           blob b;
6820           hashmethod m = !arg ? HASH_SHA2_256
6821             : Ustrcmp(arg, "256") == 0 ? HASH_SHA2_256
6822             : Ustrcmp(arg, "384") == 0 ? HASH_SHA2_384
6823             : Ustrcmp(arg, "512") == 0 ? HASH_SHA2_512
6824             : HASH_BADTYPE;
6825
6826           if (m == HASH_BADTYPE || !exim_sha_init(&h, m))
6827             {
6828             expand_string_message = US"unrecognised sha2 variant";
6829             goto EXPAND_FAILED;
6830             }
6831
6832           exim_sha_update(&h, sub, Ustrlen(sub));
6833           exim_sha_finish(&h, &b);
6834           while (b.len-- > 0)
6835             yield = string_fmt_append(yield, "%02X", *b.data++);
6836           }
6837 #else
6838           expand_string_message = US"sha256 only supported with TLS";
6839 #endif
6840         continue;
6841
6842       case EOP_SHA3:
6843 #ifdef EXIM_HAVE_SHA3
6844         {
6845         hctx h;
6846         blob b;
6847         hashmethod m = !arg ? HASH_SHA3_256
6848           : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224
6849           : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256
6850           : Ustrcmp(arg, "384") == 0 ? HASH_SHA3_384
6851           : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
6852           : HASH_BADTYPE;
6853
6854         if (m == HASH_BADTYPE || !exim_sha_init(&h, m))
6855           {
6856           expand_string_message = US"unrecognised sha3 variant";
6857           goto EXPAND_FAILED;
6858           }
6859
6860         exim_sha_update(&h, sub, Ustrlen(sub));
6861         exim_sha_finish(&h, &b);
6862         while (b.len-- > 0)
6863           yield = string_fmt_append(yield, "%02X", *b.data++);
6864         }
6865         continue;
6866 #else
6867         expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 + or OpenSSL 1.1.1 +";
6868         goto EXPAND_FAILED;
6869 #endif
6870
6871       /* Convert hex encoding to base64 encoding */
6872
6873       case EOP_HEX2B64:
6874         {
6875         int c = 0;
6876         int b = -1;
6877         uschar *in = sub;
6878         uschar *out = sub;
6879         uschar *enc;
6880
6881         for (enc = sub; *enc; enc++)
6882           {
6883           if (!isxdigit(*enc))
6884             {
6885             expand_string_message = string_sprintf("\"%s\" is not a hex "
6886               "string", sub);
6887             goto EXPAND_FAILED;
6888             }
6889           c++;
6890           }
6891
6892         if ((c & 1) != 0)
6893           {
6894           expand_string_message = string_sprintf("\"%s\" contains an odd "
6895             "number of characters", sub);
6896           goto EXPAND_FAILED;
6897           }
6898
6899         while ((c = *in++) != 0)
6900           {
6901           if (isdigit(c)) c -= '0';
6902           else c = toupper(c) - 'A' + 10;
6903           if (b == -1)
6904             b = c << 4;
6905           else
6906             {
6907             *out++ = b | c;
6908             b = -1;
6909             }
6910           }
6911
6912         enc = b64encode(CUS sub, out - sub);
6913         yield = string_cat(yield, enc);
6914         continue;
6915         }
6916
6917       /* Convert octets outside 0x21..0x7E to \xXX form */
6918
6919       case EOP_HEXQUOTE:
6920         {
6921         uschar *t = sub - 1;
6922         while (*(++t) != 0)
6923           {
6924           if (*t < 0x21 || 0x7E < *t)
6925             yield = string_fmt_append(yield, "\\x%02x", *t);
6926           else
6927             yield = string_catn(yield, t, 1);
6928           }
6929         continue;
6930         }
6931
6932       /* count the number of list elements */
6933
6934       case EOP_LISTCOUNT:
6935         {
6936         int cnt = 0;
6937         int sep = 0;
6938         uschar buffer[256];
6939
6940         while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
6941         yield = string_fmt_append(yield, "%d", cnt);
6942         continue;
6943         }
6944
6945       /* expand a named list given the name */
6946       /* handles nested named lists; requotes as colon-sep list */
6947
6948       case EOP_LISTNAMED:
6949         {
6950         tree_node *t = NULL;
6951         const uschar * list;
6952         int sep = 0;
6953         uschar * item;
6954         uschar * suffix = US"";
6955         BOOL needsep = FALSE;
6956         uschar buffer[256];
6957
6958         if (*sub == '+') sub++;
6959         if (arg == NULL)        /* no-argument version */
6960           {
6961           if (!(t = tree_search(addresslist_anchor, sub)) &&
6962               !(t = tree_search(domainlist_anchor,  sub)) &&
6963               !(t = tree_search(hostlist_anchor,    sub)))
6964             t = tree_search(localpartlist_anchor, sub);
6965           }
6966         else switch(*arg)       /* specific list-type version */
6967           {
6968           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
6969           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
6970           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
6971           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
6972           default:
6973             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
6974             goto EXPAND_FAILED;
6975           }
6976
6977         if(!t)
6978           {
6979           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
6980             sub, !arg?""
6981               : *arg=='a'?"address "
6982               : *arg=='d'?"domain "
6983               : *arg=='h'?"host "
6984               : *arg=='l'?"localpart "
6985               : 0);
6986           goto EXPAND_FAILED;
6987           }
6988
6989         list = ((namedlist_block *)(t->data.ptr))->string;
6990
6991         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))))
6992           {
6993           uschar * buf = US" : ";
6994           if (needsep)
6995             yield = string_catn(yield, buf, 3);
6996           else
6997             needsep = TRUE;
6998
6999           if (*item == '+')     /* list item is itself a named list */
7000             {
7001             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
7002             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
7003             }
7004           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
7005             {
7006             char * cp;
7007             char tok[3];
7008             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
7009             while ((cp= strpbrk(CCS item, tok)))
7010               {
7011               yield = string_catn(yield, item, cp - CS item);
7012               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
7013                 {
7014                 yield = string_catn(yield, US"::", 2);
7015                 item = US cp;
7016                 }
7017               else              /* sep in item; should already be doubled; emit once */
7018                 {
7019                 yield = string_catn(yield, US tok, 1);
7020                 if (*cp == sep) cp++;
7021                 item = US cp;
7022                 }
7023               }
7024             }
7025           yield = string_cat(yield, item);
7026           }
7027         continue;
7028         }
7029
7030       /* mask applies a mask to an IP address; for example the result of
7031       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
7032
7033       case EOP_MASK:
7034         {
7035         int count;
7036         uschar *endptr;
7037         int binary[4];
7038         int mask, maskoffset;
7039         int type = string_is_ip_address(sub, &maskoffset);
7040         uschar buffer[64];
7041
7042         if (type == 0)
7043           {
7044           expand_string_message = string_sprintf("\"%s\" is not an IP address",
7045            sub);
7046           goto EXPAND_FAILED;
7047           }
7048
7049         if (maskoffset == 0)
7050           {
7051           expand_string_message = string_sprintf("missing mask value in \"%s\"",
7052             sub);
7053           goto EXPAND_FAILED;
7054           }
7055
7056         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
7057
7058         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
7059           {
7060           expand_string_message = string_sprintf("mask value too big in \"%s\"",
7061             sub);
7062           goto EXPAND_FAILED;
7063           }
7064
7065         /* Convert the address to binary integer(s) and apply the mask */
7066
7067         sub[maskoffset] = 0;
7068         count = host_aton(sub, binary);
7069         host_mask(count, binary, mask);
7070
7071         /* Convert to masked textual format and add to output. */
7072
7073         yield = string_catn(yield, buffer,
7074           host_nmtoa(count, binary, mask, buffer, '.'));
7075         continue;
7076         }
7077
7078       case EOP_IPV6NORM:
7079       case EOP_IPV6DENORM:
7080         {
7081         int type = string_is_ip_address(sub, NULL);
7082         int binary[4];
7083         uschar buffer[44];
7084
7085         switch (type)
7086           {
7087           case 6:
7088             (void) host_aton(sub, binary);
7089             break;
7090
7091           case 4:       /* convert to IPv4-mapped IPv6 */
7092             binary[0] = binary[1] = 0;
7093             binary[2] = 0x0000ffff;
7094             (void) host_aton(sub, binary+3);
7095             break;
7096
7097           case 0:
7098             expand_string_message =
7099               string_sprintf("\"%s\" is not an IP address", sub);
7100             goto EXPAND_FAILED;
7101           }
7102
7103         yield = string_catn(yield, buffer, c == EOP_IPV6NORM
7104                     ? ipv6_nmtoa(binary, buffer)
7105                     : host_nmtoa(4, binary, -1, buffer, ':')
7106                   );
7107         continue;
7108         }
7109
7110       case EOP_ADDRESS:
7111       case EOP_LOCAL_PART:
7112       case EOP_DOMAIN:
7113         {
7114         uschar * error;
7115         int start, end, domain;
7116         uschar * t = parse_extract_address(sub, &error, &start, &end, &domain,
7117           FALSE);
7118         if (t)
7119           yield = c == EOP_DOMAIN
7120             ? string_cat(yield, t + domain)
7121             : c == EOP_LOCAL_PART && domain > 0
7122             ? string_catn(yield, t, domain - 1 )
7123             : string_cat(yield, t);
7124         continue;
7125         }
7126
7127       case EOP_ADDRESSES:
7128         {
7129         uschar outsep[2] = { ':', '\0' };
7130         uschar *address, *error;
7131         int save_ptr = yield->ptr;
7132         int start, end, domain;  /* Not really used */
7133
7134         while (isspace(*sub)) sub++;
7135         if (*sub == '>')
7136           if (*outsep = *++sub) ++sub;
7137           else
7138             {
7139             expand_string_message = string_sprintf("output separator "
7140               "missing in expanding ${addresses:%s}", --sub);
7141             goto EXPAND_FAILED;
7142             }
7143         f.parse_allow_group = TRUE;
7144
7145         for (;;)
7146           {
7147           uschar * p = parse_find_address_end(sub, FALSE);
7148           uschar saveend = *p;
7149           *p = '\0';
7150           address = parse_extract_address(sub, &error, &start, &end, &domain,
7151             FALSE);
7152           *p = saveend;
7153
7154           /* Add the address to the output list that we are building. This is
7155           done in chunks by searching for the separator character. At the
7156           start, unless we are dealing with the first address of the output
7157           list, add in a space if the new address begins with the separator
7158           character, or is an empty string. */
7159
7160           if (address)
7161             {
7162             if (yield->ptr != save_ptr && address[0] == *outsep)
7163               yield = string_catn(yield, US" ", 1);
7164
7165             for (;;)
7166               {
7167               size_t seglen = Ustrcspn(address, outsep);
7168               yield = string_catn(yield, address, seglen + 1);
7169
7170               /* If we got to the end of the string we output one character
7171               too many. */
7172
7173               if (address[seglen] == '\0') { yield->ptr--; break; }
7174               yield = string_catn(yield, outsep, 1);
7175               address += seglen + 1;
7176               }
7177
7178             /* Output a separator after the string: we will remove the
7179             redundant final one at the end. */
7180
7181             yield = string_catn(yield, outsep, 1);
7182             }
7183
7184           if (saveend == '\0') break;
7185           sub = p + 1;
7186           }
7187
7188         /* If we have generated anything, remove the redundant final
7189         separator. */
7190
7191         if (yield->ptr != save_ptr) yield->ptr--;
7192         f.parse_allow_group = FALSE;
7193         continue;
7194         }
7195
7196
7197       /* quote puts a string in quotes if it is empty or contains anything
7198       other than alphamerics, underscore, dot, or hyphen.
7199
7200       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
7201       be quoted in order to be a valid local part.
7202
7203       In both cases, newlines and carriage returns are converted into \n and \r
7204       respectively */
7205
7206       case EOP_QUOTE:
7207       case EOP_QUOTE_LOCAL_PART:
7208       if (arg == NULL)
7209         {
7210         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
7211         uschar *t = sub - 1;
7212
7213         if (c == EOP_QUOTE)
7214           {
7215           while (!needs_quote && *(++t) != 0)
7216             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
7217           }
7218         else  /* EOP_QUOTE_LOCAL_PART */
7219           {
7220           while (!needs_quote && *(++t) != 0)
7221             needs_quote = !isalnum(*t) &&
7222               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
7223               (*t != '.' || t == sub || t[1] == 0);
7224           }
7225
7226         if (needs_quote)
7227           {
7228           yield = string_catn(yield, US"\"", 1);
7229           t = sub - 1;
7230           while (*(++t) != 0)
7231             {
7232             if (*t == '\n')
7233               yield = string_catn(yield, US"\\n", 2);
7234             else if (*t == '\r')
7235               yield = string_catn(yield, US"\\r", 2);
7236             else
7237               {
7238               if (*t == '\\' || *t == '"')
7239                 yield = string_catn(yield, US"\\", 1);
7240               yield = string_catn(yield, t, 1);
7241               }
7242             }
7243           yield = string_catn(yield, US"\"", 1);
7244           }
7245         else yield = string_cat(yield, sub);
7246         continue;
7247         }
7248
7249       /* quote_lookuptype does lookup-specific quoting */
7250
7251       else
7252         {
7253         int n;
7254         uschar *opt = Ustrchr(arg, '_');
7255
7256         if (opt != NULL) *opt++ = 0;
7257
7258         n = search_findtype(arg, Ustrlen(arg));
7259         if (n < 0)
7260           {
7261           expand_string_message = search_error_message;
7262           goto EXPAND_FAILED;
7263           }
7264
7265         if (lookup_list[n]->quote != NULL)
7266           sub = (lookup_list[n]->quote)(sub, opt);
7267         else if (opt != NULL) sub = NULL;
7268
7269         if (sub == NULL)
7270           {
7271           expand_string_message = string_sprintf(
7272             "\"%s\" unrecognized after \"${quote_%s\"",
7273             opt, arg);
7274           goto EXPAND_FAILED;
7275           }
7276
7277         yield = string_cat(yield, sub);
7278         continue;
7279         }
7280
7281       /* rx quote sticks in \ before any non-alphameric character so that
7282       the insertion works in a regular expression. */
7283
7284       case EOP_RXQUOTE:
7285         {
7286         uschar *t = sub - 1;
7287         while (*(++t) != 0)
7288           {
7289           if (!isalnum(*t))
7290             yield = string_catn(yield, US"\\", 1);
7291           yield = string_catn(yield, t, 1);
7292           }
7293         continue;
7294         }
7295
7296       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
7297       prescribed by the RFC, if there are characters that need to be encoded */
7298
7299       case EOP_RFC2047:
7300         {
7301         uschar buffer[2048];
7302         yield = string_cat(yield,
7303                             parse_quote_2047(sub, Ustrlen(sub), headers_charset,
7304                               buffer, sizeof(buffer), FALSE));
7305         continue;
7306         }
7307
7308       /* RFC 2047 decode */
7309
7310       case EOP_RFC2047D:
7311         {
7312         int len;
7313         uschar *error;
7314         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
7315           headers_charset, '?', &len, &error);
7316         if (error != NULL)
7317           {
7318           expand_string_message = error;
7319           goto EXPAND_FAILED;
7320           }
7321         yield = string_catn(yield, decoded, len);
7322         continue;
7323         }
7324
7325       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
7326       underscores */
7327
7328       case EOP_FROM_UTF8:
7329         {
7330         while (*sub != 0)
7331           {
7332           int c;
7333           uschar buff[4];
7334           GETUTF8INC(c, sub);
7335           if (c > 255) c = '_';
7336           buff[0] = c;
7337           yield = string_catn(yield, buff, 1);
7338           }
7339         continue;
7340         }
7341
7342           /* replace illegal UTF-8 sequences by replacement character  */
7343
7344       #define UTF8_REPLACEMENT_CHAR US"?"
7345
7346       case EOP_UTF8CLEAN:
7347         {
7348         int seq_len = 0, index = 0;
7349         int bytes_left = 0;
7350         long codepoint = -1;
7351         int complete;
7352         uschar seq_buff[4];                     /* accumulate utf-8 here */
7353
7354         while (*sub != 0)
7355           {
7356           complete = 0;
7357           uschar c = *sub++;
7358
7359           if (bytes_left)
7360             {
7361             if ((c & 0xc0) != 0x80)
7362                     /* wrong continuation byte; invalidate all bytes */
7363               complete = 1; /* error */
7364             else
7365               {
7366               codepoint = (codepoint << 6) | (c & 0x3f);
7367               seq_buff[index++] = c;
7368               if (--bytes_left == 0)            /* codepoint complete */
7369                 if(codepoint > 0x10FFFF)        /* is it too large? */
7370                   complete = -1;        /* error (RFC3629 limit) */
7371                 else
7372                   {             /* finished; output utf-8 sequence */
7373                   yield = string_catn(yield, seq_buff, seq_len);
7374                   index = 0;
7375                   }
7376               }
7377             }
7378           else  /* no bytes left: new sequence */
7379             {
7380             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
7381               {
7382               yield = string_catn(yield, &c, 1);
7383               continue;
7384               }
7385             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
7386               {
7387               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
7388                 complete = -1;
7389               else
7390                 {
7391                   bytes_left = 1;
7392                   codepoint = c & 0x1f;
7393                 }
7394               }
7395             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
7396               {
7397               bytes_left = 2;
7398               codepoint = c & 0x0f;
7399               }
7400             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
7401               {
7402               bytes_left = 3;
7403               codepoint = c & 0x07;
7404               }
7405             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
7406               complete = -1;
7407
7408             seq_buff[index++] = c;
7409             seq_len = bytes_left + 1;
7410             }           /* if(bytes_left) */
7411
7412           if (complete != 0)
7413             {
7414             bytes_left = index = 0;
7415             yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1);
7416             }
7417           if ((complete == 1) && ((c & 0x80) == 0))
7418                         /* ASCII character follows incomplete sequence */
7419               yield = string_catn(yield, &c, 1);
7420           }
7421         /* If given a sequence truncated mid-character, we also want to report ?
7422         * Eg, ${length_1:フィル} is one byte, not one character, so we expect
7423         * ${utf8clean:${length_1:フィル}} to yield '?' */
7424         if (bytes_left != 0)
7425           {
7426           yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1);
7427           }
7428         continue;
7429         }
7430
7431 #ifdef SUPPORT_I18N
7432       case EOP_UTF8_DOMAIN_TO_ALABEL:
7433         {
7434         uschar * error = NULL;
7435         uschar * s = string_domain_utf8_to_alabel(sub, &error);
7436         if (error)
7437           {
7438           expand_string_message = string_sprintf(
7439             "error converting utf8 (%s) to alabel: %s",
7440             string_printing(sub), error);
7441           goto EXPAND_FAILED;
7442           }
7443         yield = string_cat(yield, s);
7444         continue;
7445         }
7446
7447       case EOP_UTF8_DOMAIN_FROM_ALABEL:
7448         {
7449         uschar * error = NULL;
7450         uschar * s = string_domain_alabel_to_utf8(sub, &error);
7451         if (error)
7452           {
7453           expand_string_message = string_sprintf(
7454             "error converting alabel (%s) to utf8: %s",
7455             string_printing(sub), error);
7456           goto EXPAND_FAILED;
7457           }
7458         yield = string_cat(yield, s);
7459         continue;
7460         }
7461
7462       case EOP_UTF8_LOCALPART_TO_ALABEL:
7463         {
7464         uschar * error = NULL;
7465         uschar * s = string_localpart_utf8_to_alabel(sub, &error);
7466         if (error)
7467           {
7468           expand_string_message = string_sprintf(
7469             "error converting utf8 (%s) to alabel: %s",
7470             string_printing(sub), error);
7471           goto EXPAND_FAILED;
7472           }
7473         yield = string_cat(yield, s);
7474         DEBUG(D_expand) debug_printf_indent("yield: '%s'\n", yield->s);
7475         continue;
7476         }
7477
7478       case EOP_UTF8_LOCALPART_FROM_ALABEL:
7479         {
7480         uschar * error = NULL;
7481         uschar * s = string_localpart_alabel_to_utf8(sub, &error);
7482         if (error)
7483           {
7484           expand_string_message = string_sprintf(
7485             "error converting alabel (%s) to utf8: %s",
7486             string_printing(sub), error);
7487           goto EXPAND_FAILED;
7488           }
7489         yield = string_cat(yield, s);
7490         continue;
7491         }
7492 #endif  /* EXPERIMENTAL_INTERNATIONAL */
7493
7494       /* escape turns all non-printing characters into escape sequences. */
7495
7496       case EOP_ESCAPE:
7497         {
7498         const uschar * t = string_printing(sub);
7499         yield = string_cat(yield, t);
7500         continue;
7501         }
7502
7503       case EOP_ESCAPE8BIT:
7504         {
7505         uschar c;
7506
7507         for (const uschar * s = sub; (c = *s); s++)
7508           yield = c < 127 && c != '\\'
7509             ? string_catn(yield, s, 1)
7510             : string_fmt_append(yield, "\\%03o", c);
7511         continue;
7512         }
7513
7514       /* Handle numeric expression evaluation */
7515
7516       case EOP_EVAL:
7517       case EOP_EVAL10:
7518         {
7519         uschar *save_sub = sub;
7520         uschar *error = NULL;
7521         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
7522         if (error)
7523           {
7524           expand_string_message = string_sprintf("error in expression "
7525             "evaluation: %s (after processing \"%.*s\")", error,
7526             (int)(sub-save_sub), save_sub);
7527           goto EXPAND_FAILED;
7528           }
7529         yield = string_fmt_append(yield, PR_EXIM_ARITH, n);
7530         continue;
7531         }
7532
7533       /* Handle time period formatting */
7534
7535       case EOP_TIME_EVAL:
7536         {
7537         int n = readconf_readtime(sub, 0, FALSE);
7538         if (n < 0)
7539           {
7540           expand_string_message = string_sprintf("string \"%s\" is not an "
7541             "Exim time interval in \"%s\" operator", sub, name);
7542           goto EXPAND_FAILED;
7543           }
7544         yield = string_fmt_append(yield, "%d", n);
7545         continue;
7546         }
7547
7548       case EOP_TIME_INTERVAL:
7549         {
7550         int n;
7551         uschar *t = read_number(&n, sub);
7552         if (*t != 0) /* Not A Number*/
7553           {
7554           expand_string_message = string_sprintf("string \"%s\" is not a "
7555             "positive number in \"%s\" operator", sub, name);
7556           goto EXPAND_FAILED;
7557           }
7558         t = readconf_printtime(n);
7559         yield = string_cat(yield, t);
7560         continue;
7561         }
7562
7563       /* Convert string to base64 encoding */
7564
7565       case EOP_STR2B64:
7566       case EOP_BASE64:
7567         {
7568 #ifndef DISABLE_TLS
7569         uschar * s = vp && *(void **)vp->value
7570           ? tls_cert_der_b64(*(void **)vp->value)
7571           : b64encode(CUS sub, Ustrlen(sub));
7572 #else
7573         uschar * s = b64encode(CUS sub, Ustrlen(sub));
7574 #endif
7575         yield = string_cat(yield, s);
7576         continue;
7577         }
7578
7579       case EOP_BASE64D:
7580         {
7581         uschar * s;
7582         int len = b64decode(sub, &s);
7583         if (len < 0)
7584           {
7585           expand_string_message = string_sprintf("string \"%s\" is not "
7586             "well-formed for \"%s\" operator", sub, name);
7587           goto EXPAND_FAILED;
7588           }
7589         yield = string_cat(yield, s);
7590         continue;
7591         }
7592
7593       /* strlen returns the length of the string */
7594
7595       case EOP_STRLEN:
7596         yield = string_fmt_append(yield, "%d", Ustrlen(sub));
7597         continue;
7598
7599       /* length_n or l_n takes just the first n characters or the whole string,
7600       whichever is the shorter;
7601
7602       substr_m_n, and s_m_n take n characters from offset m; negative m take
7603       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
7604       takes the rest, either to the right or to the left.
7605
7606       hash_n or h_n makes a hash of length n from the string, yielding n
7607       characters from the set a-z; hash_n_m makes a hash of length n, but
7608       uses m characters from the set a-zA-Z0-9.
7609
7610       nhash_n returns a single number between 0 and n-1 (in text form), while
7611       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
7612       between 0 and n-1 and the second between 0 and m-1. */
7613
7614       case EOP_LENGTH:
7615       case EOP_L:
7616       case EOP_SUBSTR:
7617       case EOP_S:
7618       case EOP_HASH:
7619       case EOP_H:
7620       case EOP_NHASH:
7621       case EOP_NH:
7622         {
7623         int sign = 1;
7624         int value1 = 0;
7625         int value2 = -1;
7626         int *pn;
7627         int len;
7628         uschar *ret;
7629
7630         if (!arg)
7631           {
7632           expand_string_message = string_sprintf("missing values after %s",
7633             name);
7634           goto EXPAND_FAILED;
7635           }
7636
7637         /* "length" has only one argument, effectively being synonymous with
7638         substr_0_n. */
7639
7640         if (c == EOP_LENGTH || c == EOP_L)
7641           {
7642           pn = &value2;
7643           value2 = 0;
7644           }
7645
7646         /* The others have one or two arguments; for "substr" the first may be
7647         negative. The second being negative means "not supplied". */
7648
7649         else
7650           {
7651           pn = &value1;
7652           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
7653           }
7654
7655         /* Read up to two numbers, separated by underscores */
7656
7657         ret = arg;
7658         while (*arg != 0)
7659           {
7660           if (arg != ret && *arg == '_' && pn == &value1)
7661             {
7662             pn = &value2;
7663             value2 = 0;
7664             if (arg[1] != 0) arg++;
7665             }
7666           else if (!isdigit(*arg))
7667             {
7668             expand_string_message =
7669               string_sprintf("non-digit after underscore in \"%s\"", name);
7670             goto EXPAND_FAILED;
7671             }
7672           else *pn = (*pn)*10 + *arg++ - '0';
7673           }
7674         value1 *= sign;
7675
7676         /* Perform the required operation */
7677
7678         ret =
7679           (c == EOP_HASH || c == EOP_H)?
7680              compute_hash(sub, value1, value2, &len) :
7681           (c == EOP_NHASH || c == EOP_NH)?
7682              compute_nhash(sub, value1, value2, &len) :
7683              extract_substr(sub, value1, value2, &len);
7684
7685         if (ret == NULL) goto EXPAND_FAILED;
7686         yield = string_catn(yield, ret, len);
7687         continue;
7688         }
7689
7690       /* Stat a path */
7691
7692       case EOP_STAT:
7693         {
7694         uschar smode[12];
7695         uschar **modetable[3];
7696         mode_t mode;
7697         struct stat st;
7698
7699         if (expand_forbid & RDO_EXISTS)
7700           {
7701           expand_string_message = US"Use of the stat() expansion is not permitted";
7702           goto EXPAND_FAILED;
7703           }
7704
7705         if (stat(CS sub, &st) < 0)
7706           {
7707           expand_string_message = string_sprintf("stat(%s) failed: %s",
7708             sub, strerror(errno));
7709           goto EXPAND_FAILED;
7710           }
7711         mode = st.st_mode;
7712         switch (mode & S_IFMT)
7713           {
7714           case S_IFIFO: smode[0] = 'p'; break;
7715           case S_IFCHR: smode[0] = 'c'; break;
7716           case S_IFDIR: smode[0] = 'd'; break;
7717           case S_IFBLK: smode[0] = 'b'; break;
7718           case S_IFREG: smode[0] = '-'; break;
7719           default: smode[0] = '?'; break;
7720           }
7721
7722         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
7723         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
7724         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
7725
7726         for (int i = 0; i < 3; i++)
7727           {
7728           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
7729           mode >>= 3;
7730           }
7731
7732         smode[10] = 0;
7733         yield = string_fmt_append(yield,
7734           "mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
7735           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
7736           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
7737           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
7738           (long)st.st_gid, st.st_size, (long)st.st_atime,
7739           (long)st.st_mtime, (long)st.st_ctime);
7740         continue;
7741         }
7742
7743       /* vaguely random number less than N */
7744
7745       case EOP_RANDINT:
7746         {
7747         int_eximarith_t max = expanded_string_integer(sub, TRUE);
7748
7749         if (expand_string_message)
7750           goto EXPAND_FAILED;
7751         yield = string_fmt_append(yield, "%d", vaguely_random_number((int)max));
7752         continue;
7753         }
7754
7755       /* Reverse IP, including IPv6 to dotted-nibble */
7756
7757       case EOP_REVERSE_IP:
7758         {
7759         int family, maskptr;
7760         uschar reversed[128];
7761
7762         family = string_is_ip_address(sub, &maskptr);
7763         if (family == 0)
7764           {
7765           expand_string_message = string_sprintf(
7766               "reverse_ip() not given an IP address [%s]", sub);
7767           goto EXPAND_FAILED;
7768           }
7769         invert_address(reversed, sub);
7770         yield = string_cat(yield, reversed);
7771         continue;
7772         }
7773
7774       /* Unknown operator */
7775
7776       default:
7777         expand_string_message =
7778           string_sprintf("unknown expansion operator \"%s\"", name);
7779         goto EXPAND_FAILED;
7780       }
7781     }
7782
7783   /* Handle a plain name. If this is the first thing in the expansion, release
7784   the pre-allocated buffer. If the result data is known to be in a new buffer,
7785   newsize will be set to the size of that buffer, and we can just point at that
7786   store instead of copying. Many expansion strings contain just one reference,
7787   so this is a useful optimization, especially for humungous headers
7788   ($message_headers). */
7789                                                 /*{*/
7790   if (*s++ == '}')
7791     {
7792     int len;
7793     int newsize = 0;
7794     gstring * g = NULL;
7795
7796     if (!yield)
7797       g = store_get(sizeof(gstring));
7798     else if (yield->ptr == 0)
7799       {
7800       if (resetok) store_reset(yield);
7801       yield = NULL;
7802       g = store_get(sizeof(gstring));   /* alloc _before_ calling find_variable() */
7803       }
7804     if (!(value = find_variable(name, FALSE, skipping, &newsize)))
7805       {
7806       expand_string_message =
7807         string_sprintf("unknown variable in \"${%s}\"", name);
7808       check_variable_error_message(name);
7809       goto EXPAND_FAILED;
7810       }
7811     len = Ustrlen(value);
7812     if (!yield && newsize)
7813       {
7814       yield = g;
7815       yield->size = newsize;
7816       yield->ptr = len;
7817       yield->s = value;
7818       }
7819     else
7820       yield = string_catn(yield, value, len);
7821     continue;
7822     }
7823
7824   /* Else there's something wrong */
7825
7826   expand_string_message =
7827     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
7828     "in a variable reference)", name);
7829   goto EXPAND_FAILED;
7830   }
7831
7832 /* If we hit the end of the string when ket_ends is set, there is a missing
7833 terminating brace. */
7834
7835 if (ket_ends && *s == 0)
7836   {
7837   expand_string_message = malformed_header
7838     ? US"missing } at end of string - could be header name not terminated by colon"
7839     : US"missing } at end of string";
7840   goto EXPAND_FAILED;
7841   }
7842
7843 /* Expansion succeeded; yield may still be NULL here if nothing was actually
7844 added to the string. If so, set up an empty string. Add a terminating zero. If
7845 left != NULL, return a pointer to the terminator. */
7846
7847 if (!yield)
7848   yield = string_get(1);
7849 (void) string_from_gstring(yield);
7850 if (left) *left = s;
7851
7852 /* Any stacking store that was used above the final string is no longer needed.
7853 In many cases the final string will be the first one that was got and so there
7854 will be optimal store usage. */
7855
7856 if (resetok) store_reset(yield->s + (yield->size = yield->ptr + 1));
7857 else if (resetok_p) *resetok_p = FALSE;
7858
7859 DEBUG(D_expand)
7860   DEBUG(D_noutf8)
7861     {
7862     debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string);
7863     debug_printf_indent("%sresult: %s\n",
7864       skipping ? "|-----" : "\\_____", yield->s);
7865     if (skipping)
7866       debug_printf_indent("\\___skipping: result is not used\n");
7867     }
7868   else
7869     {
7870     debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ
7871       "expanding: %.*s\n",
7872       (int)(s - string), string);
7873     debug_printf_indent("%s"
7874       UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7875       "result: %s\n",
7876       skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7877       yield->s);
7878     if (skipping)
7879       debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7880         "skipping: result is not used\n");
7881     }
7882 expand_level--;
7883 return yield->s;
7884
7885 /* This is the failure exit: easiest to program with a goto. We still need
7886 to update the pointer to the terminator, for cases of nested calls with "fail".
7887 */
7888
7889 EXPAND_FAILED_CURLY:
7890 if (malformed_header)
7891   expand_string_message =
7892     US"missing or misplaced { or } - could be header name not terminated by colon";
7893
7894 else if (!expand_string_message || !*expand_string_message)
7895   expand_string_message = US"missing or misplaced { or }";
7896
7897 /* At one point, Exim reset the store to yield (if yield was not NULL), but
7898 that is a bad idea, because expand_string_message is in dynamic store. */
7899
7900 EXPAND_FAILED:
7901 if (left) *left = s;
7902 DEBUG(D_expand)
7903   DEBUG(D_noutf8)
7904     {
7905     debug_printf_indent("|failed to expand: %s\n", string);
7906     debug_printf_indent("%serror message: %s\n",
7907       f.expand_string_forcedfail ? "|---" : "\\___", expand_string_message);
7908     if (f.expand_string_forcedfail)
7909       debug_printf_indent("\\failure was forced\n");
7910     }
7911   else
7912     {
7913     debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n",
7914       string);
7915     debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ
7916       "error message: %s\n",
7917       f.expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT,
7918       expand_string_message);
7919     if (f.expand_string_forcedfail)
7920       debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n");
7921     }
7922 if (resetok_p && !resetok) *resetok_p = FALSE;
7923 expand_level--;
7924 return NULL;
7925 }
7926
7927
7928 /* This is the external function call. Do a quick check for any expansion
7929 metacharacters, and if there are none, just return the input string.
7930
7931 Argument: the string to be expanded
7932 Returns:  the expanded string, or NULL if expansion failed; if failure was
7933           due to a lookup deferring, search_find_defer will be TRUE
7934 */
7935
7936 const uschar *
7937 expand_cstring(const uschar * string)
7938 {
7939 if (Ustrpbrk(string, "$\\") != NULL)
7940   {
7941   int old_pool = store_pool;
7942   uschar * s;
7943
7944   f.search_find_defer = FALSE;
7945   malformed_header = FALSE;
7946   store_pool = POOL_MAIN;
7947     s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
7948   store_pool = old_pool;
7949   return s;
7950   }
7951 return string;
7952 }
7953
7954
7955 uschar *
7956 expand_string(uschar * string)
7957 {
7958 return US expand_cstring(CUS string);
7959 }
7960
7961
7962
7963
7964
7965 /*************************************************
7966 *              Expand and copy                   *
7967 *************************************************/
7968
7969 /* Now and again we want to expand a string and be sure that the result is in a
7970 new bit of store. This function does that.
7971 Since we know it has been copied, the de-const cast is safe.
7972
7973 Argument: the string to be expanded
7974 Returns:  the expanded string, always in a new bit of store, or NULL
7975 */
7976
7977 uschar *
7978 expand_string_copy(const uschar *string)
7979 {
7980 const uschar *yield = expand_cstring(string);
7981 if (yield == string) yield = string_copy(string);
7982 return US yield;
7983 }
7984
7985
7986
7987 /*************************************************
7988 *        Expand and interpret as an integer      *
7989 *************************************************/
7990
7991 /* Expand a string, and convert the result into an integer.
7992
7993 Arguments:
7994   string  the string to be expanded
7995   isplus  TRUE if a non-negative number is expected
7996
7997 Returns:  the integer value, or
7998           -1 for an expansion error               ) in both cases, message in
7999           -2 for an integer interpretation error  ) expand_string_message
8000           expand_string_message is set NULL for an OK integer
8001 */
8002
8003 int_eximarith_t
8004 expand_string_integer(uschar *string, BOOL isplus)
8005 {
8006 return expanded_string_integer(expand_string(string), isplus);
8007 }
8008
8009
8010 /*************************************************
8011  *         Interpret string as an integer        *
8012  *************************************************/
8013
8014 /* Convert a string (that has already been expanded) into an integer.
8015
8016 This function is used inside the expansion code.
8017
8018 Arguments:
8019   s       the string to be expanded
8020   isplus  TRUE if a non-negative number is expected
8021
8022 Returns:  the integer value, or
8023           -1 if string is NULL (which implies an expansion error)
8024           -2 for an integer interpretation error
8025           expand_string_message is set NULL for an OK integer
8026 */
8027
8028 static int_eximarith_t
8029 expanded_string_integer(const uschar *s, BOOL isplus)
8030 {
8031 int_eximarith_t value;
8032 uschar *msg = US"invalid integer \"%s\"";
8033 uschar *endptr;
8034
8035 /* If expansion failed, expand_string_message will be set. */
8036
8037 if (s == NULL) return -1;
8038
8039 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
8040 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
8041 systems, so we set it zero ourselves. */
8042
8043 errno = 0;
8044 expand_string_message = NULL;               /* Indicates no error */
8045
8046 /* Before Exim 4.64, strings consisting entirely of whitespace compared
8047 equal to 0.  Unfortunately, people actually relied upon that, so preserve
8048 the behaviour explicitly.  Stripping leading whitespace is a harmless
8049 noop change since strtol skips it anyway (provided that there is a number
8050 to find at all). */
8051 if (isspace(*s))
8052   {
8053   while (isspace(*s)) ++s;
8054   if (*s == '\0')
8055     {
8056       DEBUG(D_expand)
8057        debug_printf_indent("treating blank string as number 0\n");
8058       return 0;
8059     }
8060   }
8061
8062 value = strtoll(CS s, CSS &endptr, 10);
8063
8064 if (endptr == s)
8065   {
8066   msg = US"integer expected but \"%s\" found";
8067   }
8068 else if (value < 0 && isplus)
8069   {
8070   msg = US"non-negative integer expected but \"%s\" found";
8071   }
8072 else
8073   {
8074   switch (tolower(*endptr))
8075     {
8076     default:
8077       break;
8078     case 'k':
8079       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
8080       else value *= 1024;
8081       endptr++;
8082       break;
8083     case 'm':
8084       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
8085       else value *= 1024*1024;
8086       endptr++;
8087       break;
8088     case 'g':
8089       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
8090       else value *= 1024*1024*1024;
8091       endptr++;
8092       break;
8093     }
8094   if (errno == ERANGE)
8095     msg = US"absolute value of integer \"%s\" is too large (overflow)";
8096   else
8097     {
8098     while (isspace(*endptr)) endptr++;
8099     if (*endptr == 0) return value;
8100     }
8101   }
8102
8103 expand_string_message = string_sprintf(CS msg, s);
8104 return -2;
8105 }
8106
8107
8108 /* These values are usually fixed boolean values, but they are permitted to be
8109 expanded strings.
8110
8111 Arguments:
8112   addr       address being routed
8113   mtype      the module type
8114   mname      the module name
8115   dbg_opt    debug selectors
8116   oname      the option name
8117   bvalue     the router's boolean value
8118   svalue     the router's string value
8119   rvalue     where to put the returned value
8120
8121 Returns:     OK     value placed in rvalue
8122              DEFER  expansion failed
8123 */
8124
8125 int
8126 exp_bool(address_item *addr,
8127   uschar *mtype, uschar *mname, unsigned dbg_opt,
8128   uschar *oname, BOOL bvalue,
8129   uschar *svalue, BOOL *rvalue)
8130 {
8131 uschar *expanded;
8132 if (svalue == NULL) { *rvalue = bvalue; return OK; }
8133
8134 expanded = expand_string(svalue);
8135 if (expanded == NULL)
8136   {
8137   if (f.expand_string_forcedfail)
8138     {
8139     DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname);
8140     *rvalue = bvalue;
8141     return OK;
8142     }
8143   addr->message = string_sprintf("failed to expand \"%s\" in %s %s: %s",
8144       oname, mname, mtype, expand_string_message);
8145   DEBUG(dbg_opt) debug_printf("%s\n", addr->message);
8146   return DEFER;
8147   }
8148
8149 DEBUG(dbg_opt) debug_printf("expansion of \"%s\" yields \"%s\"\n", oname,
8150   expanded);
8151
8152 if (strcmpic(expanded, US"true") == 0 || strcmpic(expanded, US"yes") == 0)
8153   *rvalue = TRUE;
8154 else if (strcmpic(expanded, US"false") == 0 || strcmpic(expanded, US"no") == 0)
8155   *rvalue = FALSE;
8156 else
8157   {
8158   addr->message = string_sprintf("\"%s\" is not a valid value for the "
8159     "\"%s\" option in the %s %s", expanded, oname, mname, mtype);
8160   return DEFER;
8161   }
8162
8163 return OK;
8164 }
8165
8166
8167
8168 /* Avoid potentially exposing a password in a string about to be logged */
8169
8170 uschar *
8171 expand_hide_passwords(uschar * s)
8172 {
8173 return (  (  Ustrstr(s, "failed to expand") != NULL
8174           || Ustrstr(s, "expansion of ")    != NULL
8175           )
8176        && (  Ustrstr(s, "mysql")   != NULL
8177           || Ustrstr(s, "pgsql")   != NULL
8178           || Ustrstr(s, "redis")   != NULL
8179           || Ustrstr(s, "sqlite")  != NULL
8180           || Ustrstr(s, "ldap:")   != NULL
8181           || Ustrstr(s, "ldaps:")  != NULL
8182           || Ustrstr(s, "ldapi:")  != NULL
8183           || Ustrstr(s, "ldapdn:") != NULL
8184           || Ustrstr(s, "ldapm:")  != NULL
8185        )  )
8186   ? US"Temporary internal error" : s;
8187 }
8188
8189
8190 /* Read given named file into big_buffer.  Use for keying material etc.
8191 The content will have an ascii NUL appended.
8192
8193 Arguments:
8194  filename       as it says
8195
8196 Return:  pointer to buffer, or NULL on error.
8197 */
8198
8199 uschar *
8200 expand_file_big_buffer(const uschar * filename)
8201 {
8202 int fd, off = 0, len;
8203
8204 if ((fd = open(CS filename, O_RDONLY)) < 0)
8205   {
8206   log_write(0, LOG_MAIN | LOG_PANIC, "unable to open file for reading: %s",
8207              filename);
8208   return NULL;
8209   }
8210
8211 do
8212   {
8213   if ((len = read(fd, big_buffer + off, big_buffer_size - 2 - off)) < 0)
8214     {
8215     (void) close(fd);
8216     log_write(0, LOG_MAIN|LOG_PANIC, "unable to read file: %s", filename);
8217     return NULL;
8218     }
8219   off += len;
8220   }
8221 while (len > 0);
8222
8223 (void) close(fd);
8224 big_buffer[off] = '\0';
8225 return big_buffer;
8226 }
8227
8228
8229
8230 /*************************************************
8231 * Error-checking for testsuite                   *
8232 *************************************************/
8233 typedef struct {
8234   uschar *      region_start;
8235   uschar *      region_end;
8236   const uschar *var_name;
8237   const uschar *var_data;
8238 } err_ctx;
8239
8240 static void
8241 assert_variable_notin(uschar * var_name, uschar * var_data, void * ctx)
8242 {
8243 err_ctx * e = ctx;
8244 if (var_data >= e->region_start  &&  var_data < e->region_end)
8245   {
8246   e->var_name = CUS var_name;
8247   e->var_data = CUS var_data;
8248   }
8249 }
8250
8251 void
8252 assert_no_variables(void * ptr, int len, const char * filename, int linenumber)
8253 {
8254 err_ctx e = { .region_start = ptr, .region_end = US ptr + len,
8255               .var_name = NULL, .var_data = NULL };
8256
8257 /* check acl_ variables */
8258 tree_walk(acl_var_c, assert_variable_notin, &e);
8259 tree_walk(acl_var_m, assert_variable_notin, &e);
8260
8261 /* check auth<n> variables */
8262 for (int i = 0; i < AUTH_VARS; i++) if (auth_vars[i])
8263   assert_variable_notin(US"auth<n>", auth_vars[i], &e);
8264
8265 /* check regex<n> variables */
8266 for (int i = 0; i < REGEX_VARS; i++) if (regex_vars[i])
8267   assert_variable_notin(US"regex<n>", regex_vars[i], &e);
8268
8269 /* check known-name variables */
8270 for (var_entry * v = var_table; v < var_table + var_table_size; v++)
8271   if (v->type == vtype_stringptr)
8272     assert_variable_notin(US v->name, *(USS v->value), &e);
8273
8274 /* check dns and address trees */
8275 tree_walk(tree_dns_fails,     assert_variable_notin, &e);
8276 tree_walk(tree_duplicates,    assert_variable_notin, &e);
8277 tree_walk(tree_nonrecipients, assert_variable_notin, &e);
8278 tree_walk(tree_unusable,      assert_variable_notin, &e);
8279
8280 if (e.var_name)
8281   log_write(0, LOG_MAIN|LOG_PANIC_DIE,
8282     "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'",
8283     e.var_name, filename, linenumber, e.var_data);
8284 }
8285
8286
8287
8288 /*************************************************
8289 **************************************************
8290 *             Stand-alone test program           *
8291 **************************************************
8292 *************************************************/
8293
8294 #ifdef STAND_ALONE
8295
8296
8297 BOOL
8298 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
8299 {
8300 int ovector[3*(EXPAND_MAXN+1)];
8301 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
8302   ovector, nelem(ovector));
8303 BOOL yield = n >= 0;
8304 if (n == 0) n = EXPAND_MAXN + 1;
8305 if (yield)
8306   {
8307   expand_nmax = setup < 0 ? 0 : setup + 1;
8308   for (int nn = setup < 0 ? 0 : 2; nn < n*2; nn += 2)
8309     {
8310     expand_nstring[expand_nmax] = subject + ovector[nn];
8311     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
8312     }
8313   expand_nmax--;
8314   }
8315 return yield;
8316 }
8317
8318
8319 int main(int argc, uschar **argv)
8320 {
8321 uschar buffer[1024];
8322
8323 debug_selector = D_v;
8324 debug_file = stderr;
8325 debug_fd = fileno(debug_file);
8326 big_buffer = malloc(big_buffer_size);
8327
8328 for (int i = 1; i < argc; i++)
8329   {
8330   if (argv[i][0] == '+')
8331     {
8332     debug_trace_memory = 2;
8333     argv[i]++;
8334     }
8335   if (isdigit(argv[i][0]))
8336     debug_selector = Ustrtol(argv[i], NULL, 0);
8337   else
8338     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
8339         Ustrlen(argv[i]))
8340       {
8341 #ifdef LOOKUP_LDAP
8342       eldap_default_servers = argv[i];
8343 #endif
8344 #ifdef LOOKUP_MYSQL
8345       mysql_servers = argv[i];
8346 #endif
8347 #ifdef LOOKUP_PGSQL
8348       pgsql_servers = argv[i];
8349 #endif
8350 #ifdef LOOKUP_REDIS
8351       redis_servers = argv[i];
8352 #endif
8353       }
8354 #ifdef EXIM_PERL
8355   else opt_perl_startup = argv[i];
8356 #endif
8357   }
8358
8359 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
8360
8361 expand_nstring[1] = US"string 1....";
8362 expand_nlength[1] = 8;
8363 expand_nmax = 1;
8364
8365 #ifdef EXIM_PERL
8366 if (opt_perl_startup != NULL)
8367   {
8368   uschar *errstr;
8369   printf("Starting Perl interpreter\n");
8370   errstr = init_perl(opt_perl_startup);
8371   if (errstr != NULL)
8372     {
8373     printf("** error in perl_startup code: %s\n", errstr);
8374     return EXIT_FAILURE;
8375     }
8376   }
8377 #endif /* EXIM_PERL */
8378
8379 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
8380   {
8381   void *reset_point = store_get(0);
8382   uschar *yield = expand_string(buffer);
8383   if (yield != NULL)
8384     {
8385     printf("%s\n", yield);
8386     store_reset(reset_point);
8387     }
8388   else
8389     {
8390     if (f.search_find_defer) printf("search_find deferred\n");
8391     printf("Failed: %s\n", expand_string_message);
8392     if (f.expand_string_forcedfail) printf("Forced failure\n");
8393     printf("\n");
8394     }
8395   }
8396
8397 search_tidyup();
8398
8399 return 0;
8400 }
8401
8402 #endif
8403
8404 /* vi: aw ai sw=2
8405 */
8406 /* End of expand.c */