src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2014 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(uschar *, BOOL, uschar **, BOOL, BOOL, BOOL *);
  17
  18 #ifdef STAND_ALONE
  19 #ifndef SUPPORT_CRYPTEQ
  20 #define SUPPORT_CRYPTEQ
  21 #endif
  22 #endif
  23
  24 #ifdef LOOKUP_LDAP
  25 #include "lookups/ldap.h"
  26 #endif
  27
  28 #ifdef SUPPORT_CRYPTEQ
  29 #ifdef CRYPT_H
  30 #include <crypt.h>
  31 #endif
  32 #ifndef HAVE_CRYPT16
  33 extern char* crypt16(char*, char*);
  34 #endif
  35 #endif
  36
  37 /* The handling of crypt16() is a mess. I will record below the analysis of the
  38 mess that was sent to me. We decided, however, to make changing this very low
  39 priority, because in practice people are moving away from the crypt()
  40 algorithms nowadays, so it doesn't seem worth it.
  41
  42 <quote>
  43 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  44 the first 8 characters of the password using a 20-round version of crypt
  45 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  46 or an empty block if the password is less than 9 characters, using a
  47 20-round version of crypt and the same salt as was used for the first
  48 block.  Charaters after the first 16 are ignored.  It always generates
  49 a 16-byte hash, which is expressed together with the salt as a string
  50 of 24 base 64 digits.  Here are some links to peruse:
  51
  52         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  53         http://seclists.org/bugtraq/1999/Mar/0076.html
  54
  55 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  56 and OSF/1.  This is the same as the standard crypt if given a password
  57 of 8 characters or less.  If given more, it first does the same as crypt
  58 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  59 using as salt the first two base 64 digits from the first hash block.
  60 If the password is more than 16 characters then it crypts the 17th to 24th
  61 characters using as salt the first two base 64 digits from the second hash
  62 block.  And so on: I've seen references to it cutting off the password at
  63 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  64
  65         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  66         http://seclists.org/bugtraq/1999/Mar/0109.html
  67         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  68              TET1_html/sec.c222.html#no_id_208
  69
  70 Exim has something it calls "crypt16".  It will either use a native
  71 crypt16 or its own implementation.  A native crypt16 will presumably
  72 be the one that I called "crypt16" above.  The internal "crypt16"
  73 function, however, is a two-block-maximum implementation of what I called
  74 "bigcrypt".  The documentation matches the internal code.
  75
  76 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  77 that crypt16 and bigcrypt were different things.
  78
  79 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  80 to whatever it is using under that name.  This unfortunately sets a
  81 precedent for using "{crypt16}" to identify two incompatible algorithms
  82 whose output can't be distinguished.  With "{crypt16}" thus rendered
  83 ambiguous, I suggest you deprecate it and invent two new identifiers
  84 for the two algorithms.
  85
  86 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  87 of the password separately means they can be cracked separately, so
  88 the double-length hash only doubles the cracking effort instead of
  89 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  90 bcrypt ({CRYPT}$2a$).
  91 </quote>
  92 */
  93
  94
  95
  96 #ifndef nelements
  97 # define nelements(arr) (sizeof(arr) / sizeof(*arr))
  98 #endif
  99
 100 /*************************************************
 101 *            Local statics and tables            *
 102 *************************************************/
 103
 104 /* Table of item names, and corresponding switch numbers. The names must be in
 105 alphabetical order. */
 106
 107 static uschar *item_table[] = {
 108   US"acl",
 109   US"certextract",
 110   US"dlfunc",
 111   US"extract",
 112   US"filter",
 113   US"hash",
 114   US"hmac",
 115   US"if",
 116   US"length",
 117   US"listextract",
 118   US"lookup",
 119   US"map",
 120   US"nhash",
 121   US"perl",
 122   US"prvs",
 123   US"prvscheck",
 124   US"readfile",
 125   US"readsocket",
 126   US"reduce",
 127   US"run",
 128   US"sg",
 129   US"substr",
 130   US"tr" };
 131
 132 enum {
 133   EITEM_ACL,
 134   EITEM_CERTEXTRACT,
 135   EITEM_DLFUNC,
 136   EITEM_EXTRACT,
 137   EITEM_FILTER,
 138   EITEM_HASH,
 139   EITEM_HMAC,
 140   EITEM_IF,
 141   EITEM_LENGTH,
 142   EITEM_LISTEXTRACT,
 143   EITEM_LOOKUP,
 144   EITEM_MAP,
 145   EITEM_NHASH,
 146   EITEM_PERL,
 147   EITEM_PRVS,
 148   EITEM_PRVSCHECK,
 149   EITEM_READFILE,
 150   EITEM_READSOCK,
 151   EITEM_REDUCE,
 152   EITEM_RUN,
 153   EITEM_SG,
 154   EITEM_SUBSTR,
 155   EITEM_TR };
 156
 157 /* Tables of operator names, and corresponding switch numbers. The names must be
 158 in alphabetical order. There are two tables, because underscore is used in some
 159 cases to introduce arguments, whereas for other it is part of the name. This is
 160 an historical mis-design. */
 161
 162 static uschar *op_table_underscore[] = {
 163   US"from_utf8",
 164   US"local_part",
 165   US"quote_local_part",
 166   US"reverse_ip",
 167   US"time_eval",
 168   US"time_interval"};
 169
 170 enum {
 171   EOP_FROM_UTF8,
 172   EOP_LOCAL_PART,
 173   EOP_QUOTE_LOCAL_PART,
 174   EOP_REVERSE_IP,
 175   EOP_TIME_EVAL,
 176   EOP_TIME_INTERVAL };
 177
 178 static uschar *op_table_main[] = {
 179   US"address",
 180   US"addresses",
 181   US"base62",
 182   US"base62d",
 183   US"domain",
 184   US"escape",
 185   US"eval",
 186   US"eval10",
 187   US"expand",
 188   US"h",
 189   US"hash",
 190   US"hex2b64",
 191   US"hexquote",
 192   US"l",
 193   US"lc",
 194   US"length",
 195   US"listcount",
 196   US"listnamed",
 197   US"mask",
 198   US"md5",
 199   US"nh",
 200   US"nhash",
 201   US"quote",
 202   US"randint",
 203   US"rfc2047",
 204   US"rfc2047d",
 205   US"rxquote",
 206   US"s",
 207   US"sha1",
 208   US"stat",
 209   US"str2b64",
 210   US"strlen",
 211   US"substr",
 212   US"uc",
 213   US"utf8clean" };
 214
 215 enum {
 216   EOP_ADDRESS =  sizeof(op_table_underscore)/sizeof(uschar *),
 217   EOP_ADDRESSES,
 218   EOP_BASE62,
 219   EOP_BASE62D,
 220   EOP_DOMAIN,
 221   EOP_ESCAPE,
 222   EOP_EVAL,
 223   EOP_EVAL10,
 224   EOP_EXPAND,
 225   EOP_H,
 226   EOP_HASH,
 227   EOP_HEX2B64,
 228   EOP_HEXQUOTE,
 229   EOP_L,
 230   EOP_LC,
 231   EOP_LENGTH,
 232   EOP_LISTCOUNT,
 233   EOP_LISTNAMED,
 234   EOP_MASK,
 235   EOP_MD5,
 236   EOP_NH,
 237   EOP_NHASH,
 238   EOP_QUOTE,
 239   EOP_RANDINT,
 240   EOP_RFC2047,
 241   EOP_RFC2047D,
 242   EOP_RXQUOTE,
 243   EOP_S,
 244   EOP_SHA1,
 245   EOP_STAT,
 246   EOP_STR2B64,
 247   EOP_STRLEN,
 248   EOP_SUBSTR,
 249   EOP_UC,
 250   EOP_UTF8CLEAN };
 251
 252
 253 /* Table of condition names, and corresponding switch numbers. The names must
 254 be in alphabetical order. */
 255
 256 static uschar *cond_table[] = {
 257   US"<",
 258   US"<=",
 259   US"=",
 260   US"==",     /* Backward compatibility */
 261   US">",
 262   US">=",
 263   US"acl",
 264   US"and",
 265   US"bool",
 266   US"bool_lax",
 267   US"crypteq",
 268   US"def",
 269   US"eq",
 270   US"eqi",
 271   US"exists",
 272   US"first_delivery",
 273   US"forall",
 274   US"forany",
 275   US"ge",
 276   US"gei",
 277   US"gt",
 278   US"gti",
 279   US"inlist",
 280   US"inlisti",
 281   US"isip",
 282   US"isip4",
 283   US"isip6",
 284   US"ldapauth",
 285   US"le",
 286   US"lei",
 287   US"lt",
 288   US"lti",
 289   US"match",
 290   US"match_address",
 291   US"match_domain",
 292   US"match_ip",
 293   US"match_local_part",
 294   US"or",
 295   US"pam",
 296   US"pwcheck",
 297   US"queue_running",
 298   US"radius",
 299   US"saslauthd"
 300 };
 301
 302 enum {
 303   ECOND_NUM_L,
 304   ECOND_NUM_LE,
 305   ECOND_NUM_E,
 306   ECOND_NUM_EE,
 307   ECOND_NUM_G,
 308   ECOND_NUM_GE,
 309   ECOND_ACL,
 310   ECOND_AND,
 311   ECOND_BOOL,
 312   ECOND_BOOL_LAX,
 313   ECOND_CRYPTEQ,
 314   ECOND_DEF,
 315   ECOND_STR_EQ,
 316   ECOND_STR_EQI,
 317   ECOND_EXISTS,
 318   ECOND_FIRST_DELIVERY,
 319   ECOND_FORALL,
 320   ECOND_FORANY,
 321   ECOND_STR_GE,
 322   ECOND_STR_GEI,
 323   ECOND_STR_GT,
 324   ECOND_STR_GTI,
 325   ECOND_INLIST,
 326   ECOND_INLISTI,
 327   ECOND_ISIP,
 328   ECOND_ISIP4,
 329   ECOND_ISIP6,
 330   ECOND_LDAPAUTH,
 331   ECOND_STR_LE,
 332   ECOND_STR_LEI,
 333   ECOND_STR_LT,
 334   ECOND_STR_LTI,
 335   ECOND_MATCH,
 336   ECOND_MATCH_ADDRESS,
 337   ECOND_MATCH_DOMAIN,
 338   ECOND_MATCH_IP,
 339   ECOND_MATCH_LOCAL_PART,
 340   ECOND_OR,
 341   ECOND_PAM,
 342   ECOND_PWCHECK,
 343   ECOND_QUEUE_RUNNING,
 344   ECOND_RADIUS,
 345   ECOND_SASLAUTHD
 346 };
 347
 348
 349 /* Type for main variable table */
 350
 351 typedef struct {
 352   const char *name;
 353   int         type;
 354   void       *value;
 355 } var_entry;
 356
 357 /* Type for entries pointing to address/length pairs. Not currently
 358 in use. */
 359
 360 typedef struct {
 361   uschar **address;
 362   int  *length;
 363 } alblock;
 364
 365 /* Types of table entry */
 366
 367 enum {
 368   vtype_int,            /* value is address of int */
 369   vtype_filter_int,     /* ditto, but recognized only when filtering */
 370   vtype_ino,            /* value is address of ino_t (not always an int) */
 371   vtype_uid,            /* value is address of uid_t (not always an int) */
 372   vtype_gid,            /* value is address of gid_t (not always an int) */
 373   vtype_bool,           /* value is address of bool */
 374   vtype_stringptr,      /* value is address of pointer to string */
 375   vtype_msgbody,        /* as stringptr, but read when first required */
 376   vtype_msgbody_end,    /* ditto, the end of the message */
 377   vtype_msgheaders,     /* the message's headers, processed */
 378   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 379   vtype_localpart,      /* extract local part from string */
 380   vtype_domain,         /* extract domain from string */
 381   vtype_string_func,    /* value is string returned by given function */
 382   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 383   vtype_tode,           /* value not used; generate tod in epoch format */
 384   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 385   vtype_todf,           /* value not used; generate full tod */
 386   vtype_todl,           /* value not used; generate log tod */
 387   vtype_todlf,          /* value not used; generate log file datestamp tod */
 388   vtype_todzone,        /* value not used; generate time zone only */
 389   vtype_todzulu,        /* value not used; generate zulu tod */
 390   vtype_reply,          /* value not used; get reply from headers */
 391   vtype_pid,            /* value not used; result is pid */
 392   vtype_host_lookup,    /* value not used; get host name */
 393   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 394   vtype_pspace,         /* partition space; value is T/F for spool/log */
 395   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 396   vtype_cert            /* SSL certificate */
 397   #ifndef DISABLE_DKIM
 398   ,vtype_dkim           /* Lookup of value in DKIM signature */
 399   #endif
 400   };
 401
 402 static uschar * fn_recipients(void);
 403
 404 /* This table must be kept in alphabetical order. */
 405
 406 static var_entry var_table[] = {
 407   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 408      they will be confused with user-creatable ACL variables. */
 409   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 410   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 411   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 412   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 413   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 414   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 415   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 416   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 417   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 418   { "acl_narg",            vtype_int,         &acl_narg },
 419   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 420   { "address_data",        vtype_stringptr,   &deliver_address_data },
 421   { "address_file",        vtype_stringptr,   &address_file },
 422   { "address_pipe",        vtype_stringptr,   &address_pipe },
 423   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 424   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 425   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 426   { "authentication_failed",vtype_int,        &authentication_failed },
 427 #ifdef WITH_CONTENT_SCAN
 428   { "av_failed",           vtype_int,         &av_failed },
 429 #endif
 430 #ifdef EXPERIMENTAL_BRIGHTMAIL
 431   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 432   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 433   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 434   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 435 #endif
 436   { "body_linecount",      vtype_int,         &body_linecount },
 437   { "body_zerocount",      vtype_int,         &body_zerocount },
 438   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 439   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 440   { "caller_gid",          vtype_gid,         &real_gid },
 441   { "caller_uid",          vtype_uid,         &real_uid },
 442   { "compile_date",        vtype_stringptr,   &version_date },
 443   { "compile_number",      vtype_stringptr,   &version_cnumber },
 444   { "csa_status",          vtype_stringptr,   &csa_status },
 445 #ifdef EXPERIMENTAL_DCC
 446   { "dcc_header",          vtype_stringptr,   &dcc_header },
 447   { "dcc_result",          vtype_stringptr,   &dcc_result },
 448 #endif
 449 #ifdef WITH_OLD_DEMIME
 450   { "demime_errorlevel",   vtype_int,         &demime_errorlevel },
 451   { "demime_reason",       vtype_stringptr,   &demime_reason },
 452 #endif
 453 #ifndef DISABLE_DKIM
 454   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 455   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 456   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 457   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 458   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 459   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 460   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 461   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 462   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 463   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 464   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 465   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 466   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 467   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 468   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 469   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 470   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 471   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 472   { "dkim_verify_reason",  vtype_dkim,        (void *)DKIM_VERIFY_REASON },
 473   { "dkim_verify_status",  vtype_dkim,        (void *)DKIM_VERIFY_STATUS},
 474 #endif
 475 #ifdef EXPERIMENTAL_DMARC
 476   { "dmarc_ar_header",     vtype_stringptr,   &dmarc_ar_header },
 477   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 478   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 479   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 480   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 481 #endif
 482   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 483   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 484   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 485   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 486   { "domain",              vtype_stringptr,   &deliver_domain },
 487   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 488   { "exim_gid",            vtype_gid,         &exim_gid },
 489   { "exim_path",           vtype_stringptr,   &exim_path },
 490   { "exim_uid",            vtype_uid,         &exim_uid },
 491 #ifdef WITH_OLD_DEMIME
 492   { "found_extension",     vtype_stringptr,   &found_extension },
 493 #endif
 494   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 495   { "home",                vtype_stringptr,   &deliver_home },
 496   { "host",                vtype_stringptr,   &deliver_host },
 497   { "host_address",        vtype_stringptr,   &deliver_host_address },
 498   { "host_data",           vtype_stringptr,   &host_data },
 499   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 500   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 501   { "inode",               vtype_ino,         &deliver_inode },
 502   { "interface_address",   vtype_stringptr,   &interface_address },
 503   { "interface_port",      vtype_int,         &interface_port },
 504   { "item",                vtype_stringptr,   &iterate_item },
 505   #ifdef LOOKUP_LDAP
 506   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 507   #endif
 508   { "load_average",        vtype_load_avg,    NULL },
 509   { "local_part",          vtype_stringptr,   &deliver_localpart },
 510   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 511   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 512   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 513   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 514   { "local_user_gid",      vtype_gid,         &local_user_gid },
 515   { "local_user_uid",      vtype_uid,         &local_user_uid },
 516   { "localhost_number",    vtype_int,         &host_number },
 517   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 518   { "log_space",           vtype_pspace,      (void *)FALSE },
 519   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 520   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 521 #ifdef WITH_CONTENT_SCAN
 522   { "malware_name",        vtype_stringptr,   &malware_name },
 523 #endif
 524   { "max_received_linelength", vtype_int,     &max_received_linelength },
 525   { "message_age",         vtype_int,         &message_age },
 526   { "message_body",        vtype_msgbody,     &message_body },
 527   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 528   { "message_body_size",   vtype_int,         &message_body_size },
 529   { "message_exim_id",     vtype_stringptr,   &message_id },
 530   { "message_headers",     vtype_msgheaders,  NULL },
 531   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 532   { "message_id",          vtype_stringptr,   &message_id },
 533   { "message_linecount",   vtype_int,         &message_linecount },
 534   { "message_size",        vtype_int,         &message_size },
 535 #ifdef WITH_CONTENT_SCAN
 536   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 537   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 538   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 539   { "mime_charset",        vtype_stringptr,   &mime_charset },
 540   { "mime_content_description", vtype_stringptr, &mime_content_description },
 541   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 542   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 543   { "mime_content_size",   vtype_int,         &mime_content_size },
 544   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 545   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 546   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 547   { "mime_filename",       vtype_stringptr,   &mime_filename },
 548   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 549   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 550   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 551   { "mime_part_count",     vtype_int,         &mime_part_count },
 552 #endif
 553   { "n0",                  vtype_filter_int,  &filter_n[0] },
 554   { "n1",                  vtype_filter_int,  &filter_n[1] },
 555   { "n2",                  vtype_filter_int,  &filter_n[2] },
 556   { "n3",                  vtype_filter_int,  &filter_n[3] },
 557   { "n4",                  vtype_filter_int,  &filter_n[4] },
 558   { "n5",                  vtype_filter_int,  &filter_n[5] },
 559   { "n6",                  vtype_filter_int,  &filter_n[6] },
 560   { "n7",                  vtype_filter_int,  &filter_n[7] },
 561   { "n8",                  vtype_filter_int,  &filter_n[8] },
 562   { "n9",                  vtype_filter_int,  &filter_n[9] },
 563   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 564   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 565   { "originator_gid",      vtype_gid,         &originator_gid },
 566   { "originator_uid",      vtype_uid,         &originator_uid },
 567   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 568   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 569   { "pid",                 vtype_pid,         NULL },
 570   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 571 #ifdef EXPERIMENTAL_PROXY
 572   { "proxy_host_address",  vtype_stringptr,   &proxy_host_address },
 573   { "proxy_host_port",     vtype_int,         &proxy_host_port },
 574   { "proxy_session",       vtype_bool,        &proxy_session },
 575   { "proxy_target_address",vtype_stringptr,   &proxy_target_address },
 576   { "proxy_target_port",   vtype_int,         &proxy_target_port },
 577 #endif
 578   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 579   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 580   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 581   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 582   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 583   { "rcpt_count",          vtype_int,         &rcpt_count },
 584   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 585   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 586   { "received_count",      vtype_int,         &received_count },
 587   { "received_for",        vtype_stringptr,   &received_for },
 588   { "received_ip_address", vtype_stringptr,   &interface_address },
 589   { "received_port",       vtype_int,         &interface_port },
 590   { "received_protocol",   vtype_stringptr,   &received_protocol },
 591   { "received_time",       vtype_int,         &received_time },
 592   { "recipient_data",      vtype_stringptr,   &recipient_data },
 593   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 594   { "recipients",          vtype_string_func, &fn_recipients },
 595   { "recipients_count",    vtype_int,         &recipients_count },
 596 #ifdef WITH_CONTENT_SCAN
 597   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 598 #endif
 599   { "reply_address",       vtype_reply,       NULL },
 600   { "return_path",         vtype_stringptr,   &return_path },
 601   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 602   { "router_name",         vtype_stringptr,   &router_name },
 603   { "runrc",               vtype_int,         &runrc },
 604   { "self_hostname",       vtype_stringptr,   &self_hostname },
 605   { "sender_address",      vtype_stringptr,   &sender_address },
 606   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 607   { "sender_address_domain", vtype_domain,    &sender_address },
 608   { "sender_address_local_part", vtype_localpart, &sender_address },
 609   { "sender_data",         vtype_stringptr,   &sender_data },
 610   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 611   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 612   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 613   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 614   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 615   { "sender_host_name",    vtype_host_lookup, NULL },
 616   { "sender_host_port",    vtype_int,         &sender_host_port },
 617   { "sender_ident",        vtype_stringptr,   &sender_ident },
 618   { "sender_rate",         vtype_stringptr,   &sender_rate },
 619   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 620   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 621   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 622   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 623   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 624   { "sending_port",        vtype_int,         &sending_port },
 625   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 626   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 627   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 628   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 629   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 630   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 631   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 632   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 633   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 634   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 635   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 636   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 637   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 638   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 639   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 640 #ifdef WITH_CONTENT_SCAN
 641   { "spam_bar",            vtype_stringptr,   &spam_bar },
 642   { "spam_report",         vtype_stringptr,   &spam_report },
 643   { "spam_score",          vtype_stringptr,   &spam_score },
 644   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 645 #endif
 646 #ifdef EXPERIMENTAL_SPF
 647   { "spf_guess",           vtype_stringptr,   &spf_guess },
 648   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 649   { "spf_received",        vtype_stringptr,   &spf_received },
 650   { "spf_result",          vtype_stringptr,   &spf_result },
 651   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 652 #endif
 653   { "spool_directory",     vtype_stringptr,   &spool_directory },
 654   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 655   { "spool_space",         vtype_pspace,      (void *)TRUE },
 656 #ifdef EXPERIMENTAL_SRS
 657   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 658   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 659   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 660   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 661   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 662   { "srs_status",          vtype_stringptr,   &srs_status },
 663 #endif
 664   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 665
 666   /* The non-(in,out) variables are now deprecated */
 667   { "tls_bits",            vtype_int,         &tls_in.bits },
 668   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 669   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 670
 671   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 672   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 673   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 674   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 675   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 676   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 677 #if defined(SUPPORT_TLS) && !defined(USE_GNUTLS)
 678   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 679 #endif
 680   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 681   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 682   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 683   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 684   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 685   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 686 #if defined(SUPPORT_TLS) && !defined(USE_GNUTLS)
 687   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 688 #endif
 689
 690   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 691 #if defined(SUPPORT_TLS) && !defined(USE_GNUTLS)
 692   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 693 #endif
 694
 695   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 696   { "tod_epoch",           vtype_tode,        NULL },
 697   { "tod_epoch_l",         vtype_todel,       NULL },
 698   { "tod_full",            vtype_todf,        NULL },
 699   { "tod_log",             vtype_todl,        NULL },
 700   { "tod_logfile",         vtype_todlf,       NULL },
 701   { "tod_zone",            vtype_todzone,     NULL },
 702   { "tod_zulu",            vtype_todzulu,     NULL },
 703 #ifdef EXPERIMENTAL_TPDA
 704   { "tpda_defer_errno",     vtype_int,         &tpda_defer_errno },
 705   { "tpda_defer_errstr",    vtype_stringptr,   &tpda_defer_errstr },
 706   { "tpda_delivery_confirmation", vtype_stringptr,   &tpda_delivery_confirmation },
 707   { "tpda_delivery_domain", vtype_stringptr,   &tpda_delivery_domain },
 708   { "tpda_delivery_fqdn",   vtype_stringptr,   &tpda_delivery_fqdn },
 709   { "tpda_delivery_ip",     vtype_stringptr,   &tpda_delivery_ip },
 710   { "tpda_delivery_local_part",vtype_stringptr,&tpda_delivery_local_part },
 711   { "tpda_delivery_port",   vtype_int,         &tpda_delivery_port },
 712 #endif
 713   { "transport_name",      vtype_stringptr,   &transport_name },
 714   { "value",               vtype_stringptr,   &lookup_value },
 715   { "version_number",      vtype_stringptr,   &version_string },
 716   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 717   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 718   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 719   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 720   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 721   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 722 };
 723
 724 static int var_table_size = sizeof(var_table)/sizeof(var_entry);
 725 static uschar var_buffer[256];
 726 static BOOL malformed_header;
 727
 728 /* For textual hashes */
 729
 730 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 731                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 732                                "0123456789";
 733
 734 enum { HMAC_MD5, HMAC_SHA1 };
 735
 736 /* For numeric hashes */
 737
 738 static unsigned int prime[] = {
 739   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 740  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 741  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 742
 743 /* For printing modes in symbolic form */
 744
 745 static uschar *mtable_normal[] =
 746   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 747
 748 static uschar *mtable_setid[] =
 749   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 750
 751 static uschar *mtable_sticky[] =
 752   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 753
 754
 755
 756 /*************************************************
 757 *           Tables for UTF-8 support             *
 758 *************************************************/
 759
 760 /* Table of the number of extra characters, indexed by the first character
 761 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 762 0x3d. */
 763
 764 static uschar utf8_table1[] = {
 765   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 766   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 767   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 768   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 769
 770 /* These are the masks for the data bits in the first byte of a character,
 771 indexed by the number of additional bytes. */
 772
 773 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 774
 775 /* Get the next UTF-8 character, advancing the pointer. */
 776
 777 #define GETUTF8INC(c, ptr) \
 778   c = *ptr++; \
 779   if ((c & 0xc0) == 0xc0) \
 780     { \
 781     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 782     int s = 6*a; \
 783     c = (c & utf8_table2[a]) << s; \
 784     while (a-- > 0) \
 785       { \
 786       s -= 6; \
 787       c |= (*ptr++ & 0x3f) << s; \
 788       } \
 789     }
 790
 791
 792 /*************************************************
 793 *           Binary chop search on a table        *
 794 *************************************************/
 795
 796 /* This is used for matching expansion items and operators.
 797
 798 Arguments:
 799   name        the name that is being sought
 800   table       the table to search
 801   table_size  the number of items in the table
 802
 803 Returns:      the offset in the table, or -1
 804 */
 805
 806 static int
 807 chop_match(uschar *name, uschar **table, int table_size)
 808 {
 809 uschar **bot = table;
 810 uschar **top = table + table_size;
 811
 812 while (top > bot)
 813   {
 814   uschar **mid = bot + (top - bot)/2;
 815   int c = Ustrcmp(name, *mid);
 816   if (c == 0) return mid - table;
 817   if (c > 0) bot = mid + 1; else top = mid;
 818   }
 819
 820 return -1;
 821 }
 822
 823
 824
 825 /*************************************************
 826 *          Check a condition string              *
 827 *************************************************/
 828
 829 /* This function is called to expand a string, and test the result for a "true"
 830 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 831 forced fail or lookup defer.
 832
 833 We used to release all store used, but this is not not safe due
 834 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 835 is reasonably careful to release what it can.
 836
 837 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 838
 839 Arguments:
 840   condition     the condition string
 841   m1            text to be incorporated in panic error
 842   m2            ditto
 843
 844 Returns:        TRUE if condition is met, FALSE if not
 845 */
 846
 847 BOOL
 848 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 849 {
 850 int rc;
 851 uschar *ss = expand_string(condition);
 852 if (ss == NULL)
 853   {
 854   if (!expand_string_forcedfail && !search_find_defer)
 855     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 856       "for %s %s: %s", condition, m1, m2, expand_string_message);
 857   return FALSE;
 858   }
 859 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 860   strcmpic(ss, US"false") != 0;
 861 return rc;
 862 }
 863
 864
 865
 866
 867 /*************************************************
 868 *        Pseudo-random number generation         *
 869 *************************************************/
 870
 871 /* Pseudo-random number generation.  The result is not "expected" to be
 872 cryptographically strong but not so weak that someone will shoot themselves
 873 in the foot using it as a nonce in some email header scheme or whatever
 874 weirdness they'll twist this into.  The result should ideally handle fork().
 875
 876 However, if we're stuck unable to provide this, then we'll fall back to
 877 appallingly bad randomness.
 878
 879 If SUPPORT_TLS is defined then this will not be used except as an emergency
 880 fallback.
 881
 882 Arguments:
 883   max       range maximum
 884 Returns     a random number in range [0, max-1]
 885 */
 886
 887 #ifdef SUPPORT_TLS
 888 # define vaguely_random_number vaguely_random_number_fallback
 889 #endif
 890 int
 891 vaguely_random_number(int max)
 892 {
 893 #ifdef SUPPORT_TLS
 894 # undef vaguely_random_number
 895 #endif
 896   static pid_t pid = 0;
 897   pid_t p2;
 898 #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
 899   struct timeval tv;
 900 #endif
 901
 902   p2 = getpid();
 903   if (p2 != pid)
 904     {
 905     if (pid != 0)
 906       {
 907
 908 #ifdef HAVE_ARC4RANDOM
 909       /* cryptographically strong randomness, common on *BSD platforms, not
 910       so much elsewhere.  Alas. */
 911       arc4random_stir();
 912 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 913 #ifdef HAVE_SRANDOMDEV
 914       /* uses random(4) for seeding */
 915       srandomdev();
 916 #else
 917       gettimeofday(&tv, NULL);
 918       srandom(tv.tv_sec | tv.tv_usec | getpid());
 919 #endif
 920 #else
 921       /* Poor randomness and no seeding here */
 922 #endif
 923
 924       }
 925     pid = p2;
 926     }
 927
 928 #ifdef HAVE_ARC4RANDOM
 929   return arc4random() % max;
 930 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 931   return random() % max;
 932 #else
 933   /* This one returns a 16-bit number, definitely not crypto-strong */
 934   return random_number(max);
 935 #endif
 936 }
 937
 938
 939
 940
 941 /*************************************************
 942 *             Pick out a name from a string      *
 943 *************************************************/
 944
 945 /* If the name is too long, it is silently truncated.
 946
 947 Arguments:
 948   name      points to a buffer into which to put the name
 949   max       is the length of the buffer
 950   s         points to the first alphabetic character of the name
 951   extras    chars other than alphanumerics to permit
 952
 953 Returns:    pointer to the first character after the name
 954
 955 Note: The test for *s != 0 in the while loop is necessary because
 956 Ustrchr() yields non-NULL if the character is zero (which is not something
 957 I expected). */
 958
 959 static uschar *
 960 read_name(uschar *name, int max, uschar *s, uschar *extras)
 961 {
 962 int ptr = 0;
 963 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
 964   {
 965   if (ptr < max-1) name[ptr++] = *s;
 966   s++;
 967   }
 968 name[ptr] = 0;
 969 return s;
 970 }
 971
 972
 973
 974 /*************************************************
 975 *     Pick out the rest of a header name         *
 976 *************************************************/
 977
 978 /* A variable name starting $header_ (or just $h_ for those who like
 979 abbreviations) might not be the complete header name because headers can
 980 contain any printing characters in their names, except ':'. This function is
 981 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
 982 on the end, if the name was terminated by white space.
 983
 984 Arguments:
 985   name      points to a buffer in which the name read so far exists
 986   max       is the length of the buffer
 987   s         points to the first character after the name so far, i.e. the
 988             first non-alphameric character after $header_xxxxx
 989
 990 Returns:    a pointer to the first character after the header name
 991 */
 992
 993 static uschar *
 994 read_header_name(uschar *name, int max, uschar *s)
 995 {
 996 int prelen = Ustrchr(name, '_') - name + 1;
 997 int ptr = Ustrlen(name) - prelen;
 998 if (ptr > 0) memmove(name, name+prelen, ptr);
 999 while (mac_isgraph(*s) && *s != ':')
1000   {
1001   if (ptr < max-1) name[ptr++] = *s;
1002   s++;
1003   }
1004 if (*s == ':') s++;
1005 name[ptr++] = ':';
1006 name[ptr] = 0;
1007 return s;
1008 }
1009
1010
1011
1012 /*************************************************
1013 *           Pick out a number from a string      *
1014 *************************************************/
1015
1016 /* Arguments:
1017   n     points to an integer into which to put the number
1018   s     points to the first digit of the number
1019
1020 Returns:  a pointer to the character after the last digit
1021 */
1022
1023 static uschar *
1024 read_number(int *n, uschar *s)
1025 {
1026 *n = 0;
1027 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1028 return s;
1029 }
1030
1031
1032
1033 /*************************************************
1034 *        Extract keyed subfield from a string    *
1035 *************************************************/
1036
1037 /* The yield is in dynamic store; NULL means that the key was not found.
1038
1039 Arguments:
1040   key       points to the name of the key
1041   s         points to the string from which to extract the subfield
1042
1043 Returns:    NULL if the subfield was not found, or
1044             a pointer to the subfield's data
1045 */
1046
1047 static uschar *
1048 expand_getkeyed(uschar *key, uschar *s)
1049 {
1050 int length = Ustrlen(key);
1051 while (isspace(*s)) s++;
1052
1053 /* Loop to search for the key */
1054
1055 while (*s != 0)
1056   {
1057   int dkeylength;
1058   uschar *data;
1059   uschar *dkey = s;
1060
1061   while (*s != 0 && *s != '=' && !isspace(*s)) s++;
1062   dkeylength = s - dkey;
1063   while (isspace(*s)) s++;
1064   if (*s == '=') while (isspace((*(++s))));
1065
1066   data = string_dequote(&s);
1067   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1068     return data;
1069
1070   while (isspace(*s)) s++;
1071   }
1072
1073 return NULL;
1074 }
1075
1076
1077
1078 static var_entry *
1079 find_var_ent(uschar * name)
1080 {
1081 int first = 0;
1082 int last = var_table_size;
1083
1084 while (last > first)
1085   {
1086   int middle = (first + last)/2;
1087   int c = Ustrcmp(name, var_table[middle].name);
1088
1089   if (c > 0) { first = middle + 1; continue; }
1090   if (c < 0) { last = middle; continue; }
1091   return &var_table[middle];
1092   }
1093 return NULL;
1094 }
1095
1096 /*************************************************
1097 *   Extract numbered subfield from string        *
1098 *************************************************/
1099
1100 /* Extracts a numbered field from a string that is divided by tokens - for
1101 example a line from /etc/passwd is divided by colon characters.  First field is
1102 numbered one.  Negative arguments count from the right. Zero returns the whole
1103 string. Returns NULL if there are insufficient tokens in the string
1104
1105 ***WARNING***
1106 Modifies final argument - this is a dynamically generated string, so that's OK.
1107
1108 Arguments:
1109   field       number of field to be extracted,
1110                 first field = 1, whole string = 0, last field = -1
1111   separators  characters that are used to break string into tokens
1112   s           points to the string from which to extract the subfield
1113
1114 Returns:      NULL if the field was not found,
1115               a pointer to the field's data inside s (modified to add 0)
1116 */
1117
1118 static uschar *
1119 expand_gettokened (int field, uschar *separators, uschar *s)
1120 {
1121 int sep = 1;
1122 int count;
1123 uschar *ss = s;
1124 uschar *fieldtext = NULL;
1125
1126 if (field == 0) return s;
1127
1128 /* Break the line up into fields in place; for field > 0 we stop when we have
1129 done the number of fields we want. For field < 0 we continue till the end of
1130 the string, counting the number of fields. */
1131
1132 count = (field > 0)? field : INT_MAX;
1133
1134 while (count-- > 0)
1135   {
1136   size_t len;
1137
1138   /* Previous field was the last one in the string. For a positive field
1139   number, this means there are not enough fields. For a negative field number,
1140   check that there are enough, and scan back to find the one that is wanted. */
1141
1142   if (sep == 0)
1143     {
1144     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1145     if ((-field) == (INT_MAX - count - 1)) return s;
1146     while (field++ < 0)
1147       {
1148       ss--;
1149       while (ss[-1] != 0) ss--;
1150       }
1151     fieldtext = ss;
1152     break;
1153     }
1154
1155   /* Previous field was not last in the string; save its start and put a
1156   zero at its end. */
1157
1158   fieldtext = ss;
1159   len = Ustrcspn(ss, separators);
1160   sep = ss[len];
1161   ss[len] = 0;
1162   ss += len + 1;
1163   }
1164
1165 return fieldtext;
1166 }
1167
1168
1169 static uschar *
1170 expand_getlistele(int field, uschar * list)
1171 {
1172 uschar * tlist= list;
1173 int sep= 0;
1174 uschar dummy;
1175
1176 if(field<0)
1177 {
1178   for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1179   sep= 0;
1180 }
1181 if(field==0) return NULL;
1182 while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1183 return string_nextinlist(&list, &sep, NULL, 0);
1184 }
1185
1186
1187 /* Certificate fields, by name.  Worry about by-OID later */
1188
1189 #ifdef SUPPORT_TLS
1190 typedef struct
1191 {
1192 uschar * name;
1193 uschar * (*getfn)(void * cert);
1194 } certfield;
1195 static certfield certfields[] =
1196 {                       /* linear search; no special order */
1197   { US"version",        &tls_cert_version },
1198   { US"serial_number",  &tls_cert_serial_number },
1199   { US"subject",        &tls_cert_subject },
1200   { US"notbefore",      &tls_cert_not_before },
1201   { US"notafter",       &tls_cert_not_after },
1202   { US"issuer",         &tls_cert_issuer },
1203   { US"signature",      &tls_cert_signature },
1204   { US"signature_algorithm",    &tls_cert_signature_algorithm },
1205   { US"subject_altname",        &tls_cert_subject_altname },
1206   { US"ocsp_uri",       &tls_cert_ocsp_uri },
1207   { US"crl_uri",        &tls_cert_crl_uri },
1208 };
1209
1210 static uschar *
1211 expand_getcertele(uschar * field, uschar * certvar)
1212 {
1213 var_entry * vp;
1214 certfield * cp;
1215
1216 if (!(vp = find_var_ent(certvar)))
1217   {
1218   expand_string_message =
1219     string_sprintf("no variable named \"%s\"", certvar);
1220   return NULL;          /* Unknown variable name */
1221   }
1222 /* NB this stops us passing certs around in variable.  Might
1223 want to do that in future */
1224 if (vp->type != vtype_cert)
1225   {
1226   expand_string_message =
1227     string_sprintf("\"%s\" is not a certificate", certvar);
1228   return NULL;          /* Unknown variable name */
1229   }
1230 if (!*(void **)vp->value)
1231   return NULL;
1232
1233 if (*field >= '0' && *field <= '9')
1234   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1235
1236 for(cp = certfields;
1237     cp < certfields + nelements(certfields);
1238     cp++)
1239   if (Ustrcmp(cp->name, field) == 0)
1240     return (*cp->getfn)( *(void **)vp->value );
1241
1242 expand_string_message =
1243   string_sprintf("bad field selector \"%s\" for certextract", field);
1244 return NULL;
1245 }
1246 #endif  /*SUPPORT_TLS*/
1247
1248 /*************************************************
1249 *        Extract a substring from a string       *
1250 *************************************************/
1251
1252 /* Perform the ${substr or ${length expansion operations.
1253
1254 Arguments:
1255   subject     the input string
1256   value1      the offset from the start of the input string to the start of
1257                 the output string; if negative, count from the right.
1258   value2      the length of the output string, or negative (-1) for unset
1259                 if value1 is positive, unset means "all after"
1260                 if value1 is negative, unset means "all before"
1261   len         set to the length of the returned string
1262
1263 Returns:      pointer to the output string, or NULL if there is an error
1264 */
1265
1266 static uschar *
1267 extract_substr(uschar *subject, int value1, int value2, int *len)
1268 {
1269 int sublen = Ustrlen(subject);
1270
1271 if (value1 < 0)    /* count from right */
1272   {
1273   value1 += sublen;
1274
1275   /* If the position is before the start, skip to the start, and adjust the
1276   length. If the length ends up negative, the substring is null because nothing
1277   can precede. This falls out naturally when the length is unset, meaning "all
1278   to the left". */
1279
1280   if (value1 < 0)
1281     {
1282     value2 += value1;
1283     if (value2 < 0) value2 = 0;
1284     value1 = 0;
1285     }
1286
1287   /* Otherwise an unset length => characters before value1 */
1288
1289   else if (value2 < 0)
1290     {
1291     value2 = value1;
1292     value1 = 0;
1293     }
1294   }
1295
1296 /* For a non-negative offset, if the starting position is past the end of the
1297 string, the result will be the null string. Otherwise, an unset length means
1298 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1299
1300 else
1301   {
1302   if (value1 > sublen)
1303     {
1304     value1 = sublen;
1305     value2 = 0;
1306     }
1307   else if (value2 < 0) value2 = sublen;
1308   }
1309
1310 /* Cut the length down to the maximum possible for the offset value, and get
1311 the required characters. */
1312
1313 if (value1 + value2 > sublen) value2 = sublen - value1;
1314 *len = value2;
1315 return subject + value1;
1316 }
1317
1318
1319
1320
1321 /*************************************************
1322 *            Old-style hash of a string          *
1323 *************************************************/
1324
1325 /* Perform the ${hash expansion operation.
1326
1327 Arguments:
1328   subject     the input string (an expanded substring)
1329   value1      the length of the output string; if greater or equal to the
1330                 length of the input string, the input string is returned
1331   value2      the number of hash characters to use, or 26 if negative
1332   len         set to the length of the returned string
1333
1334 Returns:      pointer to the output string, or NULL if there is an error
1335 */
1336
1337 static uschar *
1338 compute_hash(uschar *subject, int value1, int value2, int *len)
1339 {
1340 int sublen = Ustrlen(subject);
1341
1342 if (value2 < 0) value2 = 26;
1343 else if (value2 > Ustrlen(hashcodes))
1344   {
1345   expand_string_message =
1346     string_sprintf("hash count \"%d\" too big", value2);
1347   return NULL;
1348   }
1349
1350 /* Calculate the hash text. We know it is shorter than the original string, so
1351 can safely place it in subject[] (we know that subject is always itself an
1352 expanded substring). */
1353
1354 if (value1 < sublen)
1355   {
1356   int c;
1357   int i = 0;
1358   int j = value1;
1359   while ((c = (subject[j])) != 0)
1360     {
1361     int shift = (c + j++) & 7;
1362     subject[i] ^= (c << shift) | (c >> (8-shift));
1363     if (++i >= value1) i = 0;
1364     }
1365   for (i = 0; i < value1; i++)
1366     subject[i] = hashcodes[(subject[i]) % value2];
1367   }
1368 else value1 = sublen;
1369
1370 *len = value1;
1371 return subject;
1372 }
1373
1374
1375
1376
1377 /*************************************************
1378 *             Numeric hash of a string           *
1379 *************************************************/
1380
1381 /* Perform the ${nhash expansion operation. The first characters of the
1382 string are treated as most important, and get the highest prime numbers.
1383
1384 Arguments:
1385   subject     the input string
1386   value1      the maximum value of the first part of the result
1387   value2      the maximum value of the second part of the result,
1388                 or negative to produce only a one-part result
1389   len         set to the length of the returned string
1390
1391 Returns:  pointer to the output string, or NULL if there is an error.
1392 */
1393
1394 static uschar *
1395 compute_nhash (uschar *subject, int value1, int value2, int *len)
1396 {
1397 uschar *s = subject;
1398 int i = 0;
1399 unsigned long int total = 0; /* no overflow */
1400
1401 while (*s != 0)
1402   {
1403   if (i == 0) i = sizeof(prime)/sizeof(int) - 1;
1404   total += prime[i--] * (unsigned int)(*s++);
1405   }
1406
1407 /* If value2 is unset, just compute one number */
1408
1409 if (value2 < 0)
1410   {
1411   s = string_sprintf("%d", total % value1);
1412   }
1413
1414 /* Otherwise do a div/mod hash */
1415
1416 else
1417   {
1418   total = total % (value1 * value2);
1419   s = string_sprintf("%d/%d", total/value2, total % value2);
1420   }
1421
1422 *len = Ustrlen(s);
1423 return s;
1424 }
1425
1426
1427
1428
1429
1430 /*************************************************
1431 *     Find the value of a header or headers      *
1432 *************************************************/
1433
1434 /* Multiple instances of the same header get concatenated, and this function
1435 can also return a concatenation of all the header lines. When concatenating
1436 specific headers that contain lists of addresses, a comma is inserted between
1437 them. Otherwise we use a straight concatenation. Because some messages can have
1438 pathologically large number of lines, there is a limit on the length that is
1439 returned. Also, to avoid massive store use which would result from using
1440 string_cat() as it copies and extends strings, we do a preliminary pass to find
1441 out exactly how much store will be needed. On "normal" messages this will be
1442 pretty trivial.
1443
1444 Arguments:
1445   name          the name of the header, without the leading $header_ or $h_,
1446                 or NULL if a concatenation of all headers is required
1447   exists_only   TRUE if called from a def: test; don't need to build a string;
1448                 just return a string that is not "" and not "0" if the header
1449                 exists
1450   newsize       return the size of memory block that was obtained; may be NULL
1451                 if exists_only is TRUE
1452   want_raw      TRUE if called for $rh_ or $rheader_ variables; no processing,
1453                 other than concatenating, will be done on the header. Also used
1454                 for $message_headers_raw.
1455   charset       name of charset to translate MIME words to; used only if
1456                 want_raw is false; if NULL, no translation is done (this is
1457                 used for $bh_ and $bheader_)
1458
1459 Returns:        NULL if the header does not exist, else a pointer to a new
1460                 store block
1461 */
1462
1463 static uschar *
1464 find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw,
1465   uschar *charset)
1466 {
1467 BOOL found = name == NULL;
1468 int comma = 0;
1469 int len = found? 0 : Ustrlen(name);
1470 int i;
1471 uschar *yield = NULL;
1472 uschar *ptr = NULL;
1473
1474 /* Loop for two passes - saves code repetition */
1475
1476 for (i = 0; i < 2; i++)
1477   {
1478   int size = 0;
1479   header_line *h;
1480
1481   for (h = header_list; size < header_insert_maxlen && h != NULL; h = h->next)
1482     {
1483     if (h->type != htype_old && h->text != NULL)  /* NULL => Received: placeholder */
1484       {
1485       if (name == NULL || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1486         {
1487         int ilen;
1488         uschar *t;
1489
1490         if (exists_only) return US"1";      /* don't need actual string */
1491         found = TRUE;
1492         t = h->text + len;                  /* text to insert */
1493         if (!want_raw)                      /* unless wanted raw, */
1494           while (isspace(*t)) t++;          /* remove leading white space */
1495         ilen = h->slen - (t - h->text);     /* length to insert */
1496
1497         /* Unless wanted raw, remove trailing whitespace, including the
1498         newline. */
1499
1500         if (!want_raw)
1501           while (ilen > 0 && isspace(t[ilen-1])) ilen--;
1502
1503         /* Set comma = 1 if handling a single header and it's one of those
1504         that contains an address list, except when asked for raw headers. Only
1505         need to do this once. */
1506
1507         if (!want_raw && name != NULL && comma == 0 &&
1508             Ustrchr("BCFRST", h->type) != NULL)
1509           comma = 1;
1510
1511         /* First pass - compute total store needed; second pass - compute
1512         total store used, including this header. */
1513
1514         size += ilen + comma + 1;  /* +1 for the newline */
1515
1516         /* Second pass - concatentate the data, up to a maximum. Note that
1517         the loop stops when size hits the limit. */
1518
1519         if (i != 0)
1520           {
1521           if (size > header_insert_maxlen)
1522             {
1523             ilen -= size - header_insert_maxlen - 1;
1524             comma = 0;
1525             }
1526           Ustrncpy(ptr, t, ilen);
1527           ptr += ilen;
1528
1529           /* For a non-raw header, put in the comma if needed, then add
1530           back the newline we removed above, provided there was some text in
1531           the header. */
1532
1533           if (!want_raw && ilen > 0)
1534             {
1535             if (comma != 0) *ptr++ = ',';
1536             *ptr++ = '\n';
1537             }
1538           }
1539         }
1540       }
1541     }
1542
1543   /* At end of first pass, return NULL if no header found. Then truncate size
1544   if necessary, and get the buffer to hold the data, returning the buffer size.
1545   */
1546
1547   if (i == 0)
1548     {
1549     if (!found) return NULL;
1550     if (size > header_insert_maxlen) size = header_insert_maxlen;
1551     *newsize = size + 1;
1552     ptr = yield = store_get(*newsize);
1553     }
1554   }
1555
1556 /* That's all we do for raw header expansion. */
1557
1558 if (want_raw)
1559   {
1560   *ptr = 0;
1561   }
1562
1563 /* Otherwise, remove a final newline and a redundant added comma. Then we do
1564 RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2()
1565 function can return an error with decoded data if the charset translation
1566 fails. If decoding fails, it returns NULL. */
1567
1568 else
1569   {
1570   uschar *decoded, *error;
1571   if (ptr > yield && ptr[-1] == '\n') ptr--;
1572   if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--;
1573   *ptr = 0;
1574   decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL,
1575     newsize, &error);
1576   if (error != NULL)
1577     {
1578     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1579       "    input was: %s\n", error, yield);
1580     }
1581   if (decoded != NULL) yield = decoded;
1582   }
1583
1584 return yield;
1585 }
1586
1587
1588
1589
1590 /*************************************************
1591 *               Return list of recipients        *
1592 *************************************************/
1593 /* A recipients list is available only during system message filtering,
1594 during ACL processing after DATA, and while expanding pipe commands
1595 generated from a system filter, but not elsewhere. */
1596
1597 static uschar *
1598 fn_recipients(void)
1599 {
1600 if (!enable_dollar_recipients) return NULL; else
1601   {
1602   int size = 128;
1603   int ptr = 0;
1604   int i;
1605   uschar * s = store_get(size);
1606   for (i = 0; i < recipients_count; i++)
1607     {
1608     if (i != 0) s = string_cat(s, &size, &ptr, US", ", 2);
1609     s = string_cat(s, &size, &ptr, recipients_list[i].address,
1610       Ustrlen(recipients_list[i].address));
1611     }
1612   s[ptr] = 0;     /* string_cat() leaves room */
1613   return s;
1614   }
1615 }
1616
1617
1618 /*************************************************
1619 *               Find value of a variable         *
1620 *************************************************/
1621
1622 /* The table of variables is kept in alphabetic order, so we can search it
1623 using a binary chop. The "choplen" variable is nothing to do with the binary
1624 chop.
1625
1626 Arguments:
1627   name          the name of the variable being sought
1628   exists_only   TRUE if this is a def: test; passed on to find_header()
1629   skipping      TRUE => skip any processing evaluation; this is not the same as
1630                   exists_only because def: may test for values that are first
1631                   evaluated here
1632   newsize       pointer to an int which is initially zero; if the answer is in
1633                 a new memory buffer, *newsize is set to its size
1634
1635 Returns:        NULL if the variable does not exist, or
1636                 a pointer to the variable's contents, or
1637                 something non-NULL if exists_only is TRUE
1638 */
1639
1640 static uschar *
1641 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1642 {
1643 var_entry * vp;
1644 uschar *s, *domain;
1645 uschar **ss;
1646 void * val;
1647
1648 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1649 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1650 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1651 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1652 (this gave backwards compatibility at the changeover). There may be built-in
1653 variables whose names start acl_ but they should never start in this way. This
1654 slightly messy specification is a consequence of the history, needless to say.
1655
1656 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1657 set, in which case give an error. */
1658
1659 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1660      !isalpha(name[5]))
1661   {
1662   tree_node *node =
1663     tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4);
1664   return (node == NULL)? (strict_acl_vars? NULL : US"") : node->data.ptr;
1665   }
1666
1667 /* Handle $auth<n> variables. */
1668
1669 if (Ustrncmp(name, "auth", 4) == 0)
1670   {
1671   uschar *endptr;
1672   int n = Ustrtoul(name + 4, &endptr, 10);
1673   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1674     return (auth_vars[n-1] == NULL)? US"" : auth_vars[n-1];
1675   }
1676
1677 /* For all other variables, search the table */
1678
1679 if (!(vp = find_var_ent(name)))
1680   return NULL;          /* Unknown variable name */
1681
1682 /* Found an existing variable. If in skipping state, the value isn't needed,
1683 and we want to avoid processing (such as looking up the host name). */
1684
1685 if (skipping)
1686   return US"";
1687
1688 val = vp->value;
1689 switch (vp->type)
1690   {
1691   case vtype_filter_int:
1692   if (!filter_running) return NULL;
1693   /* Fall through */
1694   /* VVVVVVVVVVVV */
1695   case vtype_int:
1696   sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1697   return var_buffer;
1698
1699   case vtype_ino:
1700   sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1701   return var_buffer;
1702
1703   case vtype_gid:
1704   sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1705   return var_buffer;
1706
1707   case vtype_uid:
1708   sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1709   return var_buffer;
1710
1711   case vtype_bool:
1712   sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1713   return var_buffer;
1714
1715   case vtype_stringptr:                      /* Pointer to string */
1716   s = *((uschar **)(val));
1717   return (s == NULL)? US"" : s;
1718
1719   case vtype_pid:
1720   sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1721   return var_buffer;
1722
1723   case vtype_load_avg:
1724   sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1725   return var_buffer;
1726
1727   case vtype_host_lookup:                    /* Lookup if not done so */
1728   if (sender_host_name == NULL && sender_host_address != NULL &&
1729       !host_lookup_failed && host_name_lookup() == OK)
1730     host_build_sender_fullhost();
1731   return (sender_host_name == NULL)? US"" : sender_host_name;
1732
1733   case vtype_localpart:                      /* Get local part from address */
1734   s = *((uschar **)(val));
1735   if (s == NULL) return US"";
1736   domain = Ustrrchr(s, '@');
1737   if (domain == NULL) return s;
1738   if (domain - s > sizeof(var_buffer) - 1)
1739     log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1740         " in string expansion", sizeof(var_buffer));
1741   Ustrncpy(var_buffer, s, domain - s);
1742   var_buffer[domain - s] = 0;
1743   return var_buffer;
1744
1745   case vtype_domain:                         /* Get domain from address */
1746   s = *((uschar **)(val));
1747   if (s == NULL) return US"";
1748   domain = Ustrrchr(s, '@');
1749   return (domain == NULL)? US"" : domain + 1;
1750
1751   case vtype_msgheaders:
1752   return find_header(NULL, exists_only, newsize, FALSE, NULL);
1753
1754   case vtype_msgheaders_raw:
1755   return find_header(NULL, exists_only, newsize, TRUE, NULL);
1756
1757   case vtype_msgbody:                        /* Pointer to msgbody string */
1758   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1759   ss = (uschar **)(val);
1760   if (*ss == NULL && deliver_datafile >= 0)  /* Read body when needed */
1761     {
1762     uschar *body;
1763     off_t start_offset = SPOOL_DATA_START_OFFSET;
1764     int len = message_body_visible;
1765     if (len > message_size) len = message_size;
1766     *ss = body = store_malloc(len+1);
1767     body[0] = 0;
1768     if (vp->type == vtype_msgbody_end)
1769       {
1770       struct stat statbuf;
1771       if (fstat(deliver_datafile, &statbuf) == 0)
1772         {
1773         start_offset = statbuf.st_size - len;
1774         if (start_offset < SPOOL_DATA_START_OFFSET)
1775           start_offset = SPOOL_DATA_START_OFFSET;
1776         }
1777       }
1778     lseek(deliver_datafile, start_offset, SEEK_SET);
1779     len = read(deliver_datafile, body, len);
1780     if (len > 0)
1781       {
1782       body[len] = 0;
1783       if (message_body_newlines)   /* Separate loops for efficiency */
1784         {
1785         while (len > 0)
1786           { if (body[--len] == 0) body[len] = ' '; }
1787         }
1788       else
1789         {
1790         while (len > 0)
1791           { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1792         }
1793       }
1794     }
1795   return (*ss == NULL)? US"" : *ss;
1796
1797   case vtype_todbsdin:                       /* BSD inbox time of day */
1798   return tod_stamp(tod_bsdin);
1799
1800   case vtype_tode:                           /* Unix epoch time of day */
1801   return tod_stamp(tod_epoch);
1802
1803   case vtype_todel:                          /* Unix epoch/usec time of day */
1804   return tod_stamp(tod_epoch_l);
1805
1806   case vtype_todf:                           /* Full time of day */
1807   return tod_stamp(tod_full);
1808
1809   case vtype_todl:                           /* Log format time of day */
1810   return tod_stamp(tod_log_bare);            /* (without timezone) */
1811
1812   case vtype_todzone:                        /* Time zone offset only */
1813   return tod_stamp(tod_zone);
1814
1815   case vtype_todzulu:                        /* Zulu time */
1816   return tod_stamp(tod_zulu);
1817
1818   case vtype_todlf:                          /* Log file datestamp tod */
1819   return tod_stamp(tod_log_datestamp_daily);
1820
1821   case vtype_reply:                          /* Get reply address */
1822   s = find_header(US"reply-to:", exists_only, newsize, TRUE,
1823     headers_charset);
1824   if (s != NULL) while (isspace(*s)) s++;
1825   if (s == NULL || *s == 0)
1826     {
1827     *newsize = 0;                            /* For the *s==0 case */
1828     s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset);
1829     }
1830   if (s != NULL)
1831     {
1832     uschar *t;
1833     while (isspace(*s)) s++;
1834     for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1835     while (t > s && isspace(t[-1])) t--;
1836     *t = 0;
1837     }
1838   return (s == NULL)? US"" : s;
1839
1840   case vtype_string_func:
1841     {
1842     uschar * (*fn)() = val;
1843     return fn();
1844     }
1845
1846   case vtype_pspace:
1847     {
1848     int inodes;
1849     sprintf(CS var_buffer, "%d",
1850       receive_statvfs(val == (void *)TRUE, &inodes));
1851     }
1852   return var_buffer;
1853
1854   case vtype_pinodes:
1855     {
1856     int inodes;
1857     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1858     sprintf(CS var_buffer, "%d", inodes);
1859     }
1860   return var_buffer;
1861
1862   case vtype_cert:
1863   return *(void **)val ? US"<cert>" : US"";
1864
1865   #ifndef DISABLE_DKIM
1866   case vtype_dkim:
1867   return dkim_exim_expand_query((int)(long)val);
1868   #endif
1869
1870   }
1871 }
1872
1873
1874
1875
1876 void
1877 modify_variable(uschar *name, void * value)
1878 {
1879 var_entry * vp;
1880 if ((vp = find_var_ent(name))) vp->value = value;
1881 return;          /* Unknown variable name, fail silently */
1882 }
1883
1884
1885
1886
1887
1888 /*************************************************
1889 *           Read and expand substrings           *
1890 *************************************************/
1891
1892 /* This function is called to read and expand argument substrings for various
1893 expansion items. Some have a minimum requirement that is less than the maximum;
1894 in these cases, the first non-present one is set to NULL.
1895
1896 Arguments:
1897   sub        points to vector of pointers to set
1898   n          maximum number of substrings
1899   m          minimum required
1900   sptr       points to current string pointer
1901   skipping   the skipping flag
1902   check_end  if TRUE, check for final '}'
1903   name       name of item, for error message
1904   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
1905              the store.
1906
1907 Returns:     0 OK; string pointer updated
1908              1 curly bracketing error (too few arguments)
1909              2 too many arguments (only if check_end is set); message set
1910              3 other error (expansion failure)
1911 */
1912
1913 static int
1914 read_subs(uschar **sub, int n, int m, uschar **sptr, BOOL skipping,
1915   BOOL check_end, uschar *name, BOOL *resetok)
1916 {
1917 int i;
1918 uschar *s = *sptr;
1919
1920 while (isspace(*s)) s++;
1921 for (i = 0; i < n; i++)
1922   {
1923   if (*s != '{')
1924     {
1925     if (i < m) return 1;
1926     sub[i] = NULL;
1927     break;
1928     }
1929   sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok);
1930   if (sub[i] == NULL) return 3;
1931   if (*s++ != '}') return 1;
1932   while (isspace(*s)) s++;
1933   }
1934 if (check_end && *s++ != '}')
1935   {
1936   if (s[-1] == '{')
1937     {
1938     expand_string_message = string_sprintf("Too many arguments for \"%s\" "
1939       "(max is %d)", name, n);
1940     return 2;
1941     }
1942   return 1;
1943   }
1944
1945 *sptr = s;
1946 return 0;
1947 }
1948
1949
1950
1951
1952 /*************************************************
1953 *     Elaborate message for bad variable         *
1954 *************************************************/
1955
1956 /* For the "unknown variable" message, take a look at the variable's name, and
1957 give additional information about possible ACL variables. The extra information
1958 is added on to expand_string_message.
1959
1960 Argument:   the name of the variable
1961 Returns:    nothing
1962 */
1963
1964 static void
1965 check_variable_error_message(uschar *name)
1966 {
1967 if (Ustrncmp(name, "acl_", 4) == 0)
1968   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
1969     (name[4] == 'c' || name[4] == 'm')?
1970       (isalpha(name[5])?
1971         US"6th character of a user-defined ACL variable must be a digit or underscore" :
1972         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
1973       ) :
1974       US"user-defined ACL variables must start acl_c or acl_m");
1975 }
1976
1977
1978
1979 /*
1980 Load args from sub array to globals, and call acl_check().
1981 Sub array will be corrupted on return.
1982
1983 Returns:       OK         access is granted by an ACCEPT verb
1984                DISCARD    access is granted by a DISCARD verb
1985                FAIL       access is denied
1986                FAIL_DROP  access is denied; drop the connection
1987                DEFER      can't tell at the moment
1988                ERROR      disaster
1989 */
1990 static int
1991 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
1992 {
1993 int i;
1994 uschar *tmp;
1995 int sav_narg = acl_narg;
1996 int ret;
1997 extern int acl_where;
1998
1999 if(--nsub > sizeof(acl_arg)/sizeof(*acl_arg)) nsub = sizeof(acl_arg)/sizeof(*acl_arg);
2000 for (i = 0; i < nsub && sub[i+1]; i++)
2001   {
2002   tmp = acl_arg[i];
2003   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2004   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2005   }
2006 acl_narg = i;
2007 while (i < nsub)
2008   {
2009   sub[i+1] = acl_arg[i];
2010   acl_arg[i++] = NULL;
2011   }
2012
2013 DEBUG(D_expand)
2014   debug_printf("expanding: acl: %s  arg: %s%s\n",
2015     sub[0],
2016     acl_narg>0 ? acl_arg[0] : US"<none>",
2017     acl_narg>1 ? " +more"   : "");
2018
2019 ret = acl_eval(acl_where, sub[0], user_msgp, &tmp);
2020
2021 for (i = 0; i < nsub; i++)
2022   acl_arg[i] = sub[i+1];        /* restore old args */
2023 acl_narg = sav_narg;
2024
2025 return ret;
2026 }
2027
2028
2029
2030
2031 /*************************************************
2032 *        Read and evaluate a condition           *
2033 *************************************************/
2034
2035 /*
2036 Arguments:
2037   s        points to the start of the condition text
2038   resetok  points to a BOOL which is written false if it is unsafe to
2039            free memory. Certain condition types (acl) may have side-effect
2040            allocation which must be preserved.
2041   yield    points to a BOOL to hold the result of the condition test;
2042            if NULL, we are just reading through a condition that is
2043            part of an "or" combination to check syntax, or in a state
2044            where the answer isn't required
2045
2046 Returns:   a pointer to the first character after the condition, or
2047            NULL after an error
2048 */
2049
2050 static uschar *
2051 eval_condition(uschar *s, BOOL *resetok, BOOL *yield)
2052 {
2053 BOOL testfor = TRUE;
2054 BOOL tempcond, combined_cond;
2055 BOOL *subcondptr;
2056 BOOL sub2_honour_dollar = TRUE;
2057 int i, rc, cond_type, roffset;
2058 int_eximarith_t num[2];
2059 struct stat statbuf;
2060 uschar name[256];
2061 uschar *sub[10];
2062
2063 const pcre *re;
2064 const uschar *rerror;
2065
2066 for (;;)
2067   {
2068   while (isspace(*s)) s++;
2069   if (*s == '!') { testfor = !testfor; s++; } else break;
2070   }
2071
2072 /* Numeric comparisons are symbolic */
2073
2074 if (*s == '=' || *s == '>' || *s == '<')
2075   {
2076   int p = 0;
2077   name[p++] = *s++;
2078   if (*s == '=')
2079     {
2080     name[p++] = '=';
2081     s++;
2082     }
2083   name[p] = 0;
2084   }
2085
2086 /* All other conditions are named */
2087
2088 else s = read_name(name, 256, s, US"_");
2089
2090 /* If we haven't read a name, it means some non-alpha character is first. */
2091
2092 if (name[0] == 0)
2093   {
2094   expand_string_message = string_sprintf("condition name expected, "
2095     "but found \"%.16s\"", s);
2096   return NULL;
2097   }
2098
2099 /* Find which condition we are dealing with, and switch on it */
2100
2101 cond_type = chop_match(name, cond_table, sizeof(cond_table)/sizeof(uschar *));
2102 switch(cond_type)
2103   {
2104   /* def: tests for a non-empty variable, or for the existence of a header. If
2105   yield == NULL we are in a skipping state, and don't care about the answer. */
2106
2107   case ECOND_DEF:
2108   if (*s != ':')
2109     {
2110     expand_string_message = US"\":\" expected after \"def\"";
2111     return NULL;
2112     }
2113
2114   s = read_name(name, 256, s+1, US"_");
2115
2116   /* Test for a header's existence. If the name contains a closing brace
2117   character, this may be a user error where the terminating colon has been
2118   omitted. Set a flag to adjust a subsequent error message in this case. */
2119
2120   if (Ustrncmp(name, "h_", 2) == 0 ||
2121       Ustrncmp(name, "rh_", 3) == 0 ||
2122       Ustrncmp(name, "bh_", 3) == 0 ||
2123       Ustrncmp(name, "header_", 7) == 0 ||
2124       Ustrncmp(name, "rheader_", 8) == 0 ||
2125       Ustrncmp(name, "bheader_", 8) == 0)
2126     {
2127     s = read_header_name(name, 256, s);
2128     /* {-for-text-editors */
2129     if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2130     if (yield != NULL) *yield =
2131       (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor;
2132     }
2133
2134   /* Test for a variable's having a non-empty value. A non-existent variable
2135   causes an expansion failure. */
2136
2137   else
2138     {
2139     uschar *value = find_variable(name, TRUE, yield == NULL, NULL);
2140     if (value == NULL)
2141       {
2142       expand_string_message = (name[0] == 0)?
2143         string_sprintf("variable name omitted after \"def:\"") :
2144         string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2145       check_variable_error_message(name);
2146       return NULL;
2147       }
2148     if (yield != NULL) *yield = (value[0] != 0) == testfor;
2149     }
2150
2151   return s;
2152
2153
2154   /* first_delivery tests for first delivery attempt */
2155
2156   case ECOND_FIRST_DELIVERY:
2157   if (yield != NULL) *yield = deliver_firsttime == testfor;
2158   return s;
2159
2160
2161   /* queue_running tests for any process started by a queue runner */
2162
2163   case ECOND_QUEUE_RUNNING:
2164   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2165   return s;
2166
2167
2168   /* exists:  tests for file existence
2169        isip:  tests for any IP address
2170       isip4:  tests for an IPv4 address
2171       isip6:  tests for an IPv6 address
2172         pam:  does PAM authentication
2173      radius:  does RADIUS authentication
2174    ldapauth:  does LDAP authentication
2175     pwcheck:  does Cyrus SASL pwcheck authentication
2176   */
2177
2178   case ECOND_EXISTS:
2179   case ECOND_ISIP:
2180   case ECOND_ISIP4:
2181   case ECOND_ISIP6:
2182   case ECOND_PAM:
2183   case ECOND_RADIUS:
2184   case ECOND_LDAPAUTH:
2185   case ECOND_PWCHECK:
2186
2187   while (isspace(*s)) s++;
2188   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2189
2190   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2191   if (sub[0] == NULL) return NULL;
2192   /* {-for-text-editors */
2193   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2194
2195   if (yield == NULL) return s;   /* No need to run the test if skipping */
2196
2197   switch(cond_type)
2198     {
2199     case ECOND_EXISTS:
2200     if ((expand_forbid & RDO_EXISTS) != 0)
2201       {
2202       expand_string_message = US"File existence tests are not permitted";
2203       return NULL;
2204       }
2205     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2206     break;
2207
2208     case ECOND_ISIP:
2209     case ECOND_ISIP4:
2210     case ECOND_ISIP6:
2211     rc = string_is_ip_address(sub[0], NULL);
2212     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2213              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2214     break;
2215
2216     /* Various authentication tests - all optionally compiled */
2217
2218     case ECOND_PAM:
2219     #ifdef SUPPORT_PAM
2220     rc = auth_call_pam(sub[0], &expand_string_message);
2221     goto END_AUTH;
2222     #else
2223     goto COND_FAILED_NOT_COMPILED;
2224     #endif  /* SUPPORT_PAM */
2225
2226     case ECOND_RADIUS:
2227     #ifdef RADIUS_CONFIG_FILE
2228     rc = auth_call_radius(sub[0], &expand_string_message);
2229     goto END_AUTH;
2230     #else
2231     goto COND_FAILED_NOT_COMPILED;
2232     #endif  /* RADIUS_CONFIG_FILE */
2233
2234     case ECOND_LDAPAUTH:
2235     #ifdef LOOKUP_LDAP
2236       {
2237       /* Just to keep the interface the same */
2238       BOOL do_cache;
2239       int old_pool = store_pool;
2240       store_pool = POOL_SEARCH;
2241       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2242         &expand_string_message, &do_cache);
2243       store_pool = old_pool;
2244       }
2245     goto END_AUTH;
2246     #else
2247     goto COND_FAILED_NOT_COMPILED;
2248     #endif  /* LOOKUP_LDAP */
2249
2250     case ECOND_PWCHECK:
2251     #ifdef CYRUS_PWCHECK_SOCKET
2252     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2253     goto END_AUTH;
2254     #else
2255     goto COND_FAILED_NOT_COMPILED;
2256     #endif  /* CYRUS_PWCHECK_SOCKET */
2257
2258     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2259         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2260     END_AUTH:
2261     if (rc == ERROR || rc == DEFER) return NULL;
2262     *yield = (rc == OK) == testfor;
2263     #endif
2264     }
2265   return s;
2266
2267
2268   /* call ACL (in a conditional context).  Accept true, deny false.
2269   Defer is a forced-fail.  Anything set by message= goes to $value.
2270   Up to ten parameters are used; we use the braces round the name+args
2271   like the saslauthd condition does, to permit a variable number of args.
2272   See also the expansion-item version EITEM_ACL and the traditional
2273   acl modifier ACLC_ACL.
2274   Since the ACL may allocate new global variables, tell our caller to not
2275   reclaim memory.
2276   */
2277
2278   case ECOND_ACL:
2279     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2280     {
2281     uschar *user_msg;
2282     BOOL cond = FALSE;
2283     int size = 0;
2284     int ptr = 0;
2285
2286     while (isspace(*s)) s++;
2287     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2288
2289     switch(read_subs(sub, sizeof(sub)/sizeof(*sub), 1,
2290       &s, yield == NULL, TRUE, US"acl", resetok))
2291       {
2292       case 1: expand_string_message = US"too few arguments or bracketing "
2293         "error for acl";
2294       case 2:
2295       case 3: return NULL;
2296       }
2297
2298     *resetok = FALSE;
2299     if (yield != NULL) switch(eval_acl(sub, sizeof(sub)/sizeof(*sub), &user_msg))
2300         {
2301         case OK:
2302           cond = TRUE;
2303         case FAIL:
2304           lookup_value = NULL;
2305           if (user_msg)
2306             {
2307             lookup_value = string_cat(NULL, &size, &ptr, user_msg, Ustrlen(user_msg));
2308             lookup_value[ptr] = '\0';
2309             }
2310           *yield = cond == testfor;
2311           break;
2312
2313         case DEFER:
2314           expand_string_forcedfail = TRUE;
2315         default:
2316           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
2317           return NULL;
2318         }
2319     return s;
2320     }
2321
2322
2323   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2324
2325      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2326
2327   However, the last two are optional. That is why the whole set is enclosed
2328   in their own set of braces. */
2329
2330   case ECOND_SASLAUTHD:
2331   #ifndef CYRUS_SASLAUTHD_SOCKET
2332   goto COND_FAILED_NOT_COMPILED;
2333   #else
2334   while (isspace(*s)) s++;
2335   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2336   switch(read_subs(sub, 4, 2, &s, yield == NULL, TRUE, US"saslauthd", resetok))
2337     {
2338     case 1: expand_string_message = US"too few arguments or bracketing "
2339       "error for saslauthd";
2340     case 2:
2341     case 3: return NULL;
2342     }
2343   if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2344   if (yield != NULL)
2345     {
2346     int rc;
2347     rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2348       &expand_string_message);
2349     if (rc == ERROR || rc == DEFER) return NULL;
2350     *yield = (rc == OK) == testfor;
2351     }
2352   return s;
2353   #endif /* CYRUS_SASLAUTHD_SOCKET */
2354
2355
2356   /* symbolic operators for numeric and string comparison, and a number of
2357   other operators, all requiring two arguments.
2358
2359   crypteq:           encrypts plaintext and compares against an encrypted text,
2360                        using crypt(), crypt16(), MD5 or SHA-1
2361   inlist/inlisti:    checks if first argument is in the list of the second
2362   match:             does a regular expression match and sets up the numerical
2363                        variables if it succeeds
2364   match_address:     matches in an address list
2365   match_domain:      matches in a domain list
2366   match_ip:          matches a host list that is restricted to IP addresses
2367   match_local_part:  matches in a local part list
2368   */
2369
2370   case ECOND_MATCH_ADDRESS:
2371   case ECOND_MATCH_DOMAIN:
2372   case ECOND_MATCH_IP:
2373   case ECOND_MATCH_LOCAL_PART:
2374 #ifndef EXPAND_LISTMATCH_RHS
2375     sub2_honour_dollar = FALSE;
2376 #endif
2377     /* FALLTHROUGH */
2378
2379   case ECOND_CRYPTEQ:
2380   case ECOND_INLIST:
2381   case ECOND_INLISTI:
2382   case ECOND_MATCH:
2383
2384   case ECOND_NUM_L:     /* Numerical comparisons */
2385   case ECOND_NUM_LE:
2386   case ECOND_NUM_E:
2387   case ECOND_NUM_EE:
2388   case ECOND_NUM_G:
2389   case ECOND_NUM_GE:
2390
2391   case ECOND_STR_LT:    /* String comparisons */
2392   case ECOND_STR_LTI:
2393   case ECOND_STR_LE:
2394   case ECOND_STR_LEI:
2395   case ECOND_STR_EQ:
2396   case ECOND_STR_EQI:
2397   case ECOND_STR_GT:
2398   case ECOND_STR_GTI:
2399   case ECOND_STR_GE:
2400   case ECOND_STR_GEI:
2401
2402   for (i = 0; i < 2; i++)
2403     {
2404     /* Sometimes, we don't expand substrings; too many insecure configurations
2405     created using match_address{}{} and friends, where the second param
2406     includes information from untrustworthy sources. */
2407     BOOL honour_dollar = TRUE;
2408     if ((i > 0) && !sub2_honour_dollar)
2409       honour_dollar = FALSE;
2410
2411     while (isspace(*s)) s++;
2412     if (*s != '{')
2413       {
2414       if (i == 0) goto COND_FAILED_CURLY_START;
2415       expand_string_message = string_sprintf("missing 2nd string in {} "
2416         "after \"%s\"", name);
2417       return NULL;
2418       }
2419     sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2420         honour_dollar, resetok);
2421     if (sub[i] == NULL) return NULL;
2422     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2423
2424     /* Convert to numerical if required; we know that the names of all the
2425     conditions that compare numbers do not start with a letter. This just saves
2426     checking for them individually. */
2427
2428     if (!isalpha(name[0]) && yield != NULL)
2429       {
2430       if (sub[i][0] == 0)
2431         {
2432         num[i] = 0;
2433         DEBUG(D_expand)
2434           debug_printf("empty string cast to zero for numerical comparison\n");
2435         }
2436       else
2437         {
2438         num[i] = expand_string_integer(sub[i], FALSE);
2439         if (expand_string_message != NULL) return NULL;
2440         }
2441       }
2442     }
2443
2444   /* Result not required */
2445
2446   if (yield == NULL) return s;
2447
2448   /* Do an appropriate comparison */
2449
2450   switch(cond_type)
2451     {
2452     case ECOND_NUM_E:
2453     case ECOND_NUM_EE:
2454     tempcond = (num[0] == num[1]);
2455     break;
2456
2457     case ECOND_NUM_G:
2458     tempcond = (num[0] > num[1]);
2459     break;
2460
2461     case ECOND_NUM_GE:
2462     tempcond = (num[0] >= num[1]);
2463     break;
2464
2465     case ECOND_NUM_L:
2466     tempcond = (num[0] < num[1]);
2467     break;
2468
2469     case ECOND_NUM_LE:
2470     tempcond = (num[0] <= num[1]);
2471     break;
2472
2473     case ECOND_STR_LT:
2474     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2475     break;
2476
2477     case ECOND_STR_LTI:
2478     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2479     break;
2480
2481     case ECOND_STR_LE:
2482     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2483     break;
2484
2485     case ECOND_STR_LEI:
2486     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2487     break;
2488
2489     case ECOND_STR_EQ:
2490     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2491     break;
2492
2493     case ECOND_STR_EQI:
2494     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2495     break;
2496
2497     case ECOND_STR_GT:
2498     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2499     break;
2500
2501     case ECOND_STR_GTI:
2502     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2503     break;
2504
2505     case ECOND_STR_GE:
2506     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2507     break;
2508
2509     case ECOND_STR_GEI:
2510     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2511     break;
2512
2513     case ECOND_MATCH:   /* Regular expression match */
2514     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2515       NULL);
2516     if (re == NULL)
2517       {
2518       expand_string_message = string_sprintf("regular expression error in "
2519         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2520       return NULL;
2521       }
2522     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2523     break;
2524
2525     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2526     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2527     goto MATCHED_SOMETHING;
2528
2529     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2530     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2531       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2532     goto MATCHED_SOMETHING;
2533
2534     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2535     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2536       {
2537       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2538         sub[0]);
2539       return NULL;
2540       }
2541     else
2542       {
2543       unsigned int *nullcache = NULL;
2544       check_host_block cb;
2545
2546       cb.host_name = US"";
2547       cb.host_address = sub[0];
2548
2549       /* If the host address starts off ::ffff: it is an IPv6 address in
2550       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2551       addresses. */
2552
2553       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2554         cb.host_address + 7 : cb.host_address;
2555
2556       rc = match_check_list(
2557              &sub[1],                   /* the list */
2558              0,                         /* separator character */
2559              &hostlist_anchor,          /* anchor pointer */
2560              &nullcache,                /* cache pointer */
2561              check_host,                /* function for testing */
2562              &cb,                       /* argument for function */
2563              MCL_HOST,                  /* type of check */
2564              sub[0],                    /* text for debugging */
2565              NULL);                     /* where to pass back data */
2566       }
2567     goto MATCHED_SOMETHING;
2568
2569     case ECOND_MATCH_LOCAL_PART:
2570     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2571       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2572     /* Fall through */
2573     /* VVVVVVVVVVVV */
2574     MATCHED_SOMETHING:
2575     switch(rc)
2576       {
2577       case OK:
2578       tempcond = TRUE;
2579       break;
2580
2581       case FAIL:
2582       tempcond = FALSE;
2583       break;
2584
2585       case DEFER:
2586       expand_string_message = string_sprintf("unable to complete match "
2587         "against \"%s\": %s", sub[1], search_error_message);
2588       return NULL;
2589       }
2590
2591     break;
2592
2593     /* Various "encrypted" comparisons. If the second string starts with
2594     "{" then an encryption type is given. Default to crypt() or crypt16()
2595     (build-time choice). */
2596     /* }-for-text-editors */
2597
2598     case ECOND_CRYPTEQ:
2599     #ifndef SUPPORT_CRYPTEQ
2600     goto COND_FAILED_NOT_COMPILED;
2601     #else
2602     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2603       {
2604       int sublen = Ustrlen(sub[1]+5);
2605       md5 base;
2606       uschar digest[16];
2607
2608       md5_start(&base);
2609       md5_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
2610
2611       /* If the length that we are comparing against is 24, the MD5 digest
2612       is expressed as a base64 string. This is the way LDAP does it. However,
2613       some other software uses a straightforward hex representation. We assume
2614       this if the length is 32. Other lengths fail. */
2615
2616       if (sublen == 24)
2617         {
2618         uschar *coded = auth_b64encode((uschar *)digest, 16);
2619         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2620           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2621         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2622         }
2623       else if (sublen == 32)
2624         {
2625         int i;
2626         uschar coded[36];
2627         for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2628         coded[32] = 0;
2629         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2630           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2631         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2632         }
2633       else
2634         {
2635         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2636           "fail\n  crypted=%s\n", sub[1]+5);
2637         tempcond = FALSE;
2638         }
2639       }
2640
2641     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2642       {
2643       int sublen = Ustrlen(sub[1]+6);
2644       sha1 base;
2645       uschar digest[20];
2646
2647       sha1_start(&base);
2648       sha1_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
2649
2650       /* If the length that we are comparing against is 28, assume the SHA1
2651       digest is expressed as a base64 string. If the length is 40, assume a
2652       straightforward hex representation. Other lengths fail. */
2653
2654       if (sublen == 28)
2655         {
2656         uschar *coded = auth_b64encode((uschar *)digest, 20);
2657         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2658           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2659         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2660         }
2661       else if (sublen == 40)
2662         {
2663         int i;
2664         uschar coded[44];
2665         for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2666         coded[40] = 0;
2667         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2668           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2669         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2670         }
2671       else
2672         {
2673         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2674           "fail\n  crypted=%s\n", sub[1]+6);
2675         tempcond = FALSE;
2676         }
2677       }
2678
2679     else   /* {crypt} or {crypt16} and non-{ at start */
2680            /* }-for-text-editors */
2681       {
2682       int which = 0;
2683       uschar *coded;
2684
2685       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2686         {
2687         sub[1] += 7;
2688         which = 1;
2689         }
2690       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2691         {
2692         sub[1] += 9;
2693         which = 2;
2694         }
2695       else if (sub[1][0] == '{')                /* }-for-text-editors */
2696         {
2697         expand_string_message = string_sprintf("unknown encryption mechanism "
2698           "in \"%s\"", sub[1]);
2699         return NULL;
2700         }
2701
2702       switch(which)
2703         {
2704         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2705         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2706         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2707         }
2708
2709       #define STR(s) # s
2710       #define XSTR(s) STR(s)
2711       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2712         "  subject=%s\n  crypted=%s\n",
2713         (which == 0)? XSTR(DEFAULT_CRYPT) : (which == 1)? "crypt" : "crypt16",
2714         coded, sub[1]);
2715       #undef STR
2716       #undef XSTR
2717
2718       /* If the encrypted string contains fewer than two characters (for the
2719       salt), force failure. Otherwise we get false positives: with an empty
2720       string the yield of crypt() is an empty string! */
2721
2722       tempcond = (Ustrlen(sub[1]) < 2)? FALSE :
2723         (Ustrcmp(coded, sub[1]) == 0);
2724       }
2725     break;
2726     #endif  /* SUPPORT_CRYPTEQ */
2727
2728     case ECOND_INLIST:
2729     case ECOND_INLISTI:
2730       {
2731       int sep = 0;
2732       uschar *save_iterate_item = iterate_item;
2733       int (*compare)(const uschar *, const uschar *);
2734
2735       tempcond = FALSE;
2736       if (cond_type == ECOND_INLISTI)
2737         compare = strcmpic;
2738       else
2739         compare = (int (*)(const uschar *, const uschar *)) strcmp;
2740
2741       while ((iterate_item = string_nextinlist(&sub[1], &sep, NULL, 0)) != NULL)
2742         if (compare(sub[0], iterate_item) == 0)
2743           {
2744           tempcond = TRUE;
2745           break;
2746           }
2747       iterate_item = save_iterate_item;
2748       }
2749
2750     }   /* Switch for comparison conditions */
2751
2752   *yield = tempcond == testfor;
2753   return s;    /* End of comparison conditions */
2754
2755
2756   /* and/or: computes logical and/or of several conditions */
2757
2758   case ECOND_AND:
2759   case ECOND_OR:
2760   subcondptr = (yield == NULL)? NULL : &tempcond;
2761   combined_cond = (cond_type == ECOND_AND);
2762
2763   while (isspace(*s)) s++;
2764   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2765
2766   for (;;)
2767     {
2768     while (isspace(*s)) s++;
2769     /* {-for-text-editors */
2770     if (*s == '}') break;
2771     if (*s != '{')                                      /* }-for-text-editors */
2772       {
2773       expand_string_message = string_sprintf("each subcondition "
2774         "inside an \"%s{...}\" condition must be in its own {}", name);
2775       return NULL;
2776       }
2777
2778     if (!(s = eval_condition(s+1, resetok, subcondptr)))
2779       {
2780       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
2781         expand_string_message, name);
2782       return NULL;
2783       }
2784     while (isspace(*s)) s++;
2785
2786     /* {-for-text-editors */
2787     if (*s++ != '}')
2788       {
2789       /* {-for-text-editors */
2790       expand_string_message = string_sprintf("missing } at end of condition "
2791         "inside \"%s\" group", name);
2792       return NULL;
2793       }
2794
2795     if (yield != NULL)
2796       {
2797       if (cond_type == ECOND_AND)
2798         {
2799         combined_cond &= tempcond;
2800         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
2801         }                                       /* evaluate any more */
2802       else
2803         {
2804         combined_cond |= tempcond;
2805         if (combined_cond) subcondptr = NULL;   /* once true, don't */
2806         }                                       /* evaluate any more */
2807       }
2808     }
2809
2810   if (yield != NULL) *yield = (combined_cond == testfor);
2811   return ++s;
2812
2813
2814   /* forall/forany: iterates a condition with different values */
2815
2816   case ECOND_FORALL:
2817   case ECOND_FORANY:
2818     {
2819     int sep = 0;
2820     uschar *save_iterate_item = iterate_item;
2821
2822     while (isspace(*s)) s++;
2823     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2824     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
2825     if (sub[0] == NULL) return NULL;
2826     /* {-for-text-editors */
2827     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2828
2829     while (isspace(*s)) s++;
2830     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2831
2832     sub[1] = s;
2833
2834     /* Call eval_condition once, with result discarded (as if scanning a
2835     "false" part). This allows us to find the end of the condition, because if
2836     the list it empty, we won't actually evaluate the condition for real. */
2837
2838     if (!(s = eval_condition(sub[1], resetok, NULL)))
2839       {
2840       expand_string_message = string_sprintf("%s inside \"%s\" condition",
2841         expand_string_message, name);
2842       return NULL;
2843       }
2844     while (isspace(*s)) s++;
2845
2846     /* {-for-text-editors */
2847     if (*s++ != '}')
2848       {
2849       /* {-for-text-editors */
2850       expand_string_message = string_sprintf("missing } at end of condition "
2851         "inside \"%s\"", name);
2852       return NULL;
2853       }
2854
2855     if (yield != NULL) *yield = !testfor;
2856     while ((iterate_item = string_nextinlist(&sub[0], &sep, NULL, 0)) != NULL)
2857       {
2858       DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
2859       if (!eval_condition(sub[1], resetok, &tempcond))
2860         {
2861         expand_string_message = string_sprintf("%s inside \"%s\" condition",
2862           expand_string_message, name);
2863         iterate_item = save_iterate_item;
2864         return NULL;
2865         }
2866       DEBUG(D_expand) debug_printf("%s: condition evaluated to %s\n", name,
2867         tempcond? "true":"false");
2868
2869       if (yield != NULL) *yield = (tempcond == testfor);
2870       if (tempcond == (cond_type == ECOND_FORANY)) break;
2871       }
2872
2873     iterate_item = save_iterate_item;
2874     return s;
2875     }
2876
2877
2878   /* The bool{} expansion condition maps a string to boolean.
2879   The values supported should match those supported by the ACL condition
2880   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
2881   of true/false.  Note that Router "condition" rules have a different
2882   interpretation, where general data can be used and only a few values
2883   map to FALSE.
2884   Note that readconf.c boolean matching, for boolean configuration options,
2885   only matches true/yes/false/no.
2886   The bool_lax{} condition matches the Router logic, which is much more
2887   liberal. */
2888   case ECOND_BOOL:
2889   case ECOND_BOOL_LAX:
2890     {
2891     uschar *sub_arg[1];
2892     uschar *t, *t2;
2893     uschar *ourname;
2894     size_t len;
2895     BOOL boolvalue = FALSE;
2896     while (isspace(*s)) s++;
2897     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2898     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
2899     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
2900       {
2901       case 1: expand_string_message = string_sprintf(
2902                   "too few arguments or bracketing error for %s",
2903                   ourname);
2904       /*FALLTHROUGH*/
2905       case 2:
2906       case 3: return NULL;
2907       }
2908     t = sub_arg[0];
2909     while (isspace(*t)) t++;
2910     len = Ustrlen(t);
2911     if (len)
2912       {
2913       /* trailing whitespace: seems like a good idea to ignore it too */
2914       t2 = t + len - 1;
2915       while (isspace(*t2)) t2--;
2916       if (t2 != (t + len))
2917         {
2918         *++t2 = '\0';
2919         len = t2 - t;
2920         }
2921       }
2922     DEBUG(D_expand)
2923       debug_printf("considering %s: %s\n", ourname, len ? t : US"<empty>");
2924     /* logic for the lax case from expand_check_condition(), which also does
2925     expands, and the logic is both short and stable enough that there should
2926     be no maintenance burden from replicating it. */
2927     if (len == 0)
2928       boolvalue = FALSE;
2929     else if (*t == '-'
2930              ? Ustrspn(t+1, "0123456789") == len-1
2931              : Ustrspn(t,   "0123456789") == len)
2932       {
2933       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
2934       /* expand_check_condition only does a literal string "0" check */
2935       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
2936         boolvalue = TRUE;
2937       }
2938     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
2939       boolvalue = TRUE;
2940     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
2941       boolvalue = FALSE;
2942     else if (cond_type == ECOND_BOOL_LAX)
2943       boolvalue = TRUE;
2944     else
2945       {
2946       expand_string_message = string_sprintf("unrecognised boolean "
2947        "value \"%s\"", t);
2948       return NULL;
2949       }
2950     if (yield != NULL) *yield = (boolvalue == testfor);
2951     return s;
2952     }
2953
2954   /* Unknown condition */
2955
2956   default:
2957   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
2958   return NULL;
2959   }   /* End switch on condition type */
2960
2961 /* Missing braces at start and end of data */
2962
2963 COND_FAILED_CURLY_START:
2964 expand_string_message = string_sprintf("missing { after \"%s\"", name);
2965 return NULL;
2966
2967 COND_FAILED_CURLY_END:
2968 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
2969   name);
2970 return NULL;
2971
2972 /* A condition requires code that is not compiled */
2973
2974 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
2975     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
2976     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
2977 COND_FAILED_NOT_COMPILED:
2978 expand_string_message = string_sprintf("support for \"%s\" not compiled",
2979   name);
2980 return NULL;
2981 #endif
2982 }
2983
2984
2985
2986
2987 /*************************************************
2988 *          Save numerical variables              *
2989 *************************************************/
2990
2991 /* This function is called from items such as "if" that want to preserve and
2992 restore the numbered variables.
2993
2994 Arguments:
2995   save_expand_string    points to an array of pointers to set
2996   save_expand_nlength   points to an array of ints for the lengths
2997
2998 Returns:                the value of expand max to save
2999 */
3000
3001 static int
3002 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3003 {
3004 int i;
3005 for (i = 0; i <= expand_nmax; i++)
3006   {
3007   save_expand_nstring[i] = expand_nstring[i];
3008   save_expand_nlength[i] = expand_nlength[i];
3009   }
3010 return expand_nmax;
3011 }
3012
3013
3014
3015 /*************************************************
3016 *           Restore numerical variables          *
3017 *************************************************/
3018
3019 /* This function restored saved values of numerical strings.
3020
3021 Arguments:
3022   save_expand_nmax      the number of strings to restore
3023   save_expand_string    points to an array of pointers
3024   save_expand_nlength   points to an array of ints
3025
3026 Returns:                nothing
3027 */
3028
3029 static void
3030 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3031   int *save_expand_nlength)
3032 {
3033 int i;
3034 expand_nmax = save_expand_nmax;
3035 for (i = 0; i <= expand_nmax; i++)
3036   {
3037   expand_nstring[i] = save_expand_nstring[i];
3038   expand_nlength[i] = save_expand_nlength[i];
3039   }
3040 }
3041
3042
3043
3044
3045
3046 /*************************************************
3047 *            Handle yes/no substrings            *
3048 *************************************************/
3049
3050 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3051 alternative substrings that depend on whether or not the condition was true,
3052 or the lookup or extraction succeeded. The substrings always have to be
3053 expanded, to check their syntax, but "skipping" is set when the result is not
3054 needed - this avoids unnecessary nested lookups.
3055
3056 Arguments:
3057   skipping       TRUE if we were skipping when this item was reached
3058   yes            TRUE if the first string is to be used, else use the second
3059   save_lookup    a value to put back into lookup_value before the 2nd expansion
3060   sptr           points to the input string pointer
3061   yieldptr       points to the output string pointer
3062   sizeptr        points to the output string size
3063   ptrptr         points to the output string pointer
3064   type           "lookup" or "if" or "extract" or "run", for error message
3065   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3066                 the store.
3067
3068 Returns:         0 OK; lookup_value has been reset to save_lookup
3069                  1 expansion failed
3070                  2 expansion failed because of bracketing error
3071 */
3072
3073 static int
3074 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, uschar **sptr,
3075   uschar **yieldptr, int *sizeptr, int *ptrptr, uschar *type, BOOL *resetok)
3076 {
3077 int rc = 0;
3078 uschar *s = *sptr;    /* Local value */
3079 uschar *sub1, *sub2;
3080
3081 /* If there are no following strings, we substitute the contents of $value for
3082 lookups and for extractions in the success case. For the ${if item, the string
3083 "true" is substituted. In the fail case, nothing is substituted for all three
3084 items. */
3085
3086 while (isspace(*s)) s++;
3087 if (*s == '}')
3088   {
3089   if (type[0] == 'i')
3090     {
3091     if (yes) *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, US"true", 4);
3092     }
3093   else
3094     {
3095     if (yes && lookup_value != NULL)
3096       *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value,
3097         Ustrlen(lookup_value));
3098     lookup_value = save_lookup;
3099     }
3100   s++;
3101   goto RETURN;
3102   }
3103
3104 /* The first following string must be braced. */
3105
3106 if (*s++ != '{') goto FAILED_CURLY;
3107
3108 /* Expand the first substring. Forced failures are noticed only if we actually
3109 want this string. Set skipping in the call in the fail case (this will always
3110 be the case if we were already skipping). */
3111
3112 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3113 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
3114 expand_string_forcedfail = FALSE;
3115 if (*s++ != '}') goto FAILED_CURLY;
3116
3117 /* If we want the first string, add it to the output */
3118
3119 if (yes)
3120   *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub1, Ustrlen(sub1));
3121
3122 /* If this is called from a lookup or an extract, we want to restore $value to
3123 what it was at the start of the item, so that it has this value during the
3124 second string expansion. For the call from "if" or "run" to this function,
3125 save_lookup is set to lookup_value, so that this statement does nothing. */
3126
3127 lookup_value = save_lookup;
3128
3129 /* There now follows either another substring, or "fail", or nothing. This
3130 time, forced failures are noticed only if we want the second string. We must
3131 set skipping in the nested call if we don't want this string, or if we were
3132 already skipping. */
3133
3134 while (isspace(*s)) s++;
3135 if (*s == '{')
3136   {
3137   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3138   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
3139   expand_string_forcedfail = FALSE;
3140   if (*s++ != '}') goto FAILED_CURLY;
3141
3142   /* If we want the second string, add it to the output */
3143
3144   if (!yes)
3145     *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub2, Ustrlen(sub2));
3146   }
3147
3148 /* If there is no second string, but the word "fail" is present when the use of
3149 the second string is wanted, set a flag indicating it was a forced failure
3150 rather than a syntactic error. Swallow the terminating } in case this is nested
3151 inside another lookup or if or extract. */
3152
3153 else if (*s != '}')
3154   {
3155   uschar name[256];
3156   s = read_name(name, sizeof(name), s, US"_");
3157   if (Ustrcmp(name, "fail") == 0)
3158     {
3159     if (!yes && !skipping)
3160       {
3161       while (isspace(*s)) s++;
3162       if (*s++ != '}') goto FAILED_CURLY;
3163       expand_string_message =
3164         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3165       expand_string_forcedfail = TRUE;
3166       goto FAILED;
3167       }
3168     }
3169   else
3170     {
3171     expand_string_message =
3172       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3173     goto FAILED;
3174     }
3175   }
3176
3177 /* All we have to do now is to check on the final closing brace. */
3178
3179 while (isspace(*s)) s++;
3180 if (*s++ == '}') goto RETURN;
3181
3182 /* Get here if there is a bracketing failure */
3183
3184 FAILED_CURLY:
3185 rc++;
3186
3187 /* Get here for other failures */
3188
3189 FAILED:
3190 rc++;
3191
3192 /* Update the input pointer value before returning */
3193
3194 RETURN:
3195 *sptr = s;
3196 return rc;
3197 }
3198
3199
3200
3201
3202 /*************************************************
3203 *    Handle MD5 or SHA-1 computation for HMAC    *
3204 *************************************************/
3205
3206 /* These are some wrapping functions that enable the HMAC code to be a bit
3207 cleaner. A good compiler will spot the tail recursion.
3208
3209 Arguments:
3210   type         HMAC_MD5 or HMAC_SHA1
3211   remaining    are as for the cryptographic hash functions
3212
3213 Returns:       nothing
3214 */
3215
3216 static void
3217 chash_start(int type, void *base)
3218 {
3219 if (type == HMAC_MD5)
3220   md5_start((md5 *)base);
3221 else
3222   sha1_start((sha1 *)base);
3223 }
3224
3225 static void
3226 chash_mid(int type, void *base, uschar *string)
3227 {
3228 if (type == HMAC_MD5)
3229   md5_mid((md5 *)base, string);
3230 else
3231   sha1_mid((sha1 *)base, string);
3232 }
3233
3234 static void
3235 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3236 {
3237 if (type == HMAC_MD5)
3238   md5_end((md5 *)base, string, length, digest);
3239 else
3240   sha1_end((sha1 *)base, string, length, digest);
3241 }
3242
3243
3244
3245
3246
3247 /********************************************************
3248 * prvs: Get last three digits of days since Jan 1, 1970 *
3249 ********************************************************/
3250
3251 /* This is needed to implement the "prvs" BATV reverse
3252    path signing scheme
3253
3254 Argument: integer "days" offset to add or substract to
3255           or from the current number of days.
3256
3257 Returns:  pointer to string containing the last three
3258           digits of the number of days since Jan 1, 1970,
3259           modified by the offset argument, NULL if there
3260           was an error in the conversion.
3261
3262 */
3263
3264 static uschar *
3265 prvs_daystamp(int day_offset)
3266 {
3267 uschar *days = store_get(32);                /* Need at least 24 for cases */
3268 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3269   (time(NULL) + day_offset*86400)/86400);
3270 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3271 }
3272
3273
3274
3275 /********************************************************
3276 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3277 ********************************************************/
3278
3279 /* This is needed to implement the "prvs" BATV reverse
3280    path signing scheme
3281
3282 Arguments:
3283   address RFC2821 Address to use
3284       key The key to use (must be less than 64 characters
3285           in size)
3286   key_num Single-digit key number to use. Defaults to
3287           '0' when NULL.
3288
3289 Returns:  pointer to string containing the first three
3290           bytes of the final hash in hex format, NULL if
3291           there was an error in the process.
3292 */
3293
3294 static uschar *
3295 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3296 {
3297 uschar *hash_source, *p;
3298 int size = 0,offset = 0,i;
3299 sha1 sha1_base;
3300 void *use_base = &sha1_base;
3301 uschar innerhash[20];
3302 uschar finalhash[20];
3303 uschar innerkey[64];
3304 uschar outerkey[64];
3305 uschar *finalhash_hex = store_get(40);
3306
3307 if (key_num == NULL)
3308   key_num = US"0";
3309
3310 if (Ustrlen(key) > 64)
3311   return NULL;
3312
3313 hash_source = string_cat(NULL,&size,&offset,key_num,1);
3314 string_cat(hash_source,&size,&offset,daystamp,3);
3315 string_cat(hash_source,&size,&offset,address,Ustrlen(address));
3316 hash_source[offset] = '\0';
3317
3318 DEBUG(D_expand) debug_printf("prvs: hash source is '%s'\n", hash_source);
3319
3320 memset(innerkey, 0x36, 64);
3321 memset(outerkey, 0x5c, 64);
3322
3323 for (i = 0; i < Ustrlen(key); i++)
3324   {
3325   innerkey[i] ^= key[i];
3326   outerkey[i] ^= key[i];
3327   }
3328
3329 chash_start(HMAC_SHA1, use_base);
3330 chash_mid(HMAC_SHA1, use_base, innerkey);
3331 chash_end(HMAC_SHA1, use_base, hash_source, offset, innerhash);
3332
3333 chash_start(HMAC_SHA1, use_base);
3334 chash_mid(HMAC_SHA1, use_base, outerkey);
3335 chash_end(HMAC_SHA1, use_base, innerhash, 20, finalhash);
3336
3337 p = finalhash_hex;
3338 for (i = 0; i < 3; i++)
3339   {
3340   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3341   *p++ = hex_digits[finalhash[i] & 0x0f];
3342   }
3343 *p = '\0';
3344
3345 return finalhash_hex;
3346 }
3347
3348
3349
3350
3351 /*************************************************
3352 *        Join a file onto the output string      *
3353 *************************************************/
3354
3355 /* This is used for readfile and after a run expansion. It joins the contents
3356 of a file onto the output string, globally replacing newlines with a given
3357 string (optionally). The file is closed at the end.
3358
3359 Arguments:
3360   f            the FILE
3361   yield        pointer to the expandable string
3362   sizep        pointer to the current size
3363   ptrp         pointer to the current position
3364   eol          newline replacement string, or NULL
3365
3366 Returns:       new value of string pointer
3367 */
3368
3369 static uschar *
3370 cat_file(FILE *f, uschar *yield, int *sizep, int *ptrp, uschar *eol)
3371 {
3372 int eollen;
3373 uschar buffer[1024];
3374
3375 eollen = (eol == NULL)? 0 : Ustrlen(eol);
3376
3377 while (Ufgets(buffer, sizeof(buffer), f) != NULL)
3378   {
3379   int len = Ustrlen(buffer);
3380   if (eol != NULL && buffer[len-1] == '\n') len--;
3381   yield = string_cat(yield, sizep, ptrp, buffer, len);
3382   if (buffer[len] != 0)
3383     yield = string_cat(yield, sizep, ptrp, eol, eollen);
3384   }
3385
3386 if (yield != NULL) yield[*ptrp] = 0;
3387
3388 return yield;
3389 }
3390
3391
3392
3393
3394 /*************************************************
3395 *          Evaluate numeric expression           *
3396 *************************************************/
3397
3398 /* This is a set of mutually recursive functions that evaluate an arithmetic
3399 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3400 these functions that is called from elsewhere is eval_expr, whose interface is:
3401
3402 Arguments:
3403   sptr        pointer to the pointer to the string - gets updated
3404   decimal     TRUE if numbers are to be assumed decimal
3405   error       pointer to where to put an error message - must be NULL on input
3406   endket      TRUE if ')' must terminate - FALSE for external call
3407
3408 Returns:      on success: the value of the expression, with *error still NULL
3409               on failure: an undefined value, with *error = a message
3410 */
3411
3412 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3413
3414
3415 static int_eximarith_t
3416 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3417 {
3418 uschar *s = *sptr;
3419 int_eximarith_t x = eval_op_or(&s, decimal, error);
3420 if (*error == NULL)
3421   {
3422   if (endket)
3423     {
3424     if (*s != ')')
3425       *error = US"expecting closing parenthesis";
3426     else
3427       while (isspace(*(++s)));
3428     }
3429   else if (*s != 0) *error = US"expecting operator";
3430   }
3431 *sptr = s;
3432 return x;
3433 }
3434
3435
3436 static int_eximarith_t
3437 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3438 {
3439 register int c;
3440 int_eximarith_t n;
3441 uschar *s = *sptr;
3442 while (isspace(*s)) s++;
3443 c = *s;
3444 if (isdigit(c))
3445   {
3446   int count;
3447   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3448   s += count;
3449   switch (tolower(*s))
3450     {
3451     default: break;
3452     case 'k': n *= 1024; s++; break;
3453     case 'm': n *= 1024*1024; s++; break;
3454     case 'g': n *= 1024*1024*1024; s++; break;
3455     }
3456   while (isspace (*s)) s++;
3457   }
3458 else if (c == '(')
3459   {
3460   s++;
3461   n = eval_expr(&s, decimal, error, 1);
3462   }
3463 else
3464   {
3465   *error = US"expecting number or opening parenthesis";
3466   n = 0;
3467   }
3468 *sptr = s;
3469 return n;
3470 }
3471
3472
3473 static int_eximarith_t
3474 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3475 {
3476 uschar *s = *sptr;
3477 int_eximarith_t x;
3478 while (isspace(*s)) s++;
3479 if (*s == '+' || *s == '-' || *s == '~')
3480   {
3481   int op = *s++;
3482   x = eval_op_unary(&s, decimal, error);
3483   if (op == '-') x = -x;
3484     else if (op == '~') x = ~x;
3485   }
3486 else
3487   {
3488   x = eval_number(&s, decimal, error);
3489   }
3490 *sptr = s;
3491 return x;
3492 }
3493
3494
3495 static int_eximarith_t
3496 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3497 {
3498 uschar *s = *sptr;
3499 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3500 if (*error == NULL)
3501   {
3502   while (*s == '*' || *s == '/' || *s == '%')
3503     {
3504     int op = *s++;
3505     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3506     if (*error != NULL) break;
3507     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3508      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3509      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3510      * -N*M is INT_MIN will yielf INT_MIN.
3511      * Since we don't support floating point, this is somewhat simpler.
3512      * Ideally, we'd return an error, but since we overflow for all other
3513      * arithmetic, consistency suggests otherwise, but what's the correct value
3514      * to use?  There is none.
3515      * The C standard guarantees overflow for unsigned arithmetic but signed
3516      * overflow invokes undefined behaviour; in practice, this is overflow
3517      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3518      * that long/longlong larger than int are available, or we could just work
3519      * with larger types.  We should consider whether to guarantee 32bit eval
3520      * and 64-bit working variables, with errors returned.  For now ...
3521      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3522      * can just let the other invalid results occur otherwise, as they have
3523      * until now.  For this one case, we can coerce.
3524      */
3525     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3526       {
3527       DEBUG(D_expand)
3528         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3529             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3530       x = EXIM_ARITH_MAX;
3531       continue;
3532       }
3533     if (op == '*')
3534       x *= y;
3535     else
3536       {
3537       if (y == 0)
3538         {
3539         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3540         x = 0;
3541         break;
3542         }
3543       if (op == '/')
3544         x /= y;
3545       else
3546         x %= y;
3547       }
3548     }
3549   }
3550 *sptr = s;
3551 return x;
3552 }
3553
3554
3555 static int_eximarith_t
3556 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3557 {
3558 uschar *s = *sptr;
3559 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3560 if (*error == NULL)
3561   {
3562   while (*s == '+' || *s == '-')
3563     {
3564     int op = *s++;
3565     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3566     if (*error != NULL) break;
3567     if (op == '+') x += y; else x -= y;
3568     }
3569   }
3570 *sptr = s;
3571 return x;
3572 }
3573
3574
3575 static int_eximarith_t
3576 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3577 {
3578 uschar *s = *sptr;
3579 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3580 if (*error == NULL)
3581   {
3582   while ((*s == '<' || *s == '>') && s[1] == s[0])
3583     {
3584     int_eximarith_t y;
3585     int op = *s++;
3586     s++;
3587     y = eval_op_sum(&s, decimal, error);
3588     if (*error != NULL) break;
3589     if (op == '<') x <<= y; else x >>= y;
3590     }
3591   }
3592 *sptr = s;
3593 return x;
3594 }
3595
3596
3597 static int_eximarith_t
3598 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3599 {
3600 uschar *s = *sptr;
3601 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3602 if (*error == NULL)
3603   {
3604   while (*s == '&')
3605     {
3606     int_eximarith_t y;
3607     s++;
3608     y = eval_op_shift(&s, decimal, error);
3609     if (*error != NULL) break;
3610     x &= y;
3611     }
3612   }
3613 *sptr = s;
3614 return x;
3615 }
3616
3617
3618 static int_eximarith_t
3619 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3620 {
3621 uschar *s = *sptr;
3622 int_eximarith_t x = eval_op_and(&s, decimal, error);
3623 if (*error == NULL)
3624   {
3625   while (*s == '^')
3626     {
3627     int_eximarith_t y;
3628     s++;
3629     y = eval_op_and(&s, decimal, error);
3630     if (*error != NULL) break;
3631     x ^= y;
3632     }
3633   }
3634 *sptr = s;
3635 return x;
3636 }
3637
3638
3639 static int_eximarith_t
3640 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3641 {
3642 uschar *s = *sptr;
3643 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3644 if (*error == NULL)
3645   {
3646   while (*s == '|')
3647     {
3648     int_eximarith_t y;
3649     s++;
3650     y = eval_op_xor(&s, decimal, error);
3651     if (*error != NULL) break;
3652     x |= y;
3653     }
3654   }
3655 *sptr = s;
3656 return x;
3657 }
3658
3659
3660
3661 /*************************************************
3662 *                 Expand string                  *
3663 *************************************************/
3664
3665 /* Returns either an unchanged string, or the expanded string in stacking pool
3666 store. Interpreted sequences are:
3667
3668    \...                    normal escaping rules
3669    $name                   substitutes the variable
3670    ${name}                 ditto
3671    ${op:string}            operates on the expanded string value
3672    ${item{arg1}{arg2}...}  expands the args and then does the business
3673                              some literal args are not enclosed in {}
3674
3675 There are now far too many operators and item types to make it worth listing
3676 them here in detail any more.
3677
3678 We use an internal routine recursively to handle embedded substrings. The
3679 external function follows. The yield is NULL if the expansion failed, and there
3680 are two cases: if something collapsed syntactically, or if "fail" was given
3681 as the action on a lookup failure. These can be distinguised by looking at the
3682 variable expand_string_forcedfail, which is TRUE in the latter case.
3683
3684 The skipping flag is set true when expanding a substring that isn't actually
3685 going to be used (after "if" or "lookup") and it prevents lookups from
3686 happening lower down.
3687
3688 Store usage: At start, a store block of the length of the input plus 64
3689 is obtained. This is expanded as necessary by string_cat(), which might have to
3690 get a new block, or might be able to expand the original. At the end of the
3691 function we can release any store above that portion of the yield block that
3692 was actually used. In many cases this will be optimal.
3693
3694 However: if the first item in the expansion is a variable name or header name,
3695 we reset the store before processing it; if the result is in fresh store, we
3696 use that without copying. This is helpful for expanding strings like
3697 $message_headers which can get very long.
3698
3699 There's a problem if a ${dlfunc item has side-effects that cause allocation,
3700 since resetting the store at the end of the expansion will free store that was
3701 allocated by the plugin code as well as the slop after the expanded string. So
3702 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
3703 and, given the acl condition, ${if }. This is an unfortunate consequence of
3704 string expansion becoming too powerful.
3705
3706 Arguments:
3707   string         the string to be expanded
3708   ket_ends       true if expansion is to stop at }
3709   left           if not NULL, a pointer to the first character after the
3710                  expansion is placed here (typically used with ket_ends)
3711   skipping       TRUE for recursive calls when the value isn't actually going
3712                  to be used (to allow for optimisation)
3713   honour_dollar  TRUE if $ is to be expanded,
3714                  FALSE if it's just another character
3715   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
3716                  the store.
3717
3718 Returns:         NULL if expansion fails:
3719                    expand_string_forcedfail is set TRUE if failure was forced
3720                    expand_string_message contains a textual error message
3721                  a pointer to the expanded string on success
3722 */
3723
3724 static uschar *
3725 expand_string_internal(uschar *string, BOOL ket_ends, uschar **left,
3726   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
3727 {
3728 int ptr = 0;
3729 int size = Ustrlen(string)+ 64;
3730 int item_type;
3731 uschar *yield = store_get(size);
3732 uschar *s = string;
3733 uschar *save_expand_nstring[EXPAND_MAXN+1];
3734 int save_expand_nlength[EXPAND_MAXN+1];
3735 BOOL resetok = TRUE;
3736
3737 expand_string_forcedfail = FALSE;
3738 expand_string_message = US"";
3739
3740 while (*s != 0)
3741   {
3742   uschar *value;
3743   uschar name[256];
3744
3745   /* \ escapes the next character, which must exist, or else
3746   the expansion fails. There's a special escape, \N, which causes
3747   copying of the subject verbatim up to the next \N. Otherwise,
3748   the escapes are the standard set. */
3749
3750   if (*s == '\\')
3751     {
3752     if (s[1] == 0)
3753       {
3754       expand_string_message = US"\\ at end of string";
3755       goto EXPAND_FAILED;
3756       }
3757
3758     if (s[1] == 'N')
3759       {
3760       uschar *t = s + 2;
3761       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
3762       yield = string_cat(yield, &size, &ptr, t, s - t);
3763       if (*s != 0) s += 2;
3764       }
3765
3766     else
3767       {
3768       uschar ch[1];
3769       ch[0] = string_interpret_escape(&s);
3770       s++;
3771       yield = string_cat(yield, &size, &ptr, ch, 1);
3772       }
3773
3774     continue;
3775     }
3776
3777   /*{*/
3778   /* Anything other than $ is just copied verbatim, unless we are
3779   looking for a terminating } character. */
3780
3781   /*{*/
3782   if (ket_ends && *s == '}') break;
3783
3784   if (*s != '$' || !honour_dollar)
3785     {
3786     yield = string_cat(yield, &size, &ptr, s++, 1);
3787     continue;
3788     }
3789
3790   /* No { after the $ - must be a plain name or a number for string
3791   match variable. There has to be a fudge for variables that are the
3792   names of header fields preceded by "$header_" because header field
3793   names can contain any printing characters except space and colon.
3794   For those that don't like typing this much, "$h_" is a synonym for
3795   "$header_". A non-existent header yields a NULL value; nothing is
3796   inserted. */  /*}*/
3797
3798   if (isalpha((*(++s))))
3799     {
3800     int len;
3801     int newsize = 0;
3802
3803     s = read_name(name, sizeof(name), s, US"_");
3804
3805     /* If this is the first thing to be expanded, release the pre-allocated
3806     buffer. */
3807
3808     if (ptr == 0 && yield != NULL)
3809       {
3810       if (resetok) store_reset(yield);
3811       yield = NULL;
3812       size = 0;
3813       }
3814
3815     /* Header */
3816
3817     if (Ustrncmp(name, "h_", 2) == 0 ||
3818         Ustrncmp(name, "rh_", 3) == 0 ||
3819         Ustrncmp(name, "bh_", 3) == 0 ||
3820         Ustrncmp(name, "header_", 7) == 0 ||
3821         Ustrncmp(name, "rheader_", 8) == 0 ||
3822         Ustrncmp(name, "bheader_", 8) == 0)
3823       {
3824       BOOL want_raw = (name[0] == 'r')? TRUE : FALSE;
3825       uschar *charset = (name[0] == 'b')? NULL : headers_charset;
3826       s = read_header_name(name, sizeof(name), s);
3827       value = find_header(name, FALSE, &newsize, want_raw, charset);
3828
3829       /* If we didn't find the header, and the header contains a closing brace
3830       character, this may be a user error where the terminating colon
3831       has been omitted. Set a flag to adjust the error message in this case.
3832       But there is no error here - nothing gets inserted. */
3833
3834       if (value == NULL)
3835         {
3836         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
3837         continue;
3838         }
3839       }
3840
3841     /* Variable */
3842
3843     else
3844       {
3845       value = find_variable(name, FALSE, skipping, &newsize);
3846       if (value == NULL)
3847         {
3848         expand_string_message =
3849           string_sprintf("unknown variable name \"%s\"", name);
3850           check_variable_error_message(name);
3851         goto EXPAND_FAILED;
3852         }
3853       }
3854
3855     /* If the data is known to be in a new buffer, newsize will be set to the
3856     size of that buffer. If this is the first thing in an expansion string,
3857     yield will be NULL; just point it at the new store instead of copying. Many
3858     expansion strings contain just one reference, so this is a useful
3859     optimization, especially for humungous headers. */
3860
3861     len = Ustrlen(value);
3862     if (yield == NULL && newsize != 0)
3863       {
3864       yield = value;
3865       size = newsize;
3866       ptr = len;
3867       }
3868     else yield = string_cat(yield, &size, &ptr, value, len);
3869
3870     continue;
3871     }
3872
3873   if (isdigit(*s))
3874     {
3875     int n;
3876     s = read_number(&n, s);
3877     if (n >= 0 && n <= expand_nmax)
3878       yield = string_cat(yield, &size, &ptr, expand_nstring[n],
3879         expand_nlength[n]);
3880     continue;
3881     }
3882
3883   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
3884
3885   if (*s != '{')                                                        /*}*/
3886     {
3887     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
3888     goto EXPAND_FAILED;
3889     }
3890
3891   /* After { there can be various things, but they all start with
3892   an initial word, except for a number for a string match variable. */
3893
3894   if (isdigit((*(++s))))
3895     {
3896     int n;
3897     s = read_number(&n, s);             /*{*/
3898     if (*s++ != '}')
3899       {                                 /*{*/
3900       expand_string_message = US"} expected after number";
3901       goto EXPAND_FAILED;
3902       }
3903     if (n >= 0 && n <= expand_nmax)
3904       yield = string_cat(yield, &size, &ptr, expand_nstring[n],
3905         expand_nlength[n]);
3906     continue;
3907     }
3908
3909   if (!isalpha(*s))
3910     {
3911     expand_string_message = US"letter or digit expected after ${";      /*}*/
3912     goto EXPAND_FAILED;
3913     }
3914
3915   /* Allow "-" in names to cater for substrings with negative
3916   arguments. Since we are checking for known names after { this is
3917   OK. */
3918
3919   s = read_name(name, sizeof(name), s, US"_-");
3920   item_type = chop_match(name, item_table, sizeof(item_table)/sizeof(uschar *));
3921
3922   switch(item_type)
3923     {
3924     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
3925     If the ACL returns accept or reject we return content set by "message ="
3926     There is currently no limit on recursion; this would have us call
3927     acl_check_internal() directly and get a current level from somewhere.
3928     See also the acl expansion condition ECOND_ACL and the traditional
3929     acl modifier ACLC_ACL.
3930     Assume that the function has side-effects on the store that must be preserved.
3931     */
3932
3933     case EITEM_ACL:
3934       /* ${acl {name} {arg1}{arg2}...} */
3935       {
3936       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
3937       uschar *user_msg;
3938
3939       switch(read_subs(sub, 10, 1, &s, skipping, TRUE, US"acl", &resetok))
3940         {
3941         case 1: goto EXPAND_FAILED_CURLY;
3942         case 2:
3943         case 3: goto EXPAND_FAILED;
3944         }
3945       if (skipping) continue;
3946
3947       resetok = FALSE;
3948       switch(eval_acl(sub, sizeof(sub)/sizeof(*sub), &user_msg))
3949         {
3950         case OK:
3951         case FAIL:
3952           if (user_msg)
3953             yield = string_cat(yield, &size, &ptr, user_msg, Ustrlen(user_msg));
3954           continue;
3955
3956         case DEFER:
3957           expand_string_forcedfail = TRUE;
3958         default:
3959           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
3960           goto EXPAND_FAILED;
3961         }
3962       }
3963
3964     /* Handle conditionals - preserve the values of the numerical expansion
3965     variables in case they get changed by a regular expression match in the
3966     condition. If not, they retain their external settings. At the end
3967     of this "if" section, they get restored to their previous values. */
3968
3969     case EITEM_IF:
3970       {
3971       BOOL cond = FALSE;
3972       uschar *next_s;
3973       int save_expand_nmax =
3974         save_expand_strings(save_expand_nstring, save_expand_nlength);
3975
3976       while (isspace(*s)) s++;
3977       next_s = eval_condition(s, &resetok, skipping? NULL : &cond);
3978       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
3979
3980       DEBUG(D_expand)
3981         debug_printf("condition: %.*s\n   result: %s\n", (int)(next_s - s), s,
3982           cond? "true" : "false");
3983
3984       s = next_s;
3985
3986       /* The handling of "yes" and "no" result strings is now in a separate
3987       function that is also used by ${lookup} and ${extract} and ${run}. */
3988
3989       switch(process_yesno(
3990                skipping,                     /* were previously skipping */
3991                cond,                         /* success/failure indicator */
3992                lookup_value,                 /* value to reset for string2 */
3993                &s,                           /* input pointer */
3994                &yield,                       /* output pointer */
3995                &size,                        /* output size */
3996                &ptr,                         /* output current point */
3997                US"if",                       /* condition type */
3998                &resetok))
3999         {
4000         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4001         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4002         }
4003
4004       /* Restore external setting of expansion variables for continuation
4005       at this level. */
4006
4007       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4008         save_expand_nlength);
4009       continue;
4010       }
4011
4012     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4013     expanding an internal string that isn't actually going to be used. All we
4014     need to do is check the syntax, so don't do a lookup at all. Preserve the
4015     values of the numerical expansion variables in case they get changed by a
4016     partial lookup. If not, they retain their external settings. At the end
4017     of this "lookup" section, they get restored to their previous values. */
4018
4019     case EITEM_LOOKUP:
4020       {
4021       int stype, partial, affixlen, starflags;
4022       int expand_setup = 0;
4023       int nameptr = 0;
4024       uschar *key, *filename, *affix;
4025       uschar *save_lookup_value = lookup_value;
4026       int save_expand_nmax =
4027         save_expand_strings(save_expand_nstring, save_expand_nlength);
4028
4029       if ((expand_forbid & RDO_LOOKUP) != 0)
4030         {
4031         expand_string_message = US"lookup expansions are not permitted";
4032         goto EXPAND_FAILED;
4033         }
4034
4035       /* Get the key we are to look up for single-key+file style lookups.
4036       Otherwise set the key NULL pro-tem. */
4037
4038       while (isspace(*s)) s++;
4039       if (*s == '{')                                    /*}*/
4040         {
4041         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4042         if (key == NULL) goto EXPAND_FAILED;            /*{*/
4043         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4044         while (isspace(*s)) s++;
4045         }
4046       else key = NULL;
4047
4048       /* Find out the type of database */
4049
4050       if (!isalpha(*s))
4051         {
4052         expand_string_message = US"missing lookup type";
4053         goto EXPAND_FAILED;
4054         }
4055
4056       /* The type is a string that may contain special characters of various
4057       kinds. Allow everything except space or { to appear; the actual content
4058       is checked by search_findtype_partial. */         /*}*/
4059
4060       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4061         {
4062         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4063         s++;
4064         }
4065       name[nameptr] = 0;
4066       while (isspace(*s)) s++;
4067
4068       /* Now check for the individual search type and any partial or default
4069       options. Only those types that are actually in the binary are valid. */
4070
4071       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4072         &starflags);
4073       if (stype < 0)
4074         {
4075         expand_string_message = search_error_message;
4076         goto EXPAND_FAILED;
4077         }
4078
4079       /* Check that a key was provided for those lookup types that need it,
4080       and was not supplied for those that use the query style. */
4081
4082       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4083         {
4084         if (key == NULL)
4085           {
4086           expand_string_message = string_sprintf("missing {key} for single-"
4087             "key \"%s\" lookup", name);
4088           goto EXPAND_FAILED;
4089           }
4090         }
4091       else
4092         {
4093         if (key != NULL)
4094           {
4095           expand_string_message = string_sprintf("a single key was given for "
4096             "lookup type \"%s\", which is not a single-key lookup type", name);
4097           goto EXPAND_FAILED;
4098           }
4099         }
4100
4101       /* Get the next string in brackets and expand it. It is the file name for
4102       single-key+file lookups, and the whole query otherwise. In the case of
4103       queries that also require a file name (e.g. sqlite), the file name comes
4104       first. */
4105
4106       if (*s != '{') goto EXPAND_FAILED_CURLY;
4107       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4108       if (filename == NULL) goto EXPAND_FAILED;
4109       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4110       while (isspace(*s)) s++;
4111
4112       /* If this isn't a single-key+file lookup, re-arrange the variables
4113       to be appropriate for the search_ functions. For query-style lookups,
4114       there is just a "key", and no file name. For the special query-style +
4115       file types, the query (i.e. "key") starts with a file name. */
4116
4117       if (key == NULL)
4118         {
4119         while (isspace(*filename)) filename++;
4120         key = filename;
4121
4122         if (mac_islookup(stype, lookup_querystyle))
4123           {
4124           filename = NULL;
4125           }
4126         else
4127           {
4128           if (*filename != '/')
4129             {
4130             expand_string_message = string_sprintf(
4131               "absolute file name expected for \"%s\" lookup", name);
4132             goto EXPAND_FAILED;
4133             }
4134           while (*key != 0 && !isspace(*key)) key++;
4135           if (*key != 0) *key++ = 0;
4136           }
4137         }
4138
4139       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4140       the entry was not found. Note that there is no search_close() function.
4141       Files are left open in case of re-use. At suitable places in higher logic,
4142       search_tidyup() is called to tidy all open files. This can save opening
4143       the same file several times. However, files may also get closed when
4144       others are opened, if too many are open at once. The rule is that a
4145       handle should not be used after a second search_open().
4146
4147       Request that a partial search sets up $1 and maybe $2 by passing
4148       expand_setup containing zero. If its value changes, reset expand_nmax,
4149       since new variables will have been set. Note that at the end of this
4150       "lookup" section, the old numeric variables are restored. */
4151
4152       if (skipping)
4153         lookup_value = NULL;
4154       else
4155         {
4156         void *handle = search_open(filename, stype, 0, NULL, NULL);
4157         if (handle == NULL)
4158           {
4159           expand_string_message = search_error_message;
4160           goto EXPAND_FAILED;
4161           }
4162         lookup_value = search_find(handle, filename, key, partial, affix,
4163           affixlen, starflags, &expand_setup);
4164         if (search_find_defer)
4165           {
4166           expand_string_message =
4167             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4168               string_printing2(key, FALSE), search_error_message);
4169           goto EXPAND_FAILED;
4170           }
4171         if (expand_setup > 0) expand_nmax = expand_setup;
4172         }
4173
4174       /* The handling of "yes" and "no" result strings is now in a separate
4175       function that is also used by ${if} and ${extract}. */
4176
4177       switch(process_yesno(
4178                skipping,                     /* were previously skipping */
4179                lookup_value != NULL,         /* success/failure indicator */
4180                save_lookup_value,            /* value to reset for string2 */
4181                &s,                           /* input pointer */
4182                &yield,                       /* output pointer */
4183                &size,                        /* output size */
4184                &ptr,                         /* output current point */
4185                US"lookup",                   /* condition type */
4186                &resetok))
4187         {
4188         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4189         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4190         }
4191
4192       /* Restore external setting of expansion variables for carrying on
4193       at this level, and continue. */
4194
4195       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4196         save_expand_nlength);
4197       continue;
4198       }
4199
4200     /* If Perl support is configured, handle calling embedded perl subroutines,
4201     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4202     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4203     arguments (defined below). */
4204
4205     #define EXIM_PERL_MAX_ARGS 8
4206
4207     case EITEM_PERL:
4208     #ifndef EXIM_PERL
4209     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4210       "is not included in this binary";
4211     goto EXPAND_FAILED;
4212
4213     #else   /* EXIM_PERL */
4214       {
4215       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4216       uschar *new_yield;
4217
4218       if ((expand_forbid & RDO_PERL) != 0)
4219         {
4220         expand_string_message = US"Perl calls are not permitted";
4221         goto EXPAND_FAILED;
4222         }
4223
4224       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4225            US"perl", &resetok))
4226         {
4227         case 1: goto EXPAND_FAILED_CURLY;
4228         case 2:
4229         case 3: goto EXPAND_FAILED;
4230         }
4231
4232       /* If skipping, we don't actually do anything */
4233
4234       if (skipping) continue;
4235
4236       /* Start the interpreter if necessary */
4237
4238       if (!opt_perl_started)
4239         {
4240         uschar *initerror;
4241         if (opt_perl_startup == NULL)
4242           {
4243           expand_string_message = US"A setting of perl_startup is needed when "
4244             "using the Perl interpreter";
4245           goto EXPAND_FAILED;
4246           }
4247         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4248         initerror = init_perl(opt_perl_startup);
4249         if (initerror != NULL)
4250           {
4251           expand_string_message =
4252             string_sprintf("error in perl_startup code: %s\n", initerror);
4253           goto EXPAND_FAILED;
4254           }
4255         opt_perl_started = TRUE;
4256         }
4257
4258       /* Call the function */
4259
4260       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4261       new_yield = call_perl_cat(yield, &size, &ptr, &expand_string_message,
4262         sub_arg[0], sub_arg + 1);
4263
4264       /* NULL yield indicates failure; if the message pointer has been set to
4265       NULL, the yield was undef, indicating a forced failure. Otherwise the
4266       message will indicate some kind of Perl error. */
4267
4268       if (new_yield == NULL)
4269         {
4270         if (expand_string_message == NULL)
4271           {
4272           expand_string_message =
4273             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4274               "failure", sub_arg[0]);
4275           expand_string_forcedfail = TRUE;
4276           }
4277         goto EXPAND_FAILED;
4278         }
4279
4280       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4281       set during a callback from Perl. */
4282
4283       expand_string_forcedfail = FALSE;
4284       yield = new_yield;
4285       continue;
4286       }
4287     #endif /* EXIM_PERL */
4288
4289     /* Transform email address to "prvs" scheme to use
4290        as BATV-signed return path */
4291
4292     case EITEM_PRVS:
4293       {
4294       uschar *sub_arg[3];
4295       uschar *p,*domain;
4296
4297       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4298         {
4299         case 1: goto EXPAND_FAILED_CURLY;
4300         case 2:
4301         case 3: goto EXPAND_FAILED;
4302         }
4303
4304       /* If skipping, we don't actually do anything */
4305       if (skipping) continue;
4306
4307       /* sub_arg[0] is the address */
4308       domain = Ustrrchr(sub_arg[0],'@');
4309       if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) )
4310         {
4311         expand_string_message = US"prvs first argument must be a qualified email address";
4312         goto EXPAND_FAILED;
4313         }
4314
4315       /* Calculate the hash. The second argument must be a single-digit
4316       key number, or unset. */
4317
4318       if (sub_arg[2] != NULL &&
4319           (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4320         {
4321         expand_string_message = US"prvs second argument must be a single digit";
4322         goto EXPAND_FAILED;
4323         }
4324
4325       p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7));
4326       if (p == NULL)
4327         {
4328         expand_string_message = US"prvs hmac-sha1 conversion failed";
4329         goto EXPAND_FAILED;
4330         }
4331
4332       /* Now separate the domain from the local part */
4333       *domain++ = '\0';
4334
4335       yield = string_cat(yield,&size,&ptr,US"prvs=",5);
4336       string_cat(yield,&size,&ptr,(sub_arg[2] != NULL) ? sub_arg[2] : US"0", 1);
4337       string_cat(yield,&size,&ptr,prvs_daystamp(7),3);
4338       string_cat(yield,&size,&ptr,p,6);
4339       string_cat(yield,&size,&ptr,US"=",1);
4340       string_cat(yield,&size,&ptr,sub_arg[0],Ustrlen(sub_arg[0]));
4341       string_cat(yield,&size,&ptr,US"@",1);
4342       string_cat(yield,&size,&ptr,domain,Ustrlen(domain));
4343
4344       continue;
4345       }
4346
4347     /* Check a prvs-encoded address for validity */
4348
4349     case EITEM_PRVSCHECK:
4350       {
4351       uschar *sub_arg[3];
4352       int mysize = 0, myptr = 0;
4353       const pcre *re;
4354       uschar *p;
4355
4356       /* TF: Ugliness: We want to expand parameter 1 first, then set
4357          up expansion variables that are used in the expansion of
4358          parameter 2. So we clone the string for the first
4359          expansion, where we only expand parameter 1.
4360
4361          PH: Actually, that isn't necessary. The read_subs() function is
4362          designed to work this way for the ${if and ${lookup expansions. I've
4363          tidied the code.
4364       */
4365
4366       /* Reset expansion variables */
4367       prvscheck_result = NULL;
4368       prvscheck_address = NULL;
4369       prvscheck_keynum = NULL;
4370
4371       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4372         {
4373         case 1: goto EXPAND_FAILED_CURLY;
4374         case 2:
4375         case 3: goto EXPAND_FAILED;
4376         }
4377
4378       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4379                               TRUE,FALSE);
4380
4381       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4382         {
4383         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4384         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4385         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4386         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4387         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4388
4389         DEBUG(D_expand) debug_printf("prvscheck localpart: %s\n", local_part);
4390         DEBUG(D_expand) debug_printf("prvscheck key number: %s\n", key_num);
4391         DEBUG(D_expand) debug_printf("prvscheck daystamp: %s\n", daystamp);
4392         DEBUG(D_expand) debug_printf("prvscheck hash: %s\n", hash);
4393         DEBUG(D_expand) debug_printf("prvscheck domain: %s\n", domain);
4394
4395         /* Set up expansion variables */
4396         prvscheck_address = string_cat(NULL, &mysize, &myptr, local_part, Ustrlen(local_part));
4397         string_cat(prvscheck_address,&mysize,&myptr,US"@",1);
4398         string_cat(prvscheck_address,&mysize,&myptr,domain,Ustrlen(domain));
4399         prvscheck_address[myptr] = '\0';
4400         prvscheck_keynum = string_copy(key_num);
4401
4402         /* Now expand the second argument */
4403         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4404           {
4405           case 1: goto EXPAND_FAILED_CURLY;
4406           case 2:
4407           case 3: goto EXPAND_FAILED;
4408           }
4409
4410         /* Now we have the key and can check the address. */
4411
4412         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4413           daystamp);
4414
4415         if (p == NULL)
4416           {
4417           expand_string_message = US"hmac-sha1 conversion failed";
4418           goto EXPAND_FAILED;
4419           }
4420
4421         DEBUG(D_expand) debug_printf("prvscheck: received hash is %s\n", hash);
4422         DEBUG(D_expand) debug_printf("prvscheck:      own hash is %s\n", p);
4423
4424         if (Ustrcmp(p,hash) == 0)
4425           {
4426           /* Success, valid BATV address. Now check the expiry date. */
4427           uschar *now = prvs_daystamp(0);
4428           unsigned int inow = 0,iexpire = 1;
4429
4430           (void)sscanf(CS now,"%u",&inow);
4431           (void)sscanf(CS daystamp,"%u",&iexpire);
4432
4433           /* When "iexpire" is < 7, a "flip" has occured.
4434              Adjust "inow" accordingly. */
4435           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4436
4437           if (iexpire >= inow)
4438             {
4439             prvscheck_result = US"1";
4440             DEBUG(D_expand) debug_printf("prvscheck: success, $pvrs_result set to 1\n");
4441             }
4442             else
4443             {
4444             prvscheck_result = NULL;
4445             DEBUG(D_expand) debug_printf("prvscheck: signature expired, $pvrs_result unset\n");
4446             }
4447           }
4448         else
4449           {
4450           prvscheck_result = NULL;
4451           DEBUG(D_expand) debug_printf("prvscheck: hash failure, $pvrs_result unset\n");
4452           }
4453
4454         /* Now expand the final argument. We leave this till now so that
4455         it can include $prvscheck_result. */
4456
4457         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4458           {
4459           case 1: goto EXPAND_FAILED_CURLY;
4460           case 2:
4461           case 3: goto EXPAND_FAILED;
4462           }
4463
4464         if (sub_arg[0] == NULL || *sub_arg[0] == '\0')
4465           yield = string_cat(yield,&size,&ptr,prvscheck_address,Ustrlen(prvscheck_address));
4466         else
4467           yield = string_cat(yield,&size,&ptr,sub_arg[0],Ustrlen(sub_arg[0]));
4468
4469         /* Reset the "internal" variables afterwards, because they are in
4470         dynamic store that will be reclaimed if the expansion succeeded. */
4471
4472         prvscheck_address = NULL;
4473         prvscheck_keynum = NULL;
4474         }
4475       else
4476         {
4477         /* Does not look like a prvs encoded address, return the empty string.
4478            We need to make sure all subs are expanded first, so as to skip over
4479            the entire item. */
4480
4481         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4482           {
4483           case 1: goto EXPAND_FAILED_CURLY;
4484           case 2:
4485           case 3: goto EXPAND_FAILED;
4486           }
4487         }
4488
4489       continue;
4490       }
4491
4492     /* Handle "readfile" to insert an entire file */
4493
4494     case EITEM_READFILE:
4495       {
4496       FILE *f;
4497       uschar *sub_arg[2];
4498
4499       if ((expand_forbid & RDO_READFILE) != 0)
4500         {
4501         expand_string_message = US"file insertions are not permitted";
4502         goto EXPAND_FAILED;
4503         }
4504
4505       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4506         {
4507         case 1: goto EXPAND_FAILED_CURLY;
4508         case 2:
4509         case 3: goto EXPAND_FAILED;
4510         }
4511
4512       /* If skipping, we don't actually do anything */
4513
4514       if (skipping) continue;
4515
4516       /* Open the file and read it */
4517
4518       f = Ufopen(sub_arg[0], "rb");
4519       if (f == NULL)
4520         {
4521         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4522         goto EXPAND_FAILED;
4523         }
4524
4525       yield = cat_file(f, yield, &size, &ptr, sub_arg[1]);
4526       (void)fclose(f);
4527       continue;
4528       }
4529
4530     /* Handle "readsocket" to insert data from a Unix domain socket */
4531
4532     case EITEM_READSOCK:
4533       {
4534       int fd;
4535       int timeout = 5;
4536       int save_ptr = ptr;
4537       FILE *f;
4538       struct sockaddr_un sockun;         /* don't call this "sun" ! */
4539       uschar *arg;
4540       uschar *sub_arg[4];
4541
4542       if ((expand_forbid & RDO_READSOCK) != 0)
4543         {
4544         expand_string_message = US"socket insertions are not permitted";
4545         goto EXPAND_FAILED;
4546         }
4547
4548       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4549       because there may be a string for expansion on failure. */
4550
4551       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4552         {
4553         case 1: goto EXPAND_FAILED_CURLY;
4554         case 2:                             /* Won't occur: no end check */
4555         case 3: goto EXPAND_FAILED;
4556         }
4557
4558       /* Sort out timeout, if given */
4559
4560       if (sub_arg[2] != NULL)
4561         {
4562         timeout = readconf_readtime(sub_arg[2], 0, FALSE);
4563         if (timeout < 0)
4564           {
4565           expand_string_message = string_sprintf("bad time value %s",
4566             sub_arg[2]);
4567           goto EXPAND_FAILED;
4568           }
4569         }
4570       else sub_arg[3] = NULL;                     /* No eol if no timeout */
4571
4572       /* If skipping, we don't actually do anything. Otherwise, arrange to
4573       connect to either an IP or a Unix socket. */
4574
4575       if (!skipping)
4576         {
4577         /* Handle an IP (internet) domain */
4578
4579         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
4580           {
4581           int port;
4582           uschar *server_name = sub_arg[0] + 5;
4583           uschar *port_name = Ustrrchr(server_name, ':');
4584
4585           /* Sort out the port */
4586
4587           if (port_name == NULL)
4588             {
4589             expand_string_message =
4590               string_sprintf("missing port for readsocket %s", sub_arg[0]);
4591             goto EXPAND_FAILED;
4592             }
4593           *port_name++ = 0;           /* Terminate server name */
4594
4595           if (isdigit(*port_name))
4596             {
4597             uschar *end;
4598             port = Ustrtol(port_name, &end, 0);
4599             if (end != port_name + Ustrlen(port_name))
4600               {
4601               expand_string_message =
4602                 string_sprintf("invalid port number %s", port_name);
4603               goto EXPAND_FAILED;
4604               }
4605             }
4606           else
4607             {
4608             struct servent *service_info = getservbyname(CS port_name, "tcp");
4609             if (service_info == NULL)
4610               {
4611               expand_string_message = string_sprintf("unknown port \"%s\"",
4612                 port_name);
4613               goto EXPAND_FAILED;
4614               }
4615             port = ntohs(service_info->s_port);
4616             }
4617
4618           if ((fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
4619                   timeout, NULL, &expand_string_message)) < 0)
4620               goto SOCK_FAIL;
4621           }
4622
4623         /* Handle a Unix domain socket */
4624
4625         else
4626           {
4627           int rc;
4628           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
4629             {
4630             expand_string_message = string_sprintf("failed to create socket: %s",
4631               strerror(errno));
4632             goto SOCK_FAIL;
4633             }
4634
4635           sockun.sun_family = AF_UNIX;
4636           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
4637             sub_arg[0]);
4638
4639           sigalrm_seen = FALSE;
4640           alarm(timeout);
4641           rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun));
4642           alarm(0);
4643           if (sigalrm_seen)
4644             {
4645             expand_string_message = US "socket connect timed out";
4646             goto SOCK_FAIL;
4647             }
4648           if (rc < 0)
4649             {
4650             expand_string_message = string_sprintf("failed to connect to socket "
4651               "%s: %s", sub_arg[0], strerror(errno));
4652             goto SOCK_FAIL;
4653             }
4654           }
4655
4656         DEBUG(D_expand) debug_printf("connected to socket %s\n", sub_arg[0]);
4657
4658         /* Write the request string, if not empty */
4659
4660         if (sub_arg[1][0] != 0)
4661           {
4662           int len = Ustrlen(sub_arg[1]);
4663           DEBUG(D_expand) debug_printf("writing \"%s\" to socket\n",
4664             sub_arg[1]);
4665           if (write(fd, sub_arg[1], len) != len)
4666             {
4667             expand_string_message = string_sprintf("request write to socket "
4668               "failed: %s", strerror(errno));
4669             goto SOCK_FAIL;
4670             }
4671           }
4672
4673         /* Shut down the sending side of the socket. This helps some servers to
4674         recognise that it is their turn to do some work. Just in case some
4675         system doesn't have this function, make it conditional. */
4676
4677         #ifdef SHUT_WR
4678         shutdown(fd, SHUT_WR);
4679         #endif
4680
4681         /* Now we need to read from the socket, under a timeout. The function
4682         that reads a file can be used. */
4683
4684         f = fdopen(fd, "rb");
4685         sigalrm_seen = FALSE;
4686         alarm(timeout);
4687         yield = cat_file(f, yield, &size, &ptr, sub_arg[3]);
4688         alarm(0);
4689         (void)fclose(f);
4690
4691         /* After a timeout, we restore the pointer in the result, that is,
4692         make sure we add nothing from the socket. */
4693
4694         if (sigalrm_seen)
4695           {
4696           ptr = save_ptr;
4697           expand_string_message = US "socket read timed out";
4698           goto SOCK_FAIL;
4699           }
4700         }
4701
4702       /* The whole thing has worked (or we were skipping). If there is a
4703       failure string following, we need to skip it. */
4704
4705       if (*s == '{')
4706         {
4707         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
4708           goto EXPAND_FAILED;
4709         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4710         while (isspace(*s)) s++;
4711         }
4712       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4713       continue;
4714
4715       /* Come here on failure to create socket, connect socket, write to the
4716       socket, or timeout on reading. If another substring follows, expand and
4717       use it. Otherwise, those conditions give expand errors. */
4718
4719       SOCK_FAIL:
4720       if (*s != '{') goto EXPAND_FAILED;
4721       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
4722       arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok);
4723       if (arg == NULL) goto EXPAND_FAILED;
4724       yield = string_cat(yield, &size, &ptr, arg, Ustrlen(arg));
4725       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4726       while (isspace(*s)) s++;
4727       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4728       continue;
4729       }
4730
4731     /* Handle "run" to execute a program. */
4732
4733     case EITEM_RUN:
4734       {
4735       FILE *f;
4736       uschar *arg;
4737       uschar **argv;
4738       pid_t pid;
4739       int fd_in, fd_out;
4740       int lsize = 0;
4741       int lptr = 0;
4742
4743       if ((expand_forbid & RDO_RUN) != 0)
4744         {
4745         expand_string_message = US"running a command is not permitted";
4746         goto EXPAND_FAILED;
4747         }
4748
4749       while (isspace(*s)) s++;
4750       if (*s != '{') goto EXPAND_FAILED_CURLY;
4751       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4752       if (arg == NULL) goto EXPAND_FAILED;
4753       while (isspace(*s)) s++;
4754       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4755
4756       if (skipping)   /* Just pretend it worked when we're skipping */
4757         {
4758         runrc = 0;
4759         }
4760       else
4761         {
4762         if (!transport_set_up_command(&argv,    /* anchor for arg list */
4763             arg,                                /* raw command */
4764             FALSE,                              /* don't expand the arguments */
4765             0,                                  /* not relevant when... */
4766             NULL,                               /* no transporting address */
4767             US"${run} expansion",               /* for error messages */
4768             &expand_string_message))            /* where to put error message */
4769           {
4770           goto EXPAND_FAILED;
4771           }
4772
4773         /* Create the child process, making it a group leader. */
4774
4775         pid = child_open(argv, NULL, 0077, &fd_in, &fd_out, TRUE);
4776
4777         if (pid < 0)
4778           {
4779           expand_string_message =
4780             string_sprintf("couldn't create child process: %s", strerror(errno));
4781           goto EXPAND_FAILED;
4782           }
4783
4784         /* Nothing is written to the standard input. */
4785
4786         (void)close(fd_in);
4787
4788         /* Read the pipe to get the command's output into $value (which is kept
4789         in lookup_value). Read during execution, so that if the output exceeds
4790         the OS pipe buffer limit, we don't block forever. */
4791
4792         f = fdopen(fd_out, "rb");
4793         sigalrm_seen = FALSE;
4794         alarm(60);
4795         lookup_value = cat_file(f, lookup_value, &lsize, &lptr, NULL);
4796         alarm(0);
4797         (void)fclose(f);
4798
4799         /* Wait for the process to finish, applying the timeout, and inspect its
4800         return code for serious disasters. Simple non-zero returns are passed on.
4801         */
4802
4803         if (sigalrm_seen == TRUE || (runrc = child_close(pid, 30)) < 0)
4804           {
4805           if (sigalrm_seen == TRUE || runrc == -256)
4806             {
4807             expand_string_message = string_sprintf("command timed out");
4808             killpg(pid, SIGKILL);       /* Kill the whole process group */
4809             }
4810
4811           else if (runrc == -257)
4812             expand_string_message = string_sprintf("wait() failed: %s",
4813               strerror(errno));
4814
4815           else
4816             expand_string_message = string_sprintf("command killed by signal %d",
4817               -runrc);
4818
4819           goto EXPAND_FAILED;
4820           }
4821         }
4822
4823       /* Process the yes/no strings; $value may be useful in both cases */
4824
4825       switch(process_yesno(
4826                skipping,                     /* were previously skipping */
4827                runrc == 0,                   /* success/failure indicator */
4828                lookup_value,                 /* value to reset for string2 */
4829                &s,                           /* input pointer */
4830                &yield,                       /* output pointer */
4831                &size,                        /* output size */
4832                &ptr,                         /* output current point */
4833                US"run",                      /* condition type */
4834                &resetok))
4835         {
4836         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4837         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4838         }
4839
4840       continue;
4841       }
4842
4843     /* Handle character translation for "tr" */
4844
4845     case EITEM_TR:
4846       {
4847       int oldptr = ptr;
4848       int o2m;
4849       uschar *sub[3];
4850
4851       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
4852         {
4853         case 1: goto EXPAND_FAILED_CURLY;
4854         case 2:
4855         case 3: goto EXPAND_FAILED;
4856         }
4857
4858       yield = string_cat(yield, &size, &ptr, sub[0], Ustrlen(sub[0]));
4859       o2m = Ustrlen(sub[2]) - 1;
4860
4861       if (o2m >= 0) for (; oldptr < ptr; oldptr++)
4862         {
4863         uschar *m = Ustrrchr(sub[1], yield[oldptr]);
4864         if (m != NULL)
4865           {
4866           int o = m - sub[1];
4867           yield[oldptr] = sub[2][(o < o2m)? o : o2m];
4868           }
4869         }
4870
4871       continue;
4872       }
4873
4874     /* Handle "hash", "length", "nhash", and "substr" when they are given with
4875     expanded arguments. */
4876
4877     case EITEM_HASH:
4878     case EITEM_LENGTH:
4879     case EITEM_NHASH:
4880     case EITEM_SUBSTR:
4881       {
4882       int i;
4883       int len;
4884       uschar *ret;
4885       int val[2] = { 0, -1 };
4886       uschar *sub[3];
4887
4888       /* "length" takes only 2 arguments whereas the others take 2 or 3.
4889       Ensure that sub[2] is set in the ${length } case. */
4890
4891       sub[2] = NULL;
4892       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
4893              TRUE, name, &resetok))
4894         {
4895         case 1: goto EXPAND_FAILED_CURLY;
4896         case 2:
4897         case 3: goto EXPAND_FAILED;
4898         }
4899
4900       /* Juggle the arguments if there are only two of them: always move the
4901       string to the last position and make ${length{n}{str}} equivalent to
4902       ${substr{0}{n}{str}}. See the defaults for val[] above. */
4903
4904       if (sub[2] == NULL)
4905         {
4906         sub[2] = sub[1];
4907         sub[1] = NULL;
4908         if (item_type == EITEM_LENGTH)
4909           {
4910           sub[1] = sub[0];
4911           sub[0] = NULL;
4912           }
4913         }
4914
4915       for (i = 0; i < 2; i++)
4916         {
4917         if (sub[i] == NULL) continue;
4918         val[i] = (int)Ustrtol(sub[i], &ret, 10);
4919         if (*ret != 0 || (i != 0 && val[i] < 0))
4920           {
4921           expand_string_message = string_sprintf("\"%s\" is not a%s number "
4922             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
4923           goto EXPAND_FAILED;
4924           }
4925         }
4926
4927       ret =
4928         (item_type == EITEM_HASH)?
4929           compute_hash(sub[2], val[0], val[1], &len) :
4930         (item_type == EITEM_NHASH)?
4931           compute_nhash(sub[2], val[0], val[1], &len) :
4932           extract_substr(sub[2], val[0], val[1], &len);
4933
4934       if (ret == NULL) goto EXPAND_FAILED;
4935       yield = string_cat(yield, &size, &ptr, ret, len);
4936       continue;
4937       }
4938
4939     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
4940     This code originally contributed by Steve Haslam. It currently supports
4941     the use of MD5 and SHA-1 hashes.
4942
4943     We need some workspace that is large enough to handle all the supported
4944     hash types. Use macros to set the sizes rather than be too elaborate. */
4945
4946     #define MAX_HASHLEN      20
4947     #define MAX_HASHBLOCKLEN 64
4948
4949     case EITEM_HMAC:
4950       {
4951       uschar *sub[3];
4952       md5 md5_base;
4953       sha1 sha1_base;
4954       void *use_base;
4955       int type, i;
4956       int hashlen;      /* Number of octets for the hash algorithm's output */
4957       int hashblocklen; /* Number of octets the hash algorithm processes */
4958       uschar *keyptr, *p;
4959       unsigned int keylen;
4960
4961       uschar keyhash[MAX_HASHLEN];
4962       uschar innerhash[MAX_HASHLEN];
4963       uschar finalhash[MAX_HASHLEN];
4964       uschar finalhash_hex[2*MAX_HASHLEN];
4965       uschar innerkey[MAX_HASHBLOCKLEN];
4966       uschar outerkey[MAX_HASHBLOCKLEN];
4967
4968       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
4969         {
4970         case 1: goto EXPAND_FAILED_CURLY;
4971         case 2:
4972         case 3: goto EXPAND_FAILED;
4973         }
4974
4975       if (Ustrcmp(sub[0], "md5") == 0)
4976         {
4977         type = HMAC_MD5;
4978         use_base = &md5_base;
4979         hashlen = 16;
4980         hashblocklen = 64;
4981         }
4982       else if (Ustrcmp(sub[0], "sha1") == 0)
4983         {
4984         type = HMAC_SHA1;
4985         use_base = &sha1_base;
4986         hashlen = 20;
4987         hashblocklen = 64;
4988         }
4989       else
4990         {
4991         expand_string_message =
4992           string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
4993         goto EXPAND_FAILED;
4994         }
4995
4996       keyptr = sub[1];
4997       keylen = Ustrlen(keyptr);
4998
4999       /* If the key is longer than the hash block length, then hash the key
5000       first */
5001
5002       if (keylen > hashblocklen)
5003         {
5004         chash_start(type, use_base);
5005         chash_end(type, use_base, keyptr, keylen, keyhash);
5006         keyptr = keyhash;
5007         keylen = hashlen;
5008         }
5009
5010       /* Now make the inner and outer key values */
5011
5012       memset(innerkey, 0x36, hashblocklen);
5013       memset(outerkey, 0x5c, hashblocklen);
5014
5015       for (i = 0; i < keylen; i++)
5016         {
5017         innerkey[i] ^= keyptr[i];
5018         outerkey[i] ^= keyptr[i];
5019         }
5020
5021       /* Now do the hashes */
5022
5023       chash_start(type, use_base);
5024       chash_mid(type, use_base, innerkey);
5025       chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5026
5027       chash_start(type, use_base);
5028       chash_mid(type, use_base, outerkey);
5029       chash_end(type, use_base, innerhash, hashlen, finalhash);
5030
5031       /* Encode the final hash as a hex string */
5032
5033       p = finalhash_hex;
5034       for (i = 0; i < hashlen; i++)
5035         {
5036         *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5037         *p++ = hex_digits[finalhash[i] & 0x0f];
5038         }
5039
5040       DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%.*s)=%.*s\n", sub[0],
5041         (int)keylen, keyptr, Ustrlen(sub[2]), sub[2], hashlen*2, finalhash_hex);
5042
5043       yield = string_cat(yield, &size, &ptr, finalhash_hex, hashlen*2);
5044       }
5045
5046     continue;
5047
5048     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5049     We have to save the numerical variables and restore them afterwards. */
5050
5051     case EITEM_SG:
5052       {
5053       const pcre *re;
5054       int moffset, moffsetextra, slen;
5055       int roffset;
5056       int emptyopt;
5057       const uschar *rerror;
5058       uschar *subject;
5059       uschar *sub[3];
5060       int save_expand_nmax =
5061         save_expand_strings(save_expand_nstring, save_expand_nlength);
5062
5063       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5064         {
5065         case 1: goto EXPAND_FAILED_CURLY;
5066         case 2:
5067         case 3: goto EXPAND_FAILED;
5068         }
5069
5070       /* Compile the regular expression */
5071
5072       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5073         NULL);
5074
5075       if (re == NULL)
5076         {
5077         expand_string_message = string_sprintf("regular expression error in "
5078           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5079         goto EXPAND_FAILED;
5080         }
5081
5082       /* Now run a loop to do the substitutions as often as necessary. It ends
5083       when there are no more matches. Take care over matches of the null string;
5084       do the same thing as Perl does. */
5085
5086       subject = sub[0];
5087       slen = Ustrlen(sub[0]);
5088       moffset = moffsetextra = 0;
5089       emptyopt = 0;
5090
5091       for (;;)
5092         {
5093         int ovector[3*(EXPAND_MAXN+1)];
5094         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5095           PCRE_EOPT | emptyopt, ovector, sizeof(ovector)/sizeof(int));
5096         int nn;
5097         uschar *insert;
5098
5099         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5100         is not necessarily the end. We want to repeat the match from one
5101         character further along, but leaving the basic offset the same (for
5102         copying below). We can't be at the end of the string - that was checked
5103         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5104         finished; copy the remaining string and end the loop. */
5105
5106         if (n < 0)
5107           {
5108           if (emptyopt != 0)
5109             {
5110             moffsetextra = 1;
5111             emptyopt = 0;
5112             continue;
5113             }
5114           yield = string_cat(yield, &size, &ptr, subject+moffset, slen-moffset);
5115           break;
5116           }
5117
5118         /* Match - set up for expanding the replacement. */
5119
5120         if (n == 0) n = EXPAND_MAXN + 1;
5121         expand_nmax = 0;
5122         for (nn = 0; nn < n*2; nn += 2)
5123           {
5124           expand_nstring[expand_nmax] = subject + ovector[nn];
5125           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5126           }
5127         expand_nmax--;
5128
5129         /* Copy the characters before the match, plus the expanded insertion. */
5130
5131         yield = string_cat(yield, &size, &ptr, subject + moffset,
5132           ovector[0] - moffset);
5133         insert = expand_string(sub[2]);
5134         if (insert == NULL) goto EXPAND_FAILED;
5135         yield = string_cat(yield, &size, &ptr, insert, Ustrlen(insert));
5136
5137         moffset = ovector[1];
5138         moffsetextra = 0;
5139         emptyopt = 0;
5140
5141         /* If we have matched an empty string, first check to see if we are at
5142         the end of the subject. If so, the loop is over. Otherwise, mimic
5143         what Perl's /g options does. This turns out to be rather cunning. First
5144         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5145         string at the same point. If this fails (picked up above) we advance to
5146         the next character. */
5147
5148         if (ovector[0] == ovector[1])
5149           {
5150           if (ovector[0] == slen) break;
5151           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5152           }
5153         }
5154
5155       /* All done - restore numerical variables. */
5156
5157       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5158         save_expand_nlength);
5159       continue;
5160       }
5161
5162     /* Handle keyed and numbered substring extraction. If the first argument
5163     consists entirely of digits, then a numerical extraction is assumed. */
5164
5165     case EITEM_EXTRACT:
5166       {
5167       int i;
5168       int j = 2;
5169       int field_number = 1;
5170       BOOL field_number_set = FALSE;
5171       uschar *save_lookup_value = lookup_value;
5172       uschar *sub[3];
5173       int save_expand_nmax =
5174         save_expand_strings(save_expand_nstring, save_expand_nlength);
5175
5176       /* Read the arguments */
5177
5178       for (i = 0; i < j; i++)
5179         {
5180         while (isspace(*s)) s++;
5181         if (*s == '{')                                          /*}*/
5182           {
5183           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5184           if (sub[i] == NULL) goto EXPAND_FAILED;               /*{*/
5185           if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5186
5187           /* After removal of leading and trailing white space, the first
5188           argument must not be empty; if it consists entirely of digits
5189           (optionally preceded by a minus sign), this is a numerical
5190           extraction, and we expect 3 arguments. */
5191
5192           if (i == 0)
5193             {
5194             int len;
5195             int x = 0;
5196             uschar *p = sub[0];
5197
5198             while (isspace(*p)) p++;
5199             sub[0] = p;
5200
5201             len = Ustrlen(p);
5202             while (len > 0 && isspace(p[len-1])) len--;
5203             p[len] = 0;
5204
5205             if (*p == 0 && !skipping)
5206               {
5207               expand_string_message = US"first argument of \"extract\" must "
5208                 "not be empty";
5209               goto EXPAND_FAILED;
5210               }
5211
5212             if (*p == '-')
5213               {
5214               field_number = -1;
5215               p++;
5216               }
5217             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5218             if (*p == 0)
5219               {
5220               field_number *= x;
5221               j = 3;               /* Need 3 args */
5222               field_number_set = TRUE;
5223               }
5224             }
5225           }
5226         else goto EXPAND_FAILED_CURLY;
5227         }
5228
5229       /* Extract either the numbered or the keyed substring into $value. If
5230       skipping, just pretend the extraction failed. */
5231
5232       lookup_value = skipping? NULL : field_number_set?
5233         expand_gettokened(field_number, sub[1], sub[2]) :
5234         expand_getkeyed(sub[0], sub[1]);
5235
5236       /* If no string follows, $value gets substituted; otherwise there can
5237       be yes/no strings, as for lookup or if. */
5238
5239       switch(process_yesno(
5240                skipping,                     /* were previously skipping */
5241                lookup_value != NULL,         /* success/failure indicator */
5242                save_lookup_value,            /* value to reset for string2 */
5243                &s,                           /* input pointer */
5244                &yield,                       /* output pointer */
5245                &size,                        /* output size */
5246                &ptr,                         /* output current point */
5247                US"extract",                  /* condition type */
5248                &resetok))
5249         {
5250         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5251         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5252         }
5253
5254       /* All done - restore numerical variables. */
5255
5256       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5257         save_expand_nlength);
5258
5259       continue;
5260       }
5261
5262     /* return the Nth item from a list */
5263
5264     case EITEM_LISTEXTRACT:
5265       {
5266       int i;
5267       int field_number = 1;
5268       uschar *save_lookup_value = lookup_value;
5269       uschar *sub[2];
5270       int save_expand_nmax =
5271         save_expand_strings(save_expand_nstring, save_expand_nlength);
5272
5273       /* Read the field & list arguments */
5274
5275       for (i = 0; i < 2; i++)
5276         {
5277         while (isspace(*s)) s++;
5278         if (*s != '{')                                  /*}*/
5279           goto EXPAND_FAILED_CURLY;
5280
5281         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5282         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5283         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5284
5285         /* After removal of leading and trailing white space, the first
5286         argument must be numeric and nonempty. */
5287
5288         if (i == 0)
5289           {
5290           int len;
5291           int x = 0;
5292           uschar *p = sub[0];
5293
5294           while (isspace(*p)) p++;
5295           sub[0] = p;
5296
5297           len = Ustrlen(p);
5298           while (len > 0 && isspace(p[len-1])) len--;
5299           p[len] = 0;
5300
5301           if (!*p && !skipping)
5302             {
5303             expand_string_message = US"first argument of \"listextract\" must "
5304               "not be empty";
5305             goto EXPAND_FAILED;
5306             }
5307
5308           if (*p == '-')
5309             {
5310             field_number = -1;
5311             p++;
5312             }
5313           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5314           if (*p)
5315             {
5316             expand_string_message = US"first argument of \"listextract\" must "
5317               "be numeric";
5318             goto EXPAND_FAILED;
5319             }
5320           field_number *= x;
5321           }
5322         }
5323
5324       /* Extract the numbered element into $value. If
5325       skipping, just pretend the extraction failed. */
5326
5327       lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]);
5328
5329       /* If no string follows, $value gets substituted; otherwise there can
5330       be yes/no strings, as for lookup or if. */
5331
5332       switch(process_yesno(
5333                skipping,                     /* were previously skipping */
5334                lookup_value != NULL,         /* success/failure indicator */
5335                save_lookup_value,            /* value to reset for string2 */
5336                &s,                           /* input pointer */
5337                &yield,                       /* output pointer */
5338                &size,                        /* output size */
5339                &ptr,                         /* output current point */
5340                US"extract",                  /* condition type */
5341                &resetok))
5342         {
5343         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5344         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5345         }
5346
5347       /* All done - restore numerical variables. */
5348
5349       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5350         save_expand_nlength);
5351
5352       continue;
5353       }
5354
5355 #ifdef SUPPORT_TLS
5356     case EITEM_CERTEXTRACT:
5357       {
5358       int i;
5359       int field_number = 1;
5360       uschar *save_lookup_value = lookup_value;
5361       uschar *sub[2];
5362       int save_expand_nmax =
5363         save_expand_strings(save_expand_nstring, save_expand_nlength);
5364
5365       /* Read the field argument */
5366       while (isspace(*s)) s++;
5367       if (*s != '{')                                    /*}*/
5368         goto EXPAND_FAILED_CURLY;
5369       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5370       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
5371       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5372       /* strip spaces fore & aft */
5373       {
5374       int len;
5375       int x = 0;
5376       uschar *p = sub[0];
5377
5378       while (isspace(*p)) p++;
5379       sub[0] = p;
5380
5381       len = Ustrlen(p);
5382       while (len > 0 && isspace(p[len-1])) len--;
5383       p[len] = 0;
5384       }
5385
5386       /* inspect the cert argument */
5387       while (isspace(*s)) s++;
5388       if (*s != '{')                                    /*}*/
5389         goto EXPAND_FAILED_CURLY;
5390       if (*++s != '$')
5391         {
5392         expand_string_message = US"second argument of \"certextract\" must "
5393           "be a certificate variable";
5394         goto EXPAND_FAILED;
5395         }
5396       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
5397       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
5398       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5399
5400       if (skipping)
5401         lookup_value = NULL;
5402       else
5403         {
5404         lookup_value = expand_getcertele(sub[0], sub[1]);
5405         if (*expand_string_message) goto EXPAND_FAILED;
5406         }
5407       switch(process_yesno(
5408                skipping,                     /* were previously skipping */
5409                lookup_value != NULL,         /* success/failure indicator */
5410                save_lookup_value,            /* value to reset for string2 */
5411                &s,                           /* input pointer */
5412                &yield,                       /* output pointer */
5413                &size,                        /* output size */
5414                &ptr,                         /* output current point */
5415                US"extract",                  /* condition type */
5416                &resetok))
5417         {
5418         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5419         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5420         }
5421
5422       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5423         save_expand_nlength);
5424       continue;
5425       }
5426 #endif  /*SUPPORT_TLS*/
5427
5428     /* Handle list operations */
5429
5430     case EITEM_FILTER:
5431     case EITEM_MAP:
5432     case EITEM_REDUCE:
5433       {
5434       int sep = 0;
5435       int save_ptr = ptr;
5436       uschar outsep[2] = { '\0', '\0' };
5437       uschar *list, *expr, *temp;
5438       uschar *save_iterate_item = iterate_item;
5439       uschar *save_lookup_value = lookup_value;
5440
5441       while (isspace(*s)) s++;
5442       if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5443
5444       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5445       if (list == NULL) goto EXPAND_FAILED;
5446       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5447
5448       if (item_type == EITEM_REDUCE)
5449         {
5450         while (isspace(*s)) s++;
5451         if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5452         temp = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5453         if (temp == NULL) goto EXPAND_FAILED;
5454         lookup_value = temp;
5455         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5456         }
5457
5458       while (isspace(*s)) s++;
5459       if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5460
5461       expr = s;
5462
5463       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
5464       if scanning a "false" part). This allows us to find the end of the
5465       condition, because if the list is empty, we won't actually evaluate the
5466       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
5467       the normal internal expansion function. */
5468
5469       if (item_type == EITEM_FILTER)
5470         {
5471         temp = eval_condition(expr, &resetok, NULL);
5472         if (temp != NULL) s = temp;
5473         }
5474       else
5475         {
5476         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5477         }
5478
5479       if (temp == NULL)
5480         {
5481         expand_string_message = string_sprintf("%s inside \"%s\" item",
5482           expand_string_message, name);
5483         goto EXPAND_FAILED;
5484         }
5485
5486       while (isspace(*s)) s++;
5487       if (*s++ != '}')
5488         {                                               /*{*/
5489         expand_string_message = string_sprintf("missing } at end of condition "
5490           "or expression inside \"%s\"", name);
5491         goto EXPAND_FAILED;
5492         }
5493
5494       while (isspace(*s)) s++;                          /*{*/
5495       if (*s++ != '}')
5496         {                                               /*{*/
5497         expand_string_message = string_sprintf("missing } at end of \"%s\"",
5498           name);
5499         goto EXPAND_FAILED;
5500         }
5501
5502       /* If we are skipping, we can now just move on to the next item. When
5503       processing for real, we perform the iteration. */
5504
5505       if (skipping) continue;
5506       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
5507         {
5508         *outsep = (uschar)sep;      /* Separator as a string */
5509
5510         DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
5511
5512         if (item_type == EITEM_FILTER)
5513           {
5514           BOOL condresult;
5515           if (eval_condition(expr, &resetok, &condresult) == NULL)
5516             {
5517             iterate_item = save_iterate_item;
5518             lookup_value = save_lookup_value;
5519             expand_string_message = string_sprintf("%s inside \"%s\" condition",
5520               expand_string_message, name);
5521             goto EXPAND_FAILED;
5522             }
5523           DEBUG(D_expand) debug_printf("%s: condition is %s\n", name,
5524             condresult? "true":"false");
5525           if (condresult)
5526             temp = iterate_item;    /* TRUE => include this item */
5527           else
5528             continue;               /* FALSE => skip this item */
5529           }
5530
5531         /* EITEM_MAP and EITEM_REDUCE */
5532
5533         else
5534           {
5535           temp = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
5536           if (temp == NULL)
5537             {
5538             iterate_item = save_iterate_item;
5539             expand_string_message = string_sprintf("%s inside \"%s\" item",
5540               expand_string_message, name);
5541             goto EXPAND_FAILED;
5542             }
5543           if (item_type == EITEM_REDUCE)
5544             {
5545             lookup_value = temp;      /* Update the value of $value */
5546             continue;                 /* and continue the iteration */
5547             }
5548           }
5549
5550         /* We reach here for FILTER if the condition is true, always for MAP,
5551         and never for REDUCE. The value in "temp" is to be added to the output
5552         list that is being created, ensuring that any occurrences of the
5553         separator character are doubled. Unless we are dealing with the first
5554         item of the output list, add in a space if the new item begins with the
5555         separator character, or is an empty string. */
5556
5557         if (ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
5558           yield = string_cat(yield, &size, &ptr, US" ", 1);
5559
5560         /* Add the string in "temp" to the output list that we are building,
5561         This is done in chunks by searching for the separator character. */
5562
5563         for (;;)
5564           {
5565           size_t seglen = Ustrcspn(temp, outsep);
5566             yield = string_cat(yield, &size, &ptr, temp, seglen + 1);
5567
5568           /* If we got to the end of the string we output one character
5569           too many; backup and end the loop. Otherwise arrange to double the
5570           separator. */
5571
5572           if (temp[seglen] == '\0') { ptr--; break; }
5573           yield = string_cat(yield, &size, &ptr, outsep, 1);
5574           temp += seglen + 1;
5575           }
5576
5577         /* Output a separator after the string: we will remove the redundant
5578         final one at the end. */
5579
5580         yield = string_cat(yield, &size, &ptr, outsep, 1);
5581         }   /* End of iteration over the list loop */
5582
5583       /* REDUCE has generated no output above: output the final value of
5584       $value. */
5585
5586       if (item_type == EITEM_REDUCE)
5587         {
5588         yield = string_cat(yield, &size, &ptr, lookup_value,
5589           Ustrlen(lookup_value));
5590         lookup_value = save_lookup_value;  /* Restore $value */
5591         }
5592
5593       /* FILTER and MAP generate lists: if they have generated anything, remove
5594       the redundant final separator. Even though an empty item at the end of a
5595       list does not count, this is tidier. */
5596
5597       else if (ptr != save_ptr) ptr--;
5598
5599       /* Restore preserved $item */
5600
5601       iterate_item = save_iterate_item;
5602       continue;
5603       }
5604
5605
5606     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
5607     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
5608     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
5609     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
5610
5611     #define EXPAND_DLFUNC_MAX_ARGS 8
5612
5613     case EITEM_DLFUNC:
5614     #ifndef EXPAND_DLFUNC
5615     expand_string_message = US"\"${dlfunc\" encountered, but this facility "    /*}*/
5616       "is not included in this binary";
5617     goto EXPAND_FAILED;
5618
5619     #else   /* EXPAND_DLFUNC */
5620       {
5621       tree_node *t;
5622       exim_dlfunc_t *func;
5623       uschar *result;
5624       int status, argc;
5625       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
5626
5627       if ((expand_forbid & RDO_DLFUNC) != 0)
5628         {
5629         expand_string_message =
5630           US"dynamically-loaded functions are not permitted";
5631         goto EXPAND_FAILED;
5632         }
5633
5634       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
5635            TRUE, US"dlfunc", &resetok))
5636         {
5637         case 1: goto EXPAND_FAILED_CURLY;
5638         case 2:
5639         case 3: goto EXPAND_FAILED;
5640         }
5641
5642       /* If skipping, we don't actually do anything */
5643
5644       if (skipping) continue;
5645
5646       /* Look up the dynamically loaded object handle in the tree. If it isn't
5647       found, dlopen() the file and put the handle in the tree for next time. */
5648
5649       t = tree_search(dlobj_anchor, argv[0]);
5650       if (t == NULL)
5651         {
5652         void *handle = dlopen(CS argv[0], RTLD_LAZY);
5653         if (handle == NULL)
5654           {
5655           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
5656             argv[0], dlerror());
5657           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
5658           goto EXPAND_FAILED;
5659           }
5660         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
5661         Ustrcpy(t->name, argv[0]);
5662         t->data.ptr = handle;
5663         (void)tree_insertnode(&dlobj_anchor, t);
5664         }
5665
5666       /* Having obtained the dynamically loaded object handle, look up the
5667       function pointer. */
5668
5669       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
5670       if (func == NULL)
5671         {
5672         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
5673           "%s", argv[1], argv[0], dlerror());
5674         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
5675         goto EXPAND_FAILED;
5676         }
5677
5678       /* Call the function and work out what to do with the result. If it
5679       returns OK, we have a replacement string; if it returns DEFER then
5680       expansion has failed in a non-forced manner; if it returns FAIL then
5681       failure was forced; if it returns ERROR or any other value there's a
5682       problem, so panic slightly. In any case, assume that the function has
5683       side-effects on the store that must be preserved. */
5684
5685       resetok = FALSE;
5686       result = NULL;
5687       for (argc = 0; argv[argc] != NULL; argc++);
5688       status = func(&result, argc - 2, &argv[2]);
5689       if(status == OK)
5690         {
5691         if (result == NULL) result = US"";
5692         yield = string_cat(yield, &size, &ptr, result, Ustrlen(result));
5693         continue;
5694         }
5695       else
5696         {
5697         expand_string_message = result == NULL ? US"(no message)" : result;
5698         if(status == FAIL_FORCED) expand_string_forcedfail = TRUE;
5699           else if(status != FAIL)
5700             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
5701               argv[0], argv[1], status, expand_string_message);
5702         goto EXPAND_FAILED;
5703         }
5704       }
5705     #endif /* EXPAND_DLFUNC */
5706     }   /* EITEM_* switch */
5707
5708   /* Control reaches here if the name is not recognized as one of the more
5709   complicated expansion items. Check for the "operator" syntax (name terminated
5710   by a colon). Some of the operators have arguments, separated by _ from the
5711   name. */
5712
5713   if (*s == ':')
5714     {
5715     int c;
5716     uschar *arg = NULL;
5717     uschar *sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5718     if (sub == NULL) goto EXPAND_FAILED;
5719     s++;
5720
5721     /* Owing to an historical mis-design, an underscore may be part of the
5722     operator name, or it may introduce arguments.  We therefore first scan the
5723     table of names that contain underscores. If there is no match, we cut off
5724     the arguments and then scan the main table. */
5725
5726     c = chop_match(name, op_table_underscore,
5727       sizeof(op_table_underscore)/sizeof(uschar *));
5728
5729     if (c < 0)
5730       {
5731       arg = Ustrchr(name, '_');
5732       if (arg != NULL) *arg = 0;
5733       c = chop_match(name, op_table_main,
5734         sizeof(op_table_main)/sizeof(uschar *));
5735       if (c >= 0) c += sizeof(op_table_underscore)/sizeof(uschar *);
5736       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
5737       }
5738
5739     /* If we are skipping, we don't need to perform the operation at all.
5740     This matters for operations like "mask", because the data may not be
5741     in the correct format when skipping. For example, the expression may test
5742     for the existence of $sender_host_address before trying to mask it. For
5743     other operations, doing them may not fail, but it is a waste of time. */
5744
5745     if (skipping && c >= 0) continue;
5746
5747     /* Otherwise, switch on the operator type */
5748
5749     switch(c)
5750       {
5751       case EOP_BASE62:
5752         {
5753         uschar *t;
5754         unsigned long int n = Ustrtoul(sub, &t, 10);
5755         if (*t != 0)
5756           {
5757           expand_string_message = string_sprintf("argument for base62 "
5758             "operator is \"%s\", which is not a decimal number", sub);
5759           goto EXPAND_FAILED;
5760           }
5761         t = string_base62(n);
5762         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
5763         continue;
5764         }
5765
5766       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
5767
5768       case EOP_BASE62D:
5769         {
5770         uschar buf[16];
5771         uschar *tt = sub;
5772         unsigned long int n = 0;
5773         while (*tt != 0)
5774           {
5775           uschar *t = Ustrchr(base62_chars, *tt++);
5776           if (t == NULL)
5777             {
5778             expand_string_message = string_sprintf("argument for base62d "
5779               "operator is \"%s\", which is not a base %d number", sub,
5780               BASE_62);
5781             goto EXPAND_FAILED;
5782             }
5783           n = n * BASE_62 + (t - base62_chars);
5784           }
5785         (void)sprintf(CS buf, "%ld", n);
5786         yield = string_cat(yield, &size, &ptr, buf, Ustrlen(buf));
5787         continue;
5788         }
5789
5790       case EOP_EXPAND:
5791         {
5792         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
5793         if (expanded == NULL)
5794           {
5795           expand_string_message =
5796             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
5797               expand_string_message);
5798           goto EXPAND_FAILED;
5799           }
5800         yield = string_cat(yield, &size, &ptr, expanded, Ustrlen(expanded));
5801         continue;
5802         }
5803
5804       case EOP_LC:
5805         {
5806         int count = 0;
5807         uschar *t = sub - 1;
5808         while (*(++t) != 0) { *t = tolower(*t); count++; }
5809         yield = string_cat(yield, &size, &ptr, sub, count);
5810         continue;
5811         }
5812
5813       case EOP_UC:
5814         {
5815         int count = 0;
5816         uschar *t = sub - 1;
5817         while (*(++t) != 0) { *t = toupper(*t); count++; }
5818         yield = string_cat(yield, &size, &ptr, sub, count);
5819         continue;
5820         }
5821
5822       case EOP_MD5:
5823         {
5824         md5 base;
5825         uschar digest[16];
5826         int j;
5827         char st[33];
5828         md5_start(&base);
5829         md5_end(&base, sub, Ustrlen(sub), digest);
5830         for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]);
5831         yield = string_cat(yield, &size, &ptr, US st, (int)strlen(st));
5832         continue;
5833         }
5834
5835       case EOP_SHA1:
5836         {
5837         sha1 base;
5838         uschar digest[20];
5839         int j;
5840         char st[41];
5841         sha1_start(&base);
5842         sha1_end(&base, sub, Ustrlen(sub), digest);
5843         for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
5844         yield = string_cat(yield, &size, &ptr, US st, (int)strlen(st));
5845         continue;
5846         }
5847
5848       /* Convert hex encoding to base64 encoding */
5849
5850       case EOP_HEX2B64:
5851         {
5852         int c = 0;
5853         int b = -1;
5854         uschar *in = sub;
5855         uschar *out = sub;
5856         uschar *enc;
5857
5858         for (enc = sub; *enc != 0; enc++)
5859           {
5860           if (!isxdigit(*enc))
5861             {
5862             expand_string_message = string_sprintf("\"%s\" is not a hex "
5863               "string", sub);
5864             goto EXPAND_FAILED;
5865             }
5866           c++;
5867           }
5868
5869         if ((c & 1) != 0)
5870           {
5871           expand_string_message = string_sprintf("\"%s\" contains an odd "
5872             "number of characters", sub);
5873           goto EXPAND_FAILED;
5874           }
5875
5876         while ((c = *in++) != 0)
5877           {
5878           if (isdigit(c)) c -= '0';
5879           else c = toupper(c) - 'A' + 10;
5880           if (b == -1)
5881             {
5882             b = c << 4;
5883             }
5884           else
5885             {
5886             *out++ = b | c;
5887             b = -1;
5888             }
5889           }
5890
5891         enc = auth_b64encode(sub, out - sub);
5892         yield = string_cat(yield, &size, &ptr, enc, Ustrlen(enc));
5893         continue;
5894         }
5895
5896       /* Convert octets outside 0x21..0x7E to \xXX form */
5897
5898       case EOP_HEXQUOTE:
5899         {
5900         uschar *t = sub - 1;
5901         while (*(++t) != 0)
5902           {
5903           if (*t < 0x21 || 0x7E < *t)
5904             yield = string_cat(yield, &size, &ptr,
5905               string_sprintf("\\x%02x", *t), 4);
5906           else
5907             yield = string_cat(yield, &size, &ptr, t, 1);
5908           }
5909         continue;
5910         }
5911
5912       /* count the number of list elements */
5913
5914       case EOP_LISTCOUNT:
5915         {
5916         int cnt = 0;
5917         int sep = 0;
5918         uschar * cp;
5919         uschar buffer[256];
5920
5921         while (string_nextinlist(&sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
5922         cp = string_sprintf("%d", cnt);
5923         yield = string_cat(yield, &size, &ptr, cp, Ustrlen(cp));
5924         continue;
5925         }
5926
5927       /* expand a named list given the name */
5928       /* handles nested named lists; requotes as colon-sep list */
5929
5930       case EOP_LISTNAMED:
5931         {
5932         tree_node *t = NULL;
5933         uschar * list;
5934         int sep = 0;
5935         uschar * item;
5936         uschar * suffix = US"";
5937         BOOL needsep = FALSE;
5938         uschar buffer[256];
5939
5940         if (*sub == '+') sub++;
5941         if (arg == NULL)        /* no-argument version */
5942           {
5943           if (!(t = tree_search(addresslist_anchor, sub)) &&
5944               !(t = tree_search(domainlist_anchor,  sub)) &&
5945               !(t = tree_search(hostlist_anchor,    sub)))
5946             t = tree_search(localpartlist_anchor, sub);
5947           }
5948         else switch(*arg)       /* specific list-type version */
5949           {
5950           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
5951           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
5952           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
5953           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
5954           default:
5955             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
5956             goto EXPAND_FAILED;
5957           }
5958
5959         if(!t)
5960           {
5961           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
5962             sub, !arg?""
5963               : *arg=='a'?"address "
5964               : *arg=='d'?"domain "
5965               : *arg=='h'?"host "
5966               : *arg=='l'?"localpart "
5967               : 0);
5968           goto EXPAND_FAILED;
5969           }
5970
5971         list = ((namedlist_block *)(t->data.ptr))->string;
5972
5973         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) != NULL)
5974           {
5975           uschar * buf = US" : ";
5976           if (needsep)
5977             yield = string_cat(yield, &size, &ptr, buf, 3);
5978           else
5979             needsep = TRUE;
5980
5981           if (*item == '+')     /* list item is itself a named list */
5982             {
5983             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
5984             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
5985             }
5986           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
5987             {
5988             char * cp;
5989             char tok[3];
5990             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
5991             while ((cp= strpbrk((const char *)item, tok)))
5992               {
5993               yield = string_cat(yield, &size, &ptr, item, cp-(char *)item);
5994               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
5995                 {
5996                 yield = string_cat(yield, &size, &ptr, US"::", 2);
5997                 item = (uschar *)cp;
5998                 }
5999               else              /* sep in item; should already be doubled; emit once */
6000                 {
6001                 yield = string_cat(yield, &size, &ptr, (uschar *)tok, 1);
6002                 if (*cp == sep) cp++;
6003                 item = (uschar *)cp;
6004                 }
6005               }
6006             }
6007           yield = string_cat(yield, &size, &ptr, item, Ustrlen(item));
6008           }
6009         continue;
6010         }
6011
6012       /* mask applies a mask to an IP address; for example the result of
6013       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
6014
6015       case EOP_MASK:
6016         {
6017         int count;
6018         uschar *endptr;
6019         int binary[4];
6020         int mask, maskoffset;
6021         int type = string_is_ip_address(sub, &maskoffset);
6022         uschar buffer[64];
6023
6024         if (type == 0)
6025           {
6026           expand_string_message = string_sprintf("\"%s\" is not an IP address",
6027            sub);
6028           goto EXPAND_FAILED;
6029           }
6030
6031         if (maskoffset == 0)
6032           {
6033           expand_string_message = string_sprintf("missing mask value in \"%s\"",
6034             sub);
6035           goto EXPAND_FAILED;
6036           }
6037
6038         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
6039
6040         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
6041           {
6042           expand_string_message = string_sprintf("mask value too big in \"%s\"",
6043             sub);
6044           goto EXPAND_FAILED;
6045           }
6046
6047         /* Convert the address to binary integer(s) and apply the mask */
6048
6049         sub[maskoffset] = 0;
6050         count = host_aton(sub, binary);
6051         host_mask(count, binary, mask);
6052
6053         /* Convert to masked textual format and add to output. */
6054
6055         yield = string_cat(yield, &size, &ptr, buffer,
6056           host_nmtoa(count, binary, mask, buffer, '.'));
6057         continue;
6058         }
6059
6060       case EOP_ADDRESS:
6061       case EOP_LOCAL_PART:
6062       case EOP_DOMAIN:
6063         {
6064         uschar *error;
6065         int start, end, domain;
6066         uschar *t = parse_extract_address(sub, &error, &start, &end, &domain,
6067           FALSE);
6068         if (t != NULL)
6069           {
6070           if (c != EOP_DOMAIN)
6071             {
6072             if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1;
6073             yield = string_cat(yield, &size, &ptr, sub+start, end-start);
6074             }
6075           else if (domain != 0)
6076             {
6077             domain += start;
6078             yield = string_cat(yield, &size, &ptr, sub+domain, end-domain);
6079             }
6080           }
6081         continue;
6082         }
6083
6084       case EOP_ADDRESSES:
6085         {
6086         uschar outsep[2] = { ':', '\0' };
6087         uschar *address, *error;
6088         int save_ptr = ptr;
6089         int start, end, domain;  /* Not really used */
6090
6091         while (isspace(*sub)) sub++;
6092         if (*sub == '>') { *outsep = *++sub; ++sub; }
6093         parse_allow_group = TRUE;
6094
6095         for (;;)
6096           {
6097           uschar *p = parse_find_address_end(sub, FALSE);
6098           uschar saveend = *p;
6099           *p = '\0';
6100           address = parse_extract_address(sub, &error, &start, &end, &domain,
6101             FALSE);
6102           *p = saveend;
6103
6104           /* Add the address to the output list that we are building. This is
6105           done in chunks by searching for the separator character. At the
6106           start, unless we are dealing with the first address of the output
6107           list, add in a space if the new address begins with the separator
6108           character, or is an empty string. */
6109
6110           if (address != NULL)
6111             {
6112             if (ptr != save_ptr && address[0] == *outsep)
6113               yield = string_cat(yield, &size, &ptr, US" ", 1);
6114
6115             for (;;)
6116               {
6117               size_t seglen = Ustrcspn(address, outsep);
6118               yield = string_cat(yield, &size, &ptr, address, seglen + 1);
6119
6120               /* If we got to the end of the string we output one character
6121               too many. */
6122
6123               if (address[seglen] == '\0') { ptr--; break; }
6124               yield = string_cat(yield, &size, &ptr, outsep, 1);
6125               address += seglen + 1;
6126               }
6127
6128             /* Output a separator after the string: we will remove the
6129             redundant final one at the end. */
6130
6131             yield = string_cat(yield, &size, &ptr, outsep, 1);
6132             }
6133
6134           if (saveend == '\0') break;
6135           sub = p + 1;
6136           }
6137
6138         /* If we have generated anything, remove the redundant final
6139         separator. */
6140
6141         if (ptr != save_ptr) ptr--;
6142         parse_allow_group = FALSE;
6143         continue;
6144         }
6145
6146
6147       /* quote puts a string in quotes if it is empty or contains anything
6148       other than alphamerics, underscore, dot, or hyphen.
6149
6150       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
6151       be quoted in order to be a valid local part.
6152
6153       In both cases, newlines and carriage returns are converted into \n and \r
6154       respectively */
6155
6156       case EOP_QUOTE:
6157       case EOP_QUOTE_LOCAL_PART:
6158       if (arg == NULL)
6159         {
6160         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
6161         uschar *t = sub - 1;
6162
6163         if (c == EOP_QUOTE)
6164           {
6165           while (!needs_quote && *(++t) != 0)
6166             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
6167           }
6168         else  /* EOP_QUOTE_LOCAL_PART */
6169           {
6170           while (!needs_quote && *(++t) != 0)
6171             needs_quote = !isalnum(*t) &&
6172               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
6173               (*t != '.' || t == sub || t[1] == 0);
6174           }
6175
6176         if (needs_quote)
6177           {
6178           yield = string_cat(yield, &size, &ptr, US"\"", 1);
6179           t = sub - 1;
6180           while (*(++t) != 0)
6181             {
6182             if (*t == '\n')
6183               yield = string_cat(yield, &size, &ptr, US"\\n", 2);
6184             else if (*t == '\r')
6185               yield = string_cat(yield, &size, &ptr, US"\\r", 2);
6186             else
6187               {
6188               if (*t == '\\' || *t == '"')
6189                 yield = string_cat(yield, &size, &ptr, US"\\", 1);
6190               yield = string_cat(yield, &size, &ptr, t, 1);
6191               }
6192             }
6193           yield = string_cat(yield, &size, &ptr, US"\"", 1);
6194           }
6195         else yield = string_cat(yield, &size, &ptr, sub, Ustrlen(sub));
6196         continue;
6197         }
6198
6199       /* quote_lookuptype does lookup-specific quoting */
6200
6201       else
6202         {
6203         int n;
6204         uschar *opt = Ustrchr(arg, '_');
6205
6206         if (opt != NULL) *opt++ = 0;
6207
6208         n = search_findtype(arg, Ustrlen(arg));
6209         if (n < 0)
6210           {
6211           expand_string_message = search_error_message;
6212           goto EXPAND_FAILED;
6213           }
6214
6215         if (lookup_list[n]->quote != NULL)
6216           sub = (lookup_list[n]->quote)(sub, opt);
6217         else if (opt != NULL) sub = NULL;
6218
6219         if (sub == NULL)
6220           {
6221           expand_string_message = string_sprintf(
6222             "\"%s\" unrecognized after \"${quote_%s\"",
6223             opt, arg);
6224           goto EXPAND_FAILED;
6225           }
6226
6227         yield = string_cat(yield, &size, &ptr, sub, Ustrlen(sub));
6228         continue;
6229         }
6230
6231       /* rx quote sticks in \ before any non-alphameric character so that
6232       the insertion works in a regular expression. */
6233
6234       case EOP_RXQUOTE:
6235         {
6236         uschar *t = sub - 1;
6237         while (*(++t) != 0)
6238           {
6239           if (!isalnum(*t))
6240             yield = string_cat(yield, &size, &ptr, US"\\", 1);
6241           yield = string_cat(yield, &size, &ptr, t, 1);
6242           }
6243         continue;
6244         }
6245
6246       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
6247       prescribed by the RFC, if there are characters that need to be encoded */
6248
6249       case EOP_RFC2047:
6250         {
6251         uschar buffer[2048];
6252         uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset,
6253           buffer, sizeof(buffer), FALSE);
6254         yield = string_cat(yield, &size, &ptr, string, Ustrlen(string));
6255         continue;
6256         }
6257
6258       /* RFC 2047 decode */
6259
6260       case EOP_RFC2047D:
6261         {
6262         int len;
6263         uschar *error;
6264         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
6265           headers_charset, '?', &len, &error);
6266         if (error != NULL)
6267           {
6268           expand_string_message = error;
6269           goto EXPAND_FAILED;
6270           }
6271         yield = string_cat(yield, &size, &ptr, decoded, len);
6272         continue;
6273         }
6274
6275       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
6276       underscores */
6277
6278       case EOP_FROM_UTF8:
6279         {
6280         while (*sub != 0)
6281           {
6282           int c;
6283           uschar buff[4];
6284           GETUTF8INC(c, sub);
6285           if (c > 255) c = '_';
6286           buff[0] = c;
6287           yield = string_cat(yield, &size, &ptr, buff, 1);
6288           }
6289         continue;
6290         }
6291
6292           /* replace illegal UTF-8 sequences by replacement character  */
6293
6294       #define UTF8_REPLACEMENT_CHAR US"?"
6295
6296       case EOP_UTF8CLEAN:
6297         {
6298         int seq_len, index = 0;
6299         int bytes_left  = 0;
6300         uschar seq_buff[4];                     /* accumulate utf-8 here */
6301
6302         while (*sub != 0)
6303           {
6304           int complete;
6305           long codepoint;
6306           uschar c;
6307
6308           complete = 0;
6309           c = *sub++;
6310           if(bytes_left)
6311             {
6312             if ((c & 0xc0) != 0x80)
6313               {
6314                     /* wrong continuation byte; invalidate all bytes */
6315               complete = 1; /* error */
6316               }
6317             else
6318               {
6319               codepoint = (codepoint << 6) | (c & 0x3f);
6320               seq_buff[index++] = c;
6321               if (--bytes_left == 0)            /* codepoint complete */
6322                 {
6323                 if(codepoint > 0x10FFFF)        /* is it too large? */
6324                   complete = -1;        /* error */
6325                 else
6326                   {             /* finished; output utf-8 sequence */
6327                   yield = string_cat(yield, &size, &ptr, seq_buff, seq_len);
6328                   index = 0;
6329                   }
6330                 }
6331               }
6332             }
6333           else  /* no bytes left: new sequence */
6334             {
6335             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
6336               {
6337               yield = string_cat(yield, &size, &ptr, &c, 1);
6338               continue;
6339               }
6340             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
6341               {
6342               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
6343                 complete = -1;
6344               else
6345                 {
6346                   bytes_left = 1;
6347                   codepoint = c & 0x1f;
6348                 }
6349               }
6350             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
6351               {
6352               bytes_left = 2;
6353               codepoint = c & 0x0f;
6354               }
6355             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
6356               {
6357               bytes_left = 3;
6358               codepoint = c & 0x07;
6359               }
6360             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
6361               complete = -1;
6362
6363             seq_buff[index++] = c;
6364             seq_len = bytes_left + 1;
6365             }           /* if(bytes_left) */
6366
6367           if (complete != 0)
6368             {
6369             bytes_left = index = 0;
6370             yield = string_cat(yield, &size, &ptr, UTF8_REPLACEMENT_CHAR, 1);
6371             }
6372           if ((complete == 1) && ((c & 0x80) == 0))
6373             { /* ASCII character follows incomplete sequence */
6374               yield = string_cat(yield, &size, &ptr, &c, 1);
6375             }
6376           }
6377         continue;
6378         }
6379
6380       /* escape turns all non-printing characters into escape sequences. */
6381
6382       case EOP_ESCAPE:
6383         {
6384         uschar *t = string_printing(sub);
6385         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
6386         continue;
6387         }
6388
6389       /* Handle numeric expression evaluation */
6390
6391       case EOP_EVAL:
6392       case EOP_EVAL10:
6393         {
6394         uschar *save_sub = sub;
6395         uschar *error = NULL;
6396         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
6397         if (error != NULL)
6398           {
6399           expand_string_message = string_sprintf("error in expression "
6400             "evaluation: %s (after processing \"%.*s\")", error, sub-save_sub,
6401               save_sub);
6402           goto EXPAND_FAILED;
6403           }
6404         sprintf(CS var_buffer, PR_EXIM_ARITH, n);
6405         yield = string_cat(yield, &size, &ptr, var_buffer, Ustrlen(var_buffer));
6406         continue;
6407         }
6408
6409       /* Handle time period formating */
6410
6411       case EOP_TIME_EVAL:
6412         {
6413         int n = readconf_readtime(sub, 0, FALSE);
6414         if (n < 0)
6415           {
6416           expand_string_message = string_sprintf("string \"%s\" is not an "
6417             "Exim time interval in \"%s\" operator", sub, name);
6418           goto EXPAND_FAILED;
6419           }
6420         sprintf(CS var_buffer, "%d", n);
6421         yield = string_cat(yield, &size, &ptr, var_buffer, Ustrlen(var_buffer));
6422         continue;
6423         }
6424
6425       case EOP_TIME_INTERVAL:
6426         {
6427         int n;
6428         uschar *t = read_number(&n, sub);
6429         if (*t != 0) /* Not A Number*/
6430           {
6431           expand_string_message = string_sprintf("string \"%s\" is not a "
6432             "positive number in \"%s\" operator", sub, name);
6433           goto EXPAND_FAILED;
6434           }
6435         t = readconf_printtime(n);
6436         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
6437         continue;
6438         }
6439
6440       /* Convert string to base64 encoding */
6441
6442       case EOP_STR2B64:
6443         {
6444         uschar *encstr = auth_b64encode(sub, Ustrlen(sub));
6445         yield = string_cat(yield, &size, &ptr, encstr, Ustrlen(encstr));
6446         continue;
6447         }
6448
6449       /* strlen returns the length of the string */
6450
6451       case EOP_STRLEN:
6452         {
6453         uschar buff[24];
6454         (void)sprintf(CS buff, "%d", Ustrlen(sub));
6455         yield = string_cat(yield, &size, &ptr, buff, Ustrlen(buff));
6456         continue;
6457         }
6458
6459       /* length_n or l_n takes just the first n characters or the whole string,
6460       whichever is the shorter;
6461
6462       substr_m_n, and s_m_n take n characters from offset m; negative m take
6463       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
6464       takes the rest, either to the right or to the left.
6465
6466       hash_n or h_n makes a hash of length n from the string, yielding n
6467       characters from the set a-z; hash_n_m makes a hash of length n, but
6468       uses m characters from the set a-zA-Z0-9.
6469
6470       nhash_n returns a single number between 0 and n-1 (in text form), while
6471       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
6472       between 0 and n-1 and the second between 0 and m-1. */
6473
6474       case EOP_LENGTH:
6475       case EOP_L:
6476       case EOP_SUBSTR:
6477       case EOP_S:
6478       case EOP_HASH:
6479       case EOP_H:
6480       case EOP_NHASH:
6481       case EOP_NH:
6482         {
6483         int sign = 1;
6484         int value1 = 0;
6485         int value2 = -1;
6486         int *pn;
6487         int len;
6488         uschar *ret;
6489
6490         if (arg == NULL)
6491           {
6492           expand_string_message = string_sprintf("missing values after %s",
6493             name);
6494           goto EXPAND_FAILED;
6495           }
6496
6497         /* "length" has only one argument, effectively being synonymous with
6498         substr_0_n. */
6499
6500         if (c == EOP_LENGTH || c == EOP_L)
6501           {
6502           pn = &value2;
6503           value2 = 0;
6504           }
6505
6506         /* The others have one or two arguments; for "substr" the first may be
6507         negative. The second being negative means "not supplied". */
6508
6509         else
6510           {
6511           pn = &value1;
6512           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
6513           }
6514
6515         /* Read up to two numbers, separated by underscores */
6516
6517         ret = arg;
6518         while (*arg != 0)
6519           {
6520           if (arg != ret && *arg == '_' && pn == &value1)
6521             {
6522             pn = &value2;
6523             value2 = 0;
6524             if (arg[1] != 0) arg++;
6525             }
6526           else if (!isdigit(*arg))
6527             {
6528             expand_string_message =
6529               string_sprintf("non-digit after underscore in \"%s\"", name);
6530             goto EXPAND_FAILED;
6531             }
6532           else *pn = (*pn)*10 + *arg++ - '0';
6533           }
6534         value1 *= sign;
6535
6536         /* Perform the required operation */
6537
6538         ret =
6539           (c == EOP_HASH || c == EOP_H)?
6540              compute_hash(sub, value1, value2, &len) :
6541           (c == EOP_NHASH || c == EOP_NH)?
6542              compute_nhash(sub, value1, value2, &len) :
6543              extract_substr(sub, value1, value2, &len);
6544
6545         if (ret == NULL) goto EXPAND_FAILED;
6546         yield = string_cat(yield, &size, &ptr, ret, len);
6547         continue;
6548         }
6549
6550       /* Stat a path */
6551
6552       case EOP_STAT:
6553         {
6554         uschar *s;
6555         uschar smode[12];
6556         uschar **modetable[3];
6557         int i;
6558         mode_t mode;
6559         struct stat st;
6560
6561         if ((expand_forbid & RDO_EXISTS) != 0)
6562           {
6563           expand_string_message = US"Use of the stat() expansion is not permitted";
6564           goto EXPAND_FAILED;
6565           }
6566
6567         if (stat(CS sub, &st) < 0)
6568           {
6569           expand_string_message = string_sprintf("stat(%s) failed: %s",
6570             sub, strerror(errno));
6571           goto EXPAND_FAILED;
6572           }
6573         mode = st.st_mode;
6574         switch (mode & S_IFMT)
6575           {
6576           case S_IFIFO: smode[0] = 'p'; break;
6577           case S_IFCHR: smode[0] = 'c'; break;
6578           case S_IFDIR: smode[0] = 'd'; break;
6579           case S_IFBLK: smode[0] = 'b'; break;
6580           case S_IFREG: smode[0] = '-'; break;
6581           default: smode[0] = '?'; break;
6582           }
6583
6584         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
6585         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
6586         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
6587
6588         for (i = 0; i < 3; i++)
6589           {
6590           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
6591           mode >>= 3;
6592           }
6593
6594         smode[10] = 0;
6595         s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
6596           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
6597           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
6598           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
6599           (long)st.st_gid, st.st_size, (long)st.st_atime,
6600           (long)st.st_mtime, (long)st.st_ctime);
6601         yield = string_cat(yield, &size, &ptr, s, Ustrlen(s));
6602         continue;
6603         }
6604
6605       /* vaguely random number less than N */
6606
6607       case EOP_RANDINT:
6608         {
6609         int_eximarith_t max;
6610         uschar *s;
6611
6612         max = expand_string_integer(sub, TRUE);
6613         if (expand_string_message != NULL)
6614           goto EXPAND_FAILED;
6615         s = string_sprintf("%d", vaguely_random_number((int)max));
6616         yield = string_cat(yield, &size, &ptr, s, Ustrlen(s));
6617         continue;
6618         }
6619
6620       /* Reverse IP, including IPv6 to dotted-nibble */
6621
6622       case EOP_REVERSE_IP:
6623         {
6624         int family, maskptr;
6625         uschar reversed[128];
6626
6627         family = string_is_ip_address(sub, &maskptr);
6628         if (family == 0)
6629           {
6630           expand_string_message = string_sprintf(
6631               "reverse_ip() not given an IP address [%s]", sub);
6632           goto EXPAND_FAILED;
6633           }
6634         invert_address(reversed, sub);
6635         yield = string_cat(yield, &size, &ptr, reversed, Ustrlen(reversed));
6636         continue;
6637         }
6638
6639       /* Unknown operator */
6640
6641       default:
6642       expand_string_message =
6643         string_sprintf("unknown expansion operator \"%s\"", name);
6644       goto EXPAND_FAILED;
6645       }
6646     }
6647
6648   /* Handle a plain name. If this is the first thing in the expansion, release
6649   the pre-allocated buffer. If the result data is known to be in a new buffer,
6650   newsize will be set to the size of that buffer, and we can just point at that
6651   store instead of copying. Many expansion strings contain just one reference,
6652   so this is a useful optimization, especially for humungous headers
6653   ($message_headers). */
6654                                                 /*{*/
6655   if (*s++ == '}')
6656     {
6657     int len;
6658     int newsize = 0;
6659     if (ptr == 0)
6660       {
6661       if (resetok) store_reset(yield);
6662       yield = NULL;
6663       size = 0;
6664       }
6665     value = find_variable(name, FALSE, skipping, &newsize);
6666     if (value == NULL)
6667       {
6668       expand_string_message =
6669         string_sprintf("unknown variable in \"${%s}\"", name);
6670       check_variable_error_message(name);
6671       goto EXPAND_FAILED;
6672       }
6673     len = Ustrlen(value);
6674     if (yield == NULL && newsize != 0)
6675       {
6676       yield = value;
6677       size = newsize;
6678       ptr = len;
6679       }
6680     else yield = string_cat(yield, &size, &ptr, value, len);
6681     continue;
6682     }
6683
6684   /* Else there's something wrong */
6685
6686   expand_string_message =
6687     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
6688     "in a variable reference)", name);
6689   goto EXPAND_FAILED;
6690   }
6691
6692 /* If we hit the end of the string when ket_ends is set, there is a missing
6693 terminating brace. */
6694
6695 if (ket_ends && *s == 0)
6696   {
6697   expand_string_message = malformed_header?
6698     US"missing } at end of string - could be header name not terminated by colon"
6699     :
6700     US"missing } at end of string";
6701   goto EXPAND_FAILED;
6702   }
6703
6704 /* Expansion succeeded; yield may still be NULL here if nothing was actually
6705 added to the string. If so, set up an empty string. Add a terminating zero. If
6706 left != NULL, return a pointer to the terminator. */
6707
6708 if (yield == NULL) yield = store_get(1);
6709 yield[ptr] = 0;
6710 if (left != NULL) *left = s;
6711
6712 /* Any stacking store that was used above the final string is no longer needed.
6713 In many cases the final string will be the first one that was got and so there
6714 will be optimal store usage. */
6715
6716 if (resetok) store_reset(yield + ptr + 1);
6717 else if (resetok_p) *resetok_p = FALSE;
6718
6719 DEBUG(D_expand)
6720   {
6721   debug_printf("expanding: %.*s\n   result: %s\n", (int)(s - string), string,
6722     yield);
6723   if (skipping) debug_printf("skipping: result is not used\n");
6724   }
6725 return yield;
6726
6727 /* This is the failure exit: easiest to program with a goto. We still need
6728 to update the pointer to the terminator, for cases of nested calls with "fail".
6729 */
6730
6731 EXPAND_FAILED_CURLY:
6732 expand_string_message = malformed_header?
6733   US"missing or misplaced { or } - could be header name not terminated by colon"
6734   :
6735   US"missing or misplaced { or }";
6736
6737 /* At one point, Exim reset the store to yield (if yield was not NULL), but
6738 that is a bad idea, because expand_string_message is in dynamic store. */
6739
6740 EXPAND_FAILED:
6741 if (left != NULL) *left = s;
6742 DEBUG(D_expand)
6743   {
6744   debug_printf("failed to expand: %s\n", string);
6745   debug_printf("   error message: %s\n", expand_string_message);
6746   if (expand_string_forcedfail) debug_printf("failure was forced\n");
6747   }
6748 if (resetok_p) *resetok_p = resetok;
6749 return NULL;
6750 }
6751
6752
6753 /* This is the external function call. Do a quick check for any expansion
6754 metacharacters, and if there are none, just return the input string.
6755
6756 Argument: the string to be expanded
6757 Returns:  the expanded string, or NULL if expansion failed; if failure was
6758           due to a lookup deferring, search_find_defer will be TRUE
6759 */
6760
6761 uschar *
6762 expand_string(uschar *string)
6763 {
6764 search_find_defer = FALSE;
6765 malformed_header = FALSE;
6766 return (Ustrpbrk(string, "$\\") == NULL)? string :
6767   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
6768 }
6769
6770
6771
6772 /*************************************************
6773 *              Expand and copy                   *
6774 *************************************************/
6775
6776 /* Now and again we want to expand a string and be sure that the result is in a
6777 new bit of store. This function does that.
6778
6779 Argument: the string to be expanded
6780 Returns:  the expanded string, always in a new bit of store, or NULL
6781 */
6782
6783 uschar *
6784 expand_string_copy(uschar *string)
6785 {
6786 uschar *yield = expand_string(string);
6787 if (yield == string) yield = string_copy(string);
6788 return yield;
6789 }
6790
6791
6792
6793 /*************************************************
6794 *        Expand and interpret as an integer      *
6795 *************************************************/
6796
6797 /* Expand a string, and convert the result into an integer.
6798
6799 Arguments:
6800   string  the string to be expanded
6801   isplus  TRUE if a non-negative number is expected
6802
6803 Returns:  the integer value, or
6804           -1 for an expansion error               ) in both cases, message in
6805           -2 for an integer interpretation error  ) expand_string_message
6806           expand_string_message is set NULL for an OK integer
6807 */
6808
6809 int_eximarith_t
6810 expand_string_integer(uschar *string, BOOL isplus)
6811 {
6812 int_eximarith_t value;
6813 uschar *s = expand_string(string);
6814 uschar *msg = US"invalid integer \"%s\"";
6815 uschar *endptr;
6816
6817 /* If expansion failed, expand_string_message will be set. */
6818
6819 if (s == NULL) return -1;
6820
6821 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
6822 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
6823 systems, so we set it zero ourselves. */
6824
6825 errno = 0;
6826 expand_string_message = NULL;               /* Indicates no error */
6827
6828 /* Before Exim 4.64, strings consisting entirely of whitespace compared
6829 equal to 0.  Unfortunately, people actually relied upon that, so preserve
6830 the behaviour explicitly.  Stripping leading whitespace is a harmless
6831 noop change since strtol skips it anyway (provided that there is a number
6832 to find at all). */
6833 if (isspace(*s))
6834   {
6835   while (isspace(*s)) ++s;
6836   if (*s == '\0')
6837     {
6838       DEBUG(D_expand)
6839        debug_printf("treating blank string as number 0\n");
6840       return 0;
6841     }
6842   }
6843
6844 value = strtoll(CS s, CSS &endptr, 10);
6845
6846 if (endptr == s)
6847   {
6848   msg = US"integer expected but \"%s\" found";
6849   }
6850 else if (value < 0 && isplus)
6851   {
6852   msg = US"non-negative integer expected but \"%s\" found";
6853   }
6854 else
6855   {
6856   switch (tolower(*endptr))
6857     {
6858     default:
6859       break;
6860     case 'k':
6861       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
6862       else value *= 1024;
6863       endptr++;
6864       break;
6865     case 'm':
6866       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
6867       else value *= 1024*1024;
6868       endptr++;
6869       break;
6870     case 'g':
6871       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
6872       else value *= 1024*1024*1024;
6873       endptr++;
6874       break;
6875     }
6876   if (errno == ERANGE)
6877     msg = US"absolute value of integer \"%s\" is too large (overflow)";
6878   else
6879     {
6880     while (isspace(*endptr)) endptr++;
6881     if (*endptr == 0) return value;
6882     }
6883   }
6884
6885 expand_string_message = string_sprintf(CS msg, s);
6886 return -2;
6887 }
6888
6889
6890 /*************************************************
6891 **************************************************
6892 *             Stand-alone test program           *
6893 **************************************************
6894 *************************************************/
6895
6896 #ifdef STAND_ALONE
6897
6898
6899 BOOL
6900 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
6901 {
6902 int ovector[3*(EXPAND_MAXN+1)];
6903 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
6904   ovector, sizeof(ovector)/sizeof(int));
6905 BOOL yield = n >= 0;
6906 if (n == 0) n = EXPAND_MAXN + 1;
6907 if (yield)
6908   {
6909   int nn;
6910   expand_nmax = (setup < 0)? 0 : setup + 1;
6911   for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2)
6912     {
6913     expand_nstring[expand_nmax] = subject + ovector[nn];
6914     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
6915     }
6916   expand_nmax--;
6917   }
6918 return yield;
6919 }
6920
6921
6922 int main(int argc, uschar **argv)
6923 {
6924 int i;
6925 uschar buffer[1024];
6926
6927 debug_selector = D_v;
6928 debug_file = stderr;
6929 debug_fd = fileno(debug_file);
6930 big_buffer = malloc(big_buffer_size);
6931
6932 for (i = 1; i < argc; i++)
6933   {
6934   if (argv[i][0] == '+')
6935     {
6936     debug_trace_memory = 2;
6937     argv[i]++;
6938     }
6939   if (isdigit(argv[i][0]))
6940     debug_selector = Ustrtol(argv[i], NULL, 0);
6941   else
6942     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
6943         Ustrlen(argv[i]))
6944       {
6945       #ifdef LOOKUP_LDAP
6946       eldap_default_servers = argv[i];
6947       #endif
6948       #ifdef LOOKUP_MYSQL
6949       mysql_servers = argv[i];
6950       #endif
6951       #ifdef LOOKUP_PGSQL
6952       pgsql_servers = argv[i];
6953       #endif
6954       #ifdef EXPERIMENTAL_REDIS
6955       redis_servers = argv[i];
6956       #endif
6957       }
6958   #ifdef EXIM_PERL
6959   else opt_perl_startup = argv[i];
6960   #endif
6961   }
6962
6963 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
6964
6965 expand_nstring[1] = US"string 1....";
6966 expand_nlength[1] = 8;
6967 expand_nmax = 1;
6968
6969 #ifdef EXIM_PERL
6970 if (opt_perl_startup != NULL)
6971   {
6972   uschar *errstr;
6973   printf("Starting Perl interpreter\n");
6974   errstr = init_perl(opt_perl_startup);
6975   if (errstr != NULL)
6976     {
6977     printf("** error in perl_startup code: %s\n", errstr);
6978     return EXIT_FAILURE;
6979     }
6980   }
6981 #endif /* EXIM_PERL */
6982
6983 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
6984   {
6985   void *reset_point = store_get(0);
6986   uschar *yield = expand_string(buffer);
6987   if (yield != NULL)
6988     {
6989     printf("%s\n", yield);
6990     store_reset(reset_point);
6991     }
6992   else
6993     {
6994     if (search_find_defer) printf("search_find deferred\n");
6995     printf("Failed: %s\n", expand_string_message);
6996     if (expand_string_forcedfail) printf("Forced failure\n");
6997     printf("\n");
6998     }
6999   }
7000
7001 search_tidyup();
7002
7003 return 0;
7004 }
7005
7006 #endif
7007
7008 /*
7009  vi: aw ai sw=2
7010 */
7011 /* End of expand.c */