src/src/expand.c

   1 /*************************************************
   2 *     Exim - an Internet mail transport agent    *
   3 *************************************************/
   4
   5 /* Copyright (c) University of Cambridge 1995 - 2014 */
   6 /* See the file NOTICE for conditions of use and distribution. */
   7
   8
   9 /* Functions for handling string expansion. */
  10
  11
  12 #include "exim.h"
  13
  14 /* Recursively called function */
  15
  16 static uschar *expand_string_internal(uschar *, BOOL, uschar **, BOOL, BOOL, BOOL *);
  17 static int_eximarith_t expanded_string_integer(uschar *, BOOL);
  18
  19 #ifdef STAND_ALONE
  20 #ifndef SUPPORT_CRYPTEQ
  21 #define SUPPORT_CRYPTEQ
  22 #endif
  23 #endif
  24
  25 #ifdef LOOKUP_LDAP
  26 #include "lookups/ldap.h"
  27 #endif
  28
  29 #ifdef SUPPORT_CRYPTEQ
  30 #ifdef CRYPT_H
  31 #include <crypt.h>
  32 #endif
  33 #ifndef HAVE_CRYPT16
  34 extern char* crypt16(char*, char*);
  35 #endif
  36 #endif
  37
  38 /* The handling of crypt16() is a mess. I will record below the analysis of the
  39 mess that was sent to me. We decided, however, to make changing this very low
  40 priority, because in practice people are moving away from the crypt()
  41 algorithms nowadays, so it doesn't seem worth it.
  42
  43 <quote>
  44 There is an algorithm named "crypt16" in Ultrix and Tru64.  It crypts
  45 the first 8 characters of the password using a 20-round version of crypt
  46 (standard crypt does 25 rounds).  It then crypts the next 8 characters,
  47 or an empty block if the password is less than 9 characters, using a
  48 20-round version of crypt and the same salt as was used for the first
  49 block.  Charaters after the first 16 are ignored.  It always generates
  50 a 16-byte hash, which is expressed together with the salt as a string
  51 of 24 base 64 digits.  Here are some links to peruse:
  52
  53         http://cvs.pld.org.pl/pam/pamcrypt/crypt16.c?rev=1.2
  54         http://seclists.org/bugtraq/1999/Mar/0076.html
  55
  56 There's a different algorithm named "bigcrypt" in HP-UX, Digital Unix,
  57 and OSF/1.  This is the same as the standard crypt if given a password
  58 of 8 characters or less.  If given more, it first does the same as crypt
  59 using the first 8 characters, then crypts the next 8 (the 9th to 16th)
  60 using as salt the first two base 64 digits from the first hash block.
  61 If the password is more than 16 characters then it crypts the 17th to 24th
  62 characters using as salt the first two base 64 digits from the second hash
  63 block.  And so on: I've seen references to it cutting off the password at
  64 40 characters (5 blocks), 80 (10 blocks), or 128 (16 blocks).  Some links:
  65
  66         http://cvs.pld.org.pl/pam/pamcrypt/bigcrypt.c?rev=1.2
  67         http://seclists.org/bugtraq/1999/Mar/0109.html
  68         http://h30097.www3.hp.com/docs/base_doc/DOCUMENTATION/HTML/AA-Q0R2D-
  69              TET1_html/sec.c222.html#no_id_208
  70
  71 Exim has something it calls "crypt16".  It will either use a native
  72 crypt16 or its own implementation.  A native crypt16 will presumably
  73 be the one that I called "crypt16" above.  The internal "crypt16"
  74 function, however, is a two-block-maximum implementation of what I called
  75 "bigcrypt".  The documentation matches the internal code.
  76
  77 I suspect that whoever did the "crypt16" stuff for Exim didn't realise
  78 that crypt16 and bigcrypt were different things.
  79
  80 Exim uses the LDAP-style scheme identifier "{crypt16}" to refer
  81 to whatever it is using under that name.  This unfortunately sets a
  82 precedent for using "{crypt16}" to identify two incompatible algorithms
  83 whose output can't be distinguished.  With "{crypt16}" thus rendered
  84 ambiguous, I suggest you deprecate it and invent two new identifiers
  85 for the two algorithms.
  86
  87 Both crypt16 and bigcrypt are very poor algorithms, btw.  Hashing parts
  88 of the password separately means they can be cracked separately, so
  89 the double-length hash only doubles the cracking effort instead of
  90 squaring it.  I recommend salted SHA-1 ({SSHA}), or the Blowfish-based
  91 bcrypt ({CRYPT}$2a$).
  92 </quote>
  93 */
  94
  95
  96
  97 #ifndef nelements
  98 # define nelements(arr) (sizeof(arr) / sizeof(*arr))
  99 #endif
 100
 101 /*************************************************
 102 *            Local statics and tables            *
 103 *************************************************/
 104
 105 /* Table of item names, and corresponding switch numbers. The names must be in
 106 alphabetical order. */
 107
 108 static uschar *item_table[] = {
 109   US"acl",
 110   US"certextract",
 111   US"dlfunc",
 112   US"extract",
 113   US"filter",
 114   US"hash",
 115   US"hmac",
 116   US"if",
 117   US"length",
 118   US"listextract",
 119   US"lookup",
 120   US"map",
 121   US"nhash",
 122   US"perl",
 123   US"prvs",
 124   US"prvscheck",
 125   US"readfile",
 126   US"readsocket",
 127   US"reduce",
 128   US"run",
 129   US"sg",
 130   US"substr",
 131   US"tr" };
 132
 133 enum {
 134   EITEM_ACL,
 135   EITEM_CERTEXTRACT,
 136   EITEM_DLFUNC,
 137   EITEM_EXTRACT,
 138   EITEM_FILTER,
 139   EITEM_HASH,
 140   EITEM_HMAC,
 141   EITEM_IF,
 142   EITEM_LENGTH,
 143   EITEM_LISTEXTRACT,
 144   EITEM_LOOKUP,
 145   EITEM_MAP,
 146   EITEM_NHASH,
 147   EITEM_PERL,
 148   EITEM_PRVS,
 149   EITEM_PRVSCHECK,
 150   EITEM_READFILE,
 151   EITEM_READSOCK,
 152   EITEM_REDUCE,
 153   EITEM_RUN,
 154   EITEM_SG,
 155   EITEM_SUBSTR,
 156   EITEM_TR };
 157
 158 /* Tables of operator names, and corresponding switch numbers. The names must be
 159 in alphabetical order. There are two tables, because underscore is used in some
 160 cases to introduce arguments, whereas for other it is part of the name. This is
 161 an historical mis-design. */
 162
 163 static uschar *op_table_underscore[] = {
 164   US"from_utf8",
 165   US"local_part",
 166   US"quote_local_part",
 167   US"reverse_ip",
 168   US"time_eval",
 169   US"time_interval"};
 170
 171 enum {
 172   EOP_FROM_UTF8,
 173   EOP_LOCAL_PART,
 174   EOP_QUOTE_LOCAL_PART,
 175   EOP_REVERSE_IP,
 176   EOP_TIME_EVAL,
 177   EOP_TIME_INTERVAL };
 178
 179 static uschar *op_table_main[] = {
 180   US"address",
 181   US"addresses",
 182   US"base62",
 183   US"base62d",
 184   US"domain",
 185   US"escape",
 186   US"eval",
 187   US"eval10",
 188   US"expand",
 189   US"h",
 190   US"hash",
 191   US"hex2b64",
 192   US"hexquote",
 193   US"l",
 194   US"lc",
 195   US"length",
 196   US"listcount",
 197   US"listnamed",
 198   US"mask",
 199   US"md5",
 200   US"nh",
 201   US"nhash",
 202   US"quote",
 203   US"randint",
 204   US"rfc2047",
 205   US"rfc2047d",
 206   US"rxquote",
 207   US"s",
 208   US"sha1",
 209   US"sha256",
 210   US"stat",
 211   US"str2b64",
 212   US"strlen",
 213   US"substr",
 214   US"uc",
 215   US"utf8clean" };
 216
 217 enum {
 218   EOP_ADDRESS =  sizeof(op_table_underscore)/sizeof(uschar *),
 219   EOP_ADDRESSES,
 220   EOP_BASE62,
 221   EOP_BASE62D,
 222   EOP_DOMAIN,
 223   EOP_ESCAPE,
 224   EOP_EVAL,
 225   EOP_EVAL10,
 226   EOP_EXPAND,
 227   EOP_H,
 228   EOP_HASH,
 229   EOP_HEX2B64,
 230   EOP_HEXQUOTE,
 231   EOP_L,
 232   EOP_LC,
 233   EOP_LENGTH,
 234   EOP_LISTCOUNT,
 235   EOP_LISTNAMED,
 236   EOP_MASK,
 237   EOP_MD5,
 238   EOP_NH,
 239   EOP_NHASH,
 240   EOP_QUOTE,
 241   EOP_RANDINT,
 242   EOP_RFC2047,
 243   EOP_RFC2047D,
 244   EOP_RXQUOTE,
 245   EOP_S,
 246   EOP_SHA1,
 247   EOP_SHA256,
 248   EOP_STAT,
 249   EOP_STR2B64,
 250   EOP_STRLEN,
 251   EOP_SUBSTR,
 252   EOP_UC,
 253   EOP_UTF8CLEAN };
 254
 255
 256 /* Table of condition names, and corresponding switch numbers. The names must
 257 be in alphabetical order. */
 258
 259 static uschar *cond_table[] = {
 260   US"<",
 261   US"<=",
 262   US"=",
 263   US"==",     /* Backward compatibility */
 264   US">",
 265   US">=",
 266   US"acl",
 267   US"and",
 268   US"bool",
 269   US"bool_lax",
 270   US"crypteq",
 271   US"def",
 272   US"eq",
 273   US"eqi",
 274   US"exists",
 275   US"first_delivery",
 276   US"forall",
 277   US"forany",
 278   US"ge",
 279   US"gei",
 280   US"gt",
 281   US"gti",
 282   US"inlist",
 283   US"inlisti",
 284   US"isip",
 285   US"isip4",
 286   US"isip6",
 287   US"ldapauth",
 288   US"le",
 289   US"lei",
 290   US"lt",
 291   US"lti",
 292   US"match",
 293   US"match_address",
 294   US"match_domain",
 295   US"match_ip",
 296   US"match_local_part",
 297   US"or",
 298   US"pam",
 299   US"pwcheck",
 300   US"queue_running",
 301   US"radius",
 302   US"saslauthd"
 303 };
 304
 305 enum {
 306   ECOND_NUM_L,
 307   ECOND_NUM_LE,
 308   ECOND_NUM_E,
 309   ECOND_NUM_EE,
 310   ECOND_NUM_G,
 311   ECOND_NUM_GE,
 312   ECOND_ACL,
 313   ECOND_AND,
 314   ECOND_BOOL,
 315   ECOND_BOOL_LAX,
 316   ECOND_CRYPTEQ,
 317   ECOND_DEF,
 318   ECOND_STR_EQ,
 319   ECOND_STR_EQI,
 320   ECOND_EXISTS,
 321   ECOND_FIRST_DELIVERY,
 322   ECOND_FORALL,
 323   ECOND_FORANY,
 324   ECOND_STR_GE,
 325   ECOND_STR_GEI,
 326   ECOND_STR_GT,
 327   ECOND_STR_GTI,
 328   ECOND_INLIST,
 329   ECOND_INLISTI,
 330   ECOND_ISIP,
 331   ECOND_ISIP4,
 332   ECOND_ISIP6,
 333   ECOND_LDAPAUTH,
 334   ECOND_STR_LE,
 335   ECOND_STR_LEI,
 336   ECOND_STR_LT,
 337   ECOND_STR_LTI,
 338   ECOND_MATCH,
 339   ECOND_MATCH_ADDRESS,
 340   ECOND_MATCH_DOMAIN,
 341   ECOND_MATCH_IP,
 342   ECOND_MATCH_LOCAL_PART,
 343   ECOND_OR,
 344   ECOND_PAM,
 345   ECOND_PWCHECK,
 346   ECOND_QUEUE_RUNNING,
 347   ECOND_RADIUS,
 348   ECOND_SASLAUTHD
 349 };
 350
 351
 352 /* Types of table entry */
 353
 354 enum vtypes {
 355   vtype_int,            /* value is address of int */
 356   vtype_filter_int,     /* ditto, but recognized only when filtering */
 357   vtype_ino,            /* value is address of ino_t (not always an int) */
 358   vtype_uid,            /* value is address of uid_t (not always an int) */
 359   vtype_gid,            /* value is address of gid_t (not always an int) */
 360   vtype_bool,           /* value is address of bool */
 361   vtype_stringptr,      /* value is address of pointer to string */
 362   vtype_msgbody,        /* as stringptr, but read when first required */
 363   vtype_msgbody_end,    /* ditto, the end of the message */
 364   vtype_msgheaders,     /* the message's headers, processed */
 365   vtype_msgheaders_raw, /* the message's headers, unprocessed */
 366   vtype_localpart,      /* extract local part from string */
 367   vtype_domain,         /* extract domain from string */
 368   vtype_string_func,    /* value is string returned by given function */
 369   vtype_todbsdin,       /* value not used; generate BSD inbox tod */
 370   vtype_tode,           /* value not used; generate tod in epoch format */
 371   vtype_todel,          /* value not used; generate tod in epoch/usec format */
 372   vtype_todf,           /* value not used; generate full tod */
 373   vtype_todl,           /* value not used; generate log tod */
 374   vtype_todlf,          /* value not used; generate log file datestamp tod */
 375   vtype_todzone,        /* value not used; generate time zone only */
 376   vtype_todzulu,        /* value not used; generate zulu tod */
 377   vtype_reply,          /* value not used; get reply from headers */
 378   vtype_pid,            /* value not used; result is pid */
 379   vtype_host_lookup,    /* value not used; get host name */
 380   vtype_load_avg,       /* value not used; result is int from os_getloadavg */
 381   vtype_pspace,         /* partition space; value is T/F for spool/log */
 382   vtype_pinodes,        /* partition inodes; value is T/F for spool/log */
 383   vtype_cert            /* SSL certificate */
 384   #ifndef DISABLE_DKIM
 385   ,vtype_dkim           /* Lookup of value in DKIM signature */
 386   #endif
 387 };
 388
 389 /* Type for main variable table */
 390
 391 typedef struct {
 392   const char *name;
 393   enum vtypes type;
 394   void       *value;
 395 } var_entry;
 396
 397 /* Type for entries pointing to address/length pairs. Not currently
 398 in use. */
 399
 400 typedef struct {
 401   uschar **address;
 402   int  *length;
 403 } alblock;
 404
 405 static uschar * fn_recipients(void);
 406
 407 /* This table must be kept in alphabetical order. */
 408
 409 static var_entry var_table[] = {
 410   /* WARNING: Do not invent variables whose names start acl_c or acl_m because
 411      they will be confused with user-creatable ACL variables. */
 412   { "acl_arg1",            vtype_stringptr,   &acl_arg[0] },
 413   { "acl_arg2",            vtype_stringptr,   &acl_arg[1] },
 414   { "acl_arg3",            vtype_stringptr,   &acl_arg[2] },
 415   { "acl_arg4",            vtype_stringptr,   &acl_arg[3] },
 416   { "acl_arg5",            vtype_stringptr,   &acl_arg[4] },
 417   { "acl_arg6",            vtype_stringptr,   &acl_arg[5] },
 418   { "acl_arg7",            vtype_stringptr,   &acl_arg[6] },
 419   { "acl_arg8",            vtype_stringptr,   &acl_arg[7] },
 420   { "acl_arg9",            vtype_stringptr,   &acl_arg[8] },
 421   { "acl_narg",            vtype_int,         &acl_narg },
 422   { "acl_verify_message",  vtype_stringptr,   &acl_verify_message },
 423   { "address_data",        vtype_stringptr,   &deliver_address_data },
 424   { "address_file",        vtype_stringptr,   &address_file },
 425   { "address_pipe",        vtype_stringptr,   &address_pipe },
 426   { "authenticated_fail_id",vtype_stringptr,  &authenticated_fail_id },
 427   { "authenticated_id",    vtype_stringptr,   &authenticated_id },
 428   { "authenticated_sender",vtype_stringptr,   &authenticated_sender },
 429   { "authentication_failed",vtype_int,        &authentication_failed },
 430 #ifdef WITH_CONTENT_SCAN
 431   { "av_failed",           vtype_int,         &av_failed },
 432 #endif
 433 #ifdef EXPERIMENTAL_BRIGHTMAIL
 434   { "bmi_alt_location",    vtype_stringptr,   &bmi_alt_location },
 435   { "bmi_base64_tracker_verdict", vtype_stringptr, &bmi_base64_tracker_verdict },
 436   { "bmi_base64_verdict",  vtype_stringptr,   &bmi_base64_verdict },
 437   { "bmi_deliver",         vtype_int,         &bmi_deliver },
 438 #endif
 439   { "body_linecount",      vtype_int,         &body_linecount },
 440   { "body_zerocount",      vtype_int,         &body_zerocount },
 441   { "bounce_recipient",    vtype_stringptr,   &bounce_recipient },
 442   { "bounce_return_size_limit", vtype_int,    &bounce_return_size_limit },
 443   { "caller_gid",          vtype_gid,         &real_gid },
 444   { "caller_uid",          vtype_uid,         &real_uid },
 445   { "compile_date",        vtype_stringptr,   &version_date },
 446   { "compile_number",      vtype_stringptr,   &version_cnumber },
 447   { "csa_status",          vtype_stringptr,   &csa_status },
 448 #ifdef EXPERIMENTAL_DCC
 449   { "dcc_header",          vtype_stringptr,   &dcc_header },
 450   { "dcc_result",          vtype_stringptr,   &dcc_result },
 451 #endif
 452 #ifdef WITH_OLD_DEMIME
 453   { "demime_errorlevel",   vtype_int,         &demime_errorlevel },
 454   { "demime_reason",       vtype_stringptr,   &demime_reason },
 455 #endif
 456 #ifndef DISABLE_DKIM
 457   { "dkim_algo",           vtype_dkim,        (void *)DKIM_ALGO },
 458   { "dkim_bodylength",     vtype_dkim,        (void *)DKIM_BODYLENGTH },
 459   { "dkim_canon_body",     vtype_dkim,        (void *)DKIM_CANON_BODY },
 460   { "dkim_canon_headers",  vtype_dkim,        (void *)DKIM_CANON_HEADERS },
 461   { "dkim_copiedheaders",  vtype_dkim,        (void *)DKIM_COPIEDHEADERS },
 462   { "dkim_created",        vtype_dkim,        (void *)DKIM_CREATED },
 463   { "dkim_cur_signer",     vtype_stringptr,   &dkim_cur_signer },
 464   { "dkim_domain",         vtype_stringptr,   &dkim_signing_domain },
 465   { "dkim_expires",        vtype_dkim,        (void *)DKIM_EXPIRES },
 466   { "dkim_headernames",    vtype_dkim,        (void *)DKIM_HEADERNAMES },
 467   { "dkim_identity",       vtype_dkim,        (void *)DKIM_IDENTITY },
 468   { "dkim_key_granularity",vtype_dkim,        (void *)DKIM_KEY_GRANULARITY },
 469   { "dkim_key_nosubdomains",vtype_dkim,       (void *)DKIM_NOSUBDOMAINS },
 470   { "dkim_key_notes",      vtype_dkim,        (void *)DKIM_KEY_NOTES },
 471   { "dkim_key_srvtype",    vtype_dkim,        (void *)DKIM_KEY_SRVTYPE },
 472   { "dkim_key_testing",    vtype_dkim,        (void *)DKIM_KEY_TESTING },
 473   { "dkim_selector",       vtype_stringptr,   &dkim_signing_selector },
 474   { "dkim_signers",        vtype_stringptr,   &dkim_signers },
 475   { "dkim_verify_reason",  vtype_dkim,        (void *)DKIM_VERIFY_REASON },
 476   { "dkim_verify_status",  vtype_dkim,        (void *)DKIM_VERIFY_STATUS},
 477 #endif
 478 #ifdef EXPERIMENTAL_DMARC
 479   { "dmarc_ar_header",     vtype_stringptr,   &dmarc_ar_header },
 480   { "dmarc_domain_policy", vtype_stringptr,   &dmarc_domain_policy },
 481   { "dmarc_status",        vtype_stringptr,   &dmarc_status },
 482   { "dmarc_status_text",   vtype_stringptr,   &dmarc_status_text },
 483   { "dmarc_used_domain",   vtype_stringptr,   &dmarc_used_domain },
 484 #endif
 485   { "dnslist_domain",      vtype_stringptr,   &dnslist_domain },
 486   { "dnslist_matched",     vtype_stringptr,   &dnslist_matched },
 487   { "dnslist_text",        vtype_stringptr,   &dnslist_text },
 488   { "dnslist_value",       vtype_stringptr,   &dnslist_value },
 489   { "domain",              vtype_stringptr,   &deliver_domain },
 490   { "domain_data",         vtype_stringptr,   &deliver_domain_data },
 491   { "exim_gid",            vtype_gid,         &exim_gid },
 492   { "exim_path",           vtype_stringptr,   &exim_path },
 493   { "exim_uid",            vtype_uid,         &exim_uid },
 494 #ifdef WITH_OLD_DEMIME
 495   { "found_extension",     vtype_stringptr,   &found_extension },
 496 #endif
 497   { "headers_added",       vtype_string_func, &fn_hdrs_added },
 498   { "home",                vtype_stringptr,   &deliver_home },
 499   { "host",                vtype_stringptr,   &deliver_host },
 500   { "host_address",        vtype_stringptr,   &deliver_host_address },
 501   { "host_data",           vtype_stringptr,   &host_data },
 502   { "host_lookup_deferred",vtype_int,         &host_lookup_deferred },
 503   { "host_lookup_failed",  vtype_int,         &host_lookup_failed },
 504   { "inode",               vtype_ino,         &deliver_inode },
 505   { "interface_address",   vtype_stringptr,   &interface_address },
 506   { "interface_port",      vtype_int,         &interface_port },
 507   { "item",                vtype_stringptr,   &iterate_item },
 508   #ifdef LOOKUP_LDAP
 509   { "ldap_dn",             vtype_stringptr,   &eldap_dn },
 510   #endif
 511   { "load_average",        vtype_load_avg,    NULL },
 512   { "local_part",          vtype_stringptr,   &deliver_localpart },
 513   { "local_part_data",     vtype_stringptr,   &deliver_localpart_data },
 514   { "local_part_prefix",   vtype_stringptr,   &deliver_localpart_prefix },
 515   { "local_part_suffix",   vtype_stringptr,   &deliver_localpart_suffix },
 516   { "local_scan_data",     vtype_stringptr,   &local_scan_data },
 517   { "local_user_gid",      vtype_gid,         &local_user_gid },
 518   { "local_user_uid",      vtype_uid,         &local_user_uid },
 519   { "localhost_number",    vtype_int,         &host_number },
 520   { "log_inodes",          vtype_pinodes,     (void *)FALSE },
 521   { "log_space",           vtype_pspace,      (void *)FALSE },
 522   { "lookup_dnssec_authenticated",vtype_stringptr,&lookup_dnssec_authenticated},
 523   { "mailstore_basename",  vtype_stringptr,   &mailstore_basename },
 524 #ifdef WITH_CONTENT_SCAN
 525   { "malware_name",        vtype_stringptr,   &malware_name },
 526 #endif
 527   { "max_received_linelength", vtype_int,     &max_received_linelength },
 528   { "message_age",         vtype_int,         &message_age },
 529   { "message_body",        vtype_msgbody,     &message_body },
 530   { "message_body_end",    vtype_msgbody_end, &message_body_end },
 531   { "message_body_size",   vtype_int,         &message_body_size },
 532   { "message_exim_id",     vtype_stringptr,   &message_id },
 533   { "message_headers",     vtype_msgheaders,  NULL },
 534   { "message_headers_raw", vtype_msgheaders_raw, NULL },
 535   { "message_id",          vtype_stringptr,   &message_id },
 536   { "message_linecount",   vtype_int,         &message_linecount },
 537   { "message_size",        vtype_int,         &message_size },
 538 #ifdef WITH_CONTENT_SCAN
 539   { "mime_anomaly_level",  vtype_int,         &mime_anomaly_level },
 540   { "mime_anomaly_text",   vtype_stringptr,   &mime_anomaly_text },
 541   { "mime_boundary",       vtype_stringptr,   &mime_boundary },
 542   { "mime_charset",        vtype_stringptr,   &mime_charset },
 543   { "mime_content_description", vtype_stringptr, &mime_content_description },
 544   { "mime_content_disposition", vtype_stringptr, &mime_content_disposition },
 545   { "mime_content_id",     vtype_stringptr,   &mime_content_id },
 546   { "mime_content_size",   vtype_int,         &mime_content_size },
 547   { "mime_content_transfer_encoding",vtype_stringptr, &mime_content_transfer_encoding },
 548   { "mime_content_type",   vtype_stringptr,   &mime_content_type },
 549   { "mime_decoded_filename", vtype_stringptr, &mime_decoded_filename },
 550   { "mime_filename",       vtype_stringptr,   &mime_filename },
 551   { "mime_is_coverletter", vtype_int,         &mime_is_coverletter },
 552   { "mime_is_multipart",   vtype_int,         &mime_is_multipart },
 553   { "mime_is_rfc822",      vtype_int,         &mime_is_rfc822 },
 554   { "mime_part_count",     vtype_int,         &mime_part_count },
 555 #endif
 556   { "n0",                  vtype_filter_int,  &filter_n[0] },
 557   { "n1",                  vtype_filter_int,  &filter_n[1] },
 558   { "n2",                  vtype_filter_int,  &filter_n[2] },
 559   { "n3",                  vtype_filter_int,  &filter_n[3] },
 560   { "n4",                  vtype_filter_int,  &filter_n[4] },
 561   { "n5",                  vtype_filter_int,  &filter_n[5] },
 562   { "n6",                  vtype_filter_int,  &filter_n[6] },
 563   { "n7",                  vtype_filter_int,  &filter_n[7] },
 564   { "n8",                  vtype_filter_int,  &filter_n[8] },
 565   { "n9",                  vtype_filter_int,  &filter_n[9] },
 566   { "original_domain",     vtype_stringptr,   &deliver_domain_orig },
 567   { "original_local_part", vtype_stringptr,   &deliver_localpart_orig },
 568   { "originator_gid",      vtype_gid,         &originator_gid },
 569   { "originator_uid",      vtype_uid,         &originator_uid },
 570   { "parent_domain",       vtype_stringptr,   &deliver_domain_parent },
 571   { "parent_local_part",   vtype_stringptr,   &deliver_localpart_parent },
 572   { "pid",                 vtype_pid,         NULL },
 573   { "primary_hostname",    vtype_stringptr,   &primary_hostname },
 574 #ifdef EXPERIMENTAL_PROXY
 575   { "proxy_host_address",  vtype_stringptr,   &proxy_host_address },
 576   { "proxy_host_port",     vtype_int,         &proxy_host_port },
 577   { "proxy_session",       vtype_bool,        &proxy_session },
 578   { "proxy_target_address",vtype_stringptr,   &proxy_target_address },
 579   { "proxy_target_port",   vtype_int,         &proxy_target_port },
 580 #endif
 581   { "prvscheck_address",   vtype_stringptr,   &prvscheck_address },
 582   { "prvscheck_keynum",    vtype_stringptr,   &prvscheck_keynum },
 583   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
 584   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
 585   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
 586   { "rcpt_count",          vtype_int,         &rcpt_count },
 587   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
 588   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
 589   { "received_count",      vtype_int,         &received_count },
 590   { "received_for",        vtype_stringptr,   &received_for },
 591   { "received_ip_address", vtype_stringptr,   &interface_address },
 592   { "received_port",       vtype_int,         &interface_port },
 593   { "received_protocol",   vtype_stringptr,   &received_protocol },
 594   { "received_time",       vtype_int,         &received_time },
 595   { "recipient_data",      vtype_stringptr,   &recipient_data },
 596   { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure },
 597   { "recipients",          vtype_string_func, &fn_recipients },
 598   { "recipients_count",    vtype_int,         &recipients_count },
 599 #ifdef WITH_CONTENT_SCAN
 600   { "regex_match_string",  vtype_stringptr,   &regex_match_string },
 601 #endif
 602   { "reply_address",       vtype_reply,       NULL },
 603   { "return_path",         vtype_stringptr,   &return_path },
 604   { "return_size_limit",   vtype_int,         &bounce_return_size_limit },
 605   { "router_name",         vtype_stringptr,   &router_name },
 606   { "runrc",               vtype_int,         &runrc },
 607   { "self_hostname",       vtype_stringptr,   &self_hostname },
 608   { "sender_address",      vtype_stringptr,   &sender_address },
 609   { "sender_address_data", vtype_stringptr,   &sender_address_data },
 610   { "sender_address_domain", vtype_domain,    &sender_address },
 611   { "sender_address_local_part", vtype_localpart, &sender_address },
 612   { "sender_data",         vtype_stringptr,   &sender_data },
 613   { "sender_fullhost",     vtype_stringptr,   &sender_fullhost },
 614   { "sender_helo_name",    vtype_stringptr,   &sender_helo_name },
 615   { "sender_host_address", vtype_stringptr,   &sender_host_address },
 616   { "sender_host_authenticated",vtype_stringptr, &sender_host_authenticated },
 617   { "sender_host_dnssec",  vtype_bool,        &sender_host_dnssec },
 618   { "sender_host_name",    vtype_host_lookup, NULL },
 619   { "sender_host_port",    vtype_int,         &sender_host_port },
 620   { "sender_ident",        vtype_stringptr,   &sender_ident },
 621   { "sender_rate",         vtype_stringptr,   &sender_rate },
 622   { "sender_rate_limit",   vtype_stringptr,   &sender_rate_limit },
 623   { "sender_rate_period",  vtype_stringptr,   &sender_rate_period },
 624   { "sender_rcvhost",      vtype_stringptr,   &sender_rcvhost },
 625   { "sender_verify_failure",vtype_stringptr,  &sender_verify_failure },
 626   { "sending_ip_address",  vtype_stringptr,   &sending_ip_address },
 627   { "sending_port",        vtype_int,         &sending_port },
 628   { "smtp_active_hostname", vtype_stringptr,  &smtp_active_hostname },
 629   { "smtp_command",        vtype_stringptr,   &smtp_cmd_buffer },
 630   { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument },
 631   { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count },
 632   { "smtp_notquit_reason", vtype_stringptr,   &smtp_notquit_reason },
 633   { "sn0",                 vtype_filter_int,  &filter_sn[0] },
 634   { "sn1",                 vtype_filter_int,  &filter_sn[1] },
 635   { "sn2",                 vtype_filter_int,  &filter_sn[2] },
 636   { "sn3",                 vtype_filter_int,  &filter_sn[3] },
 637   { "sn4",                 vtype_filter_int,  &filter_sn[4] },
 638   { "sn5",                 vtype_filter_int,  &filter_sn[5] },
 639   { "sn6",                 vtype_filter_int,  &filter_sn[6] },
 640   { "sn7",                 vtype_filter_int,  &filter_sn[7] },
 641   { "sn8",                 vtype_filter_int,  &filter_sn[8] },
 642   { "sn9",                 vtype_filter_int,  &filter_sn[9] },
 643 #ifdef WITH_CONTENT_SCAN
 644   { "spam_bar",            vtype_stringptr,   &spam_bar },
 645   { "spam_report",         vtype_stringptr,   &spam_report },
 646   { "spam_score",          vtype_stringptr,   &spam_score },
 647   { "spam_score_int",      vtype_stringptr,   &spam_score_int },
 648 #endif
 649 #ifdef EXPERIMENTAL_SPF
 650   { "spf_guess",           vtype_stringptr,   &spf_guess },
 651   { "spf_header_comment",  vtype_stringptr,   &spf_header_comment },
 652   { "spf_received",        vtype_stringptr,   &spf_received },
 653   { "spf_result",          vtype_stringptr,   &spf_result },
 654   { "spf_smtp_comment",    vtype_stringptr,   &spf_smtp_comment },
 655 #endif
 656   { "spool_directory",     vtype_stringptr,   &spool_directory },
 657   { "spool_inodes",        vtype_pinodes,     (void *)TRUE },
 658   { "spool_space",         vtype_pspace,      (void *)TRUE },
 659 #ifdef EXPERIMENTAL_SRS
 660   { "srs_db_address",      vtype_stringptr,   &srs_db_address },
 661   { "srs_db_key",          vtype_stringptr,   &srs_db_key },
 662   { "srs_orig_recipient",  vtype_stringptr,   &srs_orig_recipient },
 663   { "srs_orig_sender",     vtype_stringptr,   &srs_orig_sender },
 664   { "srs_recipient",       vtype_stringptr,   &srs_recipient },
 665   { "srs_status",          vtype_stringptr,   &srs_status },
 666 #endif
 667   { "thisaddress",         vtype_stringptr,   &filter_thisaddress },
 668
 669   /* The non-(in,out) variables are now deprecated */
 670   { "tls_bits",            vtype_int,         &tls_in.bits },
 671   { "tls_certificate_verified", vtype_int,    &tls_in.certificate_verified },
 672   { "tls_cipher",          vtype_stringptr,   &tls_in.cipher },
 673
 674   { "tls_in_bits",         vtype_int,         &tls_in.bits },
 675   { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified },
 676   { "tls_in_cipher",       vtype_stringptr,   &tls_in.cipher },
 677   { "tls_in_ocsp",         vtype_int,         &tls_in.ocsp },
 678   { "tls_in_ourcert",      vtype_cert,        &tls_in.ourcert },
 679   { "tls_in_peercert",     vtype_cert,        &tls_in.peercert },
 680   { "tls_in_peerdn",       vtype_stringptr,   &tls_in.peerdn },
 681 #if defined(SUPPORT_TLS)
 682   { "tls_in_sni",          vtype_stringptr,   &tls_in.sni },
 683 #endif
 684   { "tls_out_bits",        vtype_int,         &tls_out.bits },
 685   { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified },
 686   { "tls_out_cipher",      vtype_stringptr,   &tls_out.cipher },
 687 #ifdef EXPERIMENTAL_DANE
 688   { "tls_out_dane",        vtype_bool,        &tls_out.dane_verified },
 689 #endif
 690   { "tls_out_ocsp",        vtype_int,         &tls_out.ocsp },
 691   { "tls_out_ourcert",     vtype_cert,        &tls_out.ourcert },
 692   { "tls_out_peercert",    vtype_cert,        &tls_out.peercert },
 693   { "tls_out_peerdn",      vtype_stringptr,   &tls_out.peerdn },
 694 #if defined(SUPPORT_TLS)
 695   { "tls_out_sni",         vtype_stringptr,   &tls_out.sni },
 696 #endif
 697 #ifdef EXPERIMENTAL_DANE
 698   { "tls_out_tlsa_usage",  vtype_int,         &tls_out.tlsa_usage },
 699 #endif
 700
 701   { "tls_peerdn",          vtype_stringptr,   &tls_in.peerdn }, /* mind the alphabetical order! */
 702 #if defined(SUPPORT_TLS)
 703   { "tls_sni",             vtype_stringptr,   &tls_in.sni },    /* mind the alphabetical order! */
 704 #endif
 705
 706   { "tod_bsdinbox",        vtype_todbsdin,    NULL },
 707   { "tod_epoch",           vtype_tode,        NULL },
 708   { "tod_epoch_l",         vtype_todel,       NULL },
 709   { "tod_full",            vtype_todf,        NULL },
 710   { "tod_log",             vtype_todl,        NULL },
 711   { "tod_logfile",         vtype_todlf,       NULL },
 712   { "tod_zone",            vtype_todzone,     NULL },
 713   { "tod_zulu",            vtype_todzulu,     NULL },
 714 #ifdef EXPERIMENTAL_TPDA
 715   { "tpda_defer_errno",     vtype_int,         &tpda_defer_errno },
 716   { "tpda_defer_errstr",    vtype_stringptr,   &tpda_defer_errstr },
 717   { "tpda_delivery_confirmation", vtype_stringptr,   &tpda_delivery_confirmation },
 718   { "tpda_delivery_domain", vtype_stringptr,   &tpda_delivery_domain },
 719   { "tpda_delivery_fqdn",   vtype_stringptr,   &tpda_delivery_fqdn },
 720   { "tpda_delivery_ip",     vtype_stringptr,   &tpda_delivery_ip },
 721   { "tpda_delivery_local_part",vtype_stringptr,&tpda_delivery_local_part },
 722   { "tpda_delivery_port",   vtype_int,         &tpda_delivery_port },
 723 #endif
 724   { "transport_name",      vtype_stringptr,   &transport_name },
 725   { "value",               vtype_stringptr,   &lookup_value },
 726   { "version_number",      vtype_stringptr,   &version_string },
 727   { "warn_message_delay",  vtype_stringptr,   &warnmsg_delay },
 728   { "warn_message_recipient",vtype_stringptr, &warnmsg_recipients },
 729   { "warn_message_recipients",vtype_stringptr,&warnmsg_recipients },
 730   { "warnmsg_delay",       vtype_stringptr,   &warnmsg_delay },
 731   { "warnmsg_recipient",   vtype_stringptr,   &warnmsg_recipients },
 732   { "warnmsg_recipients",  vtype_stringptr,   &warnmsg_recipients }
 733 };
 734
 735 static int var_table_size = sizeof(var_table)/sizeof(var_entry);
 736 static uschar var_buffer[256];
 737 static BOOL malformed_header;
 738
 739 /* For textual hashes */
 740
 741 static const char *hashcodes = "abcdefghijklmnopqrtsuvwxyz"
 742                                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 743                                "0123456789";
 744
 745 enum { HMAC_MD5, HMAC_SHA1 };
 746
 747 /* For numeric hashes */
 748
 749 static unsigned int prime[] = {
 750   2,   3,   5,   7,  11,  13,  17,  19,  23,  29,
 751  31,  37,  41,  43,  47,  53,  59,  61,  67,  71,
 752  73,  79,  83,  89,  97, 101, 103, 107, 109, 113};
 753
 754 /* For printing modes in symbolic form */
 755
 756 static uschar *mtable_normal[] =
 757   { US"---", US"--x", US"-w-", US"-wx", US"r--", US"r-x", US"rw-", US"rwx" };
 758
 759 static uschar *mtable_setid[] =
 760   { US"--S", US"--s", US"-wS", US"-ws", US"r-S", US"r-s", US"rwS", US"rws" };
 761
 762 static uschar *mtable_sticky[] =
 763   { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" };
 764
 765
 766
 767 /*************************************************
 768 *           Tables for UTF-8 support             *
 769 *************************************************/
 770
 771 /* Table of the number of extra characters, indexed by the first character
 772 masked with 0x3f. The highest number for a valid UTF-8 character is in fact
 773 0x3d. */
 774
 775 static uschar utf8_table1[] = {
 776   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 777   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
 778   2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
 779   3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
 780
 781 /* These are the masks for the data bits in the first byte of a character,
 782 indexed by the number of additional bytes. */
 783
 784 static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
 785
 786 /* Get the next UTF-8 character, advancing the pointer. */
 787
 788 #define GETUTF8INC(c, ptr) \
 789   c = *ptr++; \
 790   if ((c & 0xc0) == 0xc0) \
 791     { \
 792     int a = utf8_table1[c & 0x3f];  /* Number of additional bytes */ \
 793     int s = 6*a; \
 794     c = (c & utf8_table2[a]) << s; \
 795     while (a-- > 0) \
 796       { \
 797       s -= 6; \
 798       c |= (*ptr++ & 0x3f) << s; \
 799       } \
 800     }
 801
 802
 803 /*************************************************
 804 *           Binary chop search on a table        *
 805 *************************************************/
 806
 807 /* This is used for matching expansion items and operators.
 808
 809 Arguments:
 810   name        the name that is being sought
 811   table       the table to search
 812   table_size  the number of items in the table
 813
 814 Returns:      the offset in the table, or -1
 815 */
 816
 817 static int
 818 chop_match(uschar *name, uschar **table, int table_size)
 819 {
 820 uschar **bot = table;
 821 uschar **top = table + table_size;
 822
 823 while (top > bot)
 824   {
 825   uschar **mid = bot + (top - bot)/2;
 826   int c = Ustrcmp(name, *mid);
 827   if (c == 0) return mid - table;
 828   if (c > 0) bot = mid + 1; else top = mid;
 829   }
 830
 831 return -1;
 832 }
 833
 834
 835
 836 /*************************************************
 837 *          Check a condition string              *
 838 *************************************************/
 839
 840 /* This function is called to expand a string, and test the result for a "true"
 841 or "false" value. Failure of the expansion yields FALSE; logged unless it was a
 842 forced fail or lookup defer.
 843
 844 We used to release all store used, but this is not not safe due
 845 to ${dlfunc } and ${acl }.  In any case expand_string_internal()
 846 is reasonably careful to release what it can.
 847
 848 The actual false-value tests should be replicated for ECOND_BOOL_LAX.
 849
 850 Arguments:
 851   condition     the condition string
 852   m1            text to be incorporated in panic error
 853   m2            ditto
 854
 855 Returns:        TRUE if condition is met, FALSE if not
 856 */
 857
 858 BOOL
 859 expand_check_condition(uschar *condition, uschar *m1, uschar *m2)
 860 {
 861 int rc;
 862 uschar *ss = expand_string(condition);
 863 if (ss == NULL)
 864   {
 865   if (!expand_string_forcedfail && !search_find_defer)
 866     log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" "
 867       "for %s %s: %s", condition, m1, m2, expand_string_message);
 868   return FALSE;
 869   }
 870 rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 &&
 871   strcmpic(ss, US"false") != 0;
 872 return rc;
 873 }
 874
 875
 876
 877
 878 /*************************************************
 879 *        Pseudo-random number generation         *
 880 *************************************************/
 881
 882 /* Pseudo-random number generation.  The result is not "expected" to be
 883 cryptographically strong but not so weak that someone will shoot themselves
 884 in the foot using it as a nonce in some email header scheme or whatever
 885 weirdness they'll twist this into.  The result should ideally handle fork().
 886
 887 However, if we're stuck unable to provide this, then we'll fall back to
 888 appallingly bad randomness.
 889
 890 If SUPPORT_TLS is defined then this will not be used except as an emergency
 891 fallback.
 892
 893 Arguments:
 894   max       range maximum
 895 Returns     a random number in range [0, max-1]
 896 */
 897
 898 #ifdef SUPPORT_TLS
 899 # define vaguely_random_number vaguely_random_number_fallback
 900 #endif
 901 int
 902 vaguely_random_number(int max)
 903 {
 904 #ifdef SUPPORT_TLS
 905 # undef vaguely_random_number
 906 #endif
 907   static pid_t pid = 0;
 908   pid_t p2;
 909 #if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV)
 910   struct timeval tv;
 911 #endif
 912
 913   p2 = getpid();
 914   if (p2 != pid)
 915     {
 916     if (pid != 0)
 917       {
 918
 919 #ifdef HAVE_ARC4RANDOM
 920       /* cryptographically strong randomness, common on *BSD platforms, not
 921       so much elsewhere.  Alas. */
 922       arc4random_stir();
 923 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 924 #ifdef HAVE_SRANDOMDEV
 925       /* uses random(4) for seeding */
 926       srandomdev();
 927 #else
 928       gettimeofday(&tv, NULL);
 929       srandom(tv.tv_sec | tv.tv_usec | getpid());
 930 #endif
 931 #else
 932       /* Poor randomness and no seeding here */
 933 #endif
 934
 935       }
 936     pid = p2;
 937     }
 938
 939 #ifdef HAVE_ARC4RANDOM
 940   return arc4random() % max;
 941 #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV)
 942   return random() % max;
 943 #else
 944   /* This one returns a 16-bit number, definitely not crypto-strong */
 945   return random_number(max);
 946 #endif
 947 }
 948
 949
 950
 951
 952 /*************************************************
 953 *             Pick out a name from a string      *
 954 *************************************************/
 955
 956 /* If the name is too long, it is silently truncated.
 957
 958 Arguments:
 959   name      points to a buffer into which to put the name
 960   max       is the length of the buffer
 961   s         points to the first alphabetic character of the name
 962   extras    chars other than alphanumerics to permit
 963
 964 Returns:    pointer to the first character after the name
 965
 966 Note: The test for *s != 0 in the while loop is necessary because
 967 Ustrchr() yields non-NULL if the character is zero (which is not something
 968 I expected). */
 969
 970 static uschar *
 971 read_name(uschar *name, int max, uschar *s, uschar *extras)
 972 {
 973 int ptr = 0;
 974 while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL))
 975   {
 976   if (ptr < max-1) name[ptr++] = *s;
 977   s++;
 978   }
 979 name[ptr] = 0;
 980 return s;
 981 }
 982
 983
 984
 985 /*************************************************
 986 *     Pick out the rest of a header name         *
 987 *************************************************/
 988
 989 /* A variable name starting $header_ (or just $h_ for those who like
 990 abbreviations) might not be the complete header name because headers can
 991 contain any printing characters in their names, except ':'. This function is
 992 called to read the rest of the name, chop h[eader]_ off the front, and put ':'
 993 on the end, if the name was terminated by white space.
 994
 995 Arguments:
 996   name      points to a buffer in which the name read so far exists
 997   max       is the length of the buffer
 998   s         points to the first character after the name so far, i.e. the
 999             first non-alphameric character after $header_xxxxx
1000
1001 Returns:    a pointer to the first character after the header name
1002 */
1003
1004 static uschar *
1005 read_header_name(uschar *name, int max, uschar *s)
1006 {
1007 int prelen = Ustrchr(name, '_') - name + 1;
1008 int ptr = Ustrlen(name) - prelen;
1009 if (ptr > 0) memmove(name, name+prelen, ptr);
1010 while (mac_isgraph(*s) && *s != ':')
1011   {
1012   if (ptr < max-1) name[ptr++] = *s;
1013   s++;
1014   }
1015 if (*s == ':') s++;
1016 name[ptr++] = ':';
1017 name[ptr] = 0;
1018 return s;
1019 }
1020
1021
1022
1023 /*************************************************
1024 *           Pick out a number from a string      *
1025 *************************************************/
1026
1027 /* Arguments:
1028   n     points to an integer into which to put the number
1029   s     points to the first digit of the number
1030
1031 Returns:  a pointer to the character after the last digit
1032 */
1033
1034 static uschar *
1035 read_number(int *n, uschar *s)
1036 {
1037 *n = 0;
1038 while (isdigit(*s)) *n = *n * 10 + (*s++ - '0');
1039 return s;
1040 }
1041
1042
1043
1044 /*************************************************
1045 *        Extract keyed subfield from a string    *
1046 *************************************************/
1047
1048 /* The yield is in dynamic store; NULL means that the key was not found.
1049
1050 Arguments:
1051   key       points to the name of the key
1052   s         points to the string from which to extract the subfield
1053
1054 Returns:    NULL if the subfield was not found, or
1055             a pointer to the subfield's data
1056 */
1057
1058 static uschar *
1059 expand_getkeyed(uschar *key, uschar *s)
1060 {
1061 int length = Ustrlen(key);
1062 while (isspace(*s)) s++;
1063
1064 /* Loop to search for the key */
1065
1066 while (*s != 0)
1067   {
1068   int dkeylength;
1069   uschar *data;
1070   uschar *dkey = s;
1071
1072   while (*s != 0 && *s != '=' && !isspace(*s)) s++;
1073   dkeylength = s - dkey;
1074   while (isspace(*s)) s++;
1075   if (*s == '=') while (isspace((*(++s))));
1076
1077   data = string_dequote(&s);
1078   if (length == dkeylength && strncmpic(key, dkey, length) == 0)
1079     return data;
1080
1081   while (isspace(*s)) s++;
1082   }
1083
1084 return NULL;
1085 }
1086
1087
1088
1089 static var_entry *
1090 find_var_ent(uschar * name)
1091 {
1092 int first = 0;
1093 int last = var_table_size;
1094
1095 while (last > first)
1096   {
1097   int middle = (first + last)/2;
1098   int c = Ustrcmp(name, var_table[middle].name);
1099
1100   if (c > 0) { first = middle + 1; continue; }
1101   if (c < 0) { last = middle; continue; }
1102   return &var_table[middle];
1103   }
1104 return NULL;
1105 }
1106
1107 /*************************************************
1108 *   Extract numbered subfield from string        *
1109 *************************************************/
1110
1111 /* Extracts a numbered field from a string that is divided by tokens - for
1112 example a line from /etc/passwd is divided by colon characters.  First field is
1113 numbered one.  Negative arguments count from the right. Zero returns the whole
1114 string. Returns NULL if there are insufficient tokens in the string
1115
1116 ***WARNING***
1117 Modifies final argument - this is a dynamically generated string, so that's OK.
1118
1119 Arguments:
1120   field       number of field to be extracted,
1121                 first field = 1, whole string = 0, last field = -1
1122   separators  characters that are used to break string into tokens
1123   s           points to the string from which to extract the subfield
1124
1125 Returns:      NULL if the field was not found,
1126               a pointer to the field's data inside s (modified to add 0)
1127 */
1128
1129 static uschar *
1130 expand_gettokened (int field, uschar *separators, uschar *s)
1131 {
1132 int sep = 1;
1133 int count;
1134 uschar *ss = s;
1135 uschar *fieldtext = NULL;
1136
1137 if (field == 0) return s;
1138
1139 /* Break the line up into fields in place; for field > 0 we stop when we have
1140 done the number of fields we want. For field < 0 we continue till the end of
1141 the string, counting the number of fields. */
1142
1143 count = (field > 0)? field : INT_MAX;
1144
1145 while (count-- > 0)
1146   {
1147   size_t len;
1148
1149   /* Previous field was the last one in the string. For a positive field
1150   number, this means there are not enough fields. For a negative field number,
1151   check that there are enough, and scan back to find the one that is wanted. */
1152
1153   if (sep == 0)
1154     {
1155     if (field > 0 || (-field) > (INT_MAX - count - 1)) return NULL;
1156     if ((-field) == (INT_MAX - count - 1)) return s;
1157     while (field++ < 0)
1158       {
1159       ss--;
1160       while (ss[-1] != 0) ss--;
1161       }
1162     fieldtext = ss;
1163     break;
1164     }
1165
1166   /* Previous field was not last in the string; save its start and put a
1167   zero at its end. */
1168
1169   fieldtext = ss;
1170   len = Ustrcspn(ss, separators);
1171   sep = ss[len];
1172   ss[len] = 0;
1173   ss += len + 1;
1174   }
1175
1176 return fieldtext;
1177 }
1178
1179
1180 static uschar *
1181 expand_getlistele(int field, uschar * list)
1182 {
1183 uschar * tlist= list;
1184 int sep= 0;
1185 uschar dummy;
1186
1187 if(field<0)
1188 {
1189   for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++;
1190   sep= 0;
1191 }
1192 if(field==0) return NULL;
1193 while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ;
1194 return string_nextinlist(&list, &sep, NULL, 0);
1195 }
1196
1197
1198 /* Certificate fields, by name.  Worry about by-OID later */
1199 /* Names are chosen to not have common prefixes */
1200
1201 #ifdef SUPPORT_TLS
1202 typedef struct
1203 {
1204 uschar * name;
1205 int      namelen;
1206 uschar * (*getfn)(void * cert, uschar * mod);
1207 } certfield;
1208 static certfield certfields[] =
1209 {                       /* linear search; no special order */
1210   { US"version",         7,  &tls_cert_version },
1211   { US"serial_number",   13, &tls_cert_serial_number },
1212   { US"subject",         7,  &tls_cert_subject },
1213   { US"notbefore",       9,  &tls_cert_not_before },
1214   { US"notafter",        8,  &tls_cert_not_after },
1215   { US"issuer",          6,  &tls_cert_issuer },
1216   { US"signature",       9,  &tls_cert_signature },
1217   { US"sig_algorithm",   13, &tls_cert_signature_algorithm },
1218   { US"subj_altname",    12, &tls_cert_subject_altname },
1219   { US"ocsp_uri",        8,  &tls_cert_ocsp_uri },
1220   { US"crl_uri",         7,  &tls_cert_crl_uri },
1221 };
1222
1223 static uschar *
1224 expand_getcertele(uschar * field, uschar * certvar)
1225 {
1226 var_entry * vp;
1227 certfield * cp;
1228
1229 if (!(vp = find_var_ent(certvar)))
1230   {
1231   expand_string_message =
1232     string_sprintf("no variable named \"%s\"", certvar);
1233   return NULL;          /* Unknown variable name */
1234   }
1235 /* NB this stops us passing certs around in variable.  Might
1236 want to do that in future */
1237 if (vp->type != vtype_cert)
1238   {
1239   expand_string_message =
1240     string_sprintf("\"%s\" is not a certificate", certvar);
1241   return NULL;          /* Unknown variable name */
1242   }
1243 if (!*(void **)vp->value)
1244   return NULL;
1245
1246 if (*field >= '0' && *field <= '9')
1247   return tls_cert_ext_by_oid(*(void **)vp->value, field, 0);
1248
1249 for(cp = certfields;
1250     cp < certfields + nelements(certfields);
1251     cp++)
1252   if (Ustrncmp(cp->name, field, cp->namelen) == 0)
1253     {
1254     uschar * modifier = *(field += cp->namelen) == ','
1255       ? ++field : NULL;
1256     return (*cp->getfn)( *(void **)vp->value, modifier );
1257     }
1258
1259 expand_string_message =
1260   string_sprintf("bad field selector \"%s\" for certextract", field);
1261 return NULL;
1262 }
1263 #endif  /*SUPPORT_TLS*/
1264
1265 /*************************************************
1266 *        Extract a substring from a string       *
1267 *************************************************/
1268
1269 /* Perform the ${substr or ${length expansion operations.
1270
1271 Arguments:
1272   subject     the input string
1273   value1      the offset from the start of the input string to the start of
1274                 the output string; if negative, count from the right.
1275   value2      the length of the output string, or negative (-1) for unset
1276                 if value1 is positive, unset means "all after"
1277                 if value1 is negative, unset means "all before"
1278   len         set to the length of the returned string
1279
1280 Returns:      pointer to the output string, or NULL if there is an error
1281 */
1282
1283 static uschar *
1284 extract_substr(uschar *subject, int value1, int value2, int *len)
1285 {
1286 int sublen = Ustrlen(subject);
1287
1288 if (value1 < 0)    /* count from right */
1289   {
1290   value1 += sublen;
1291
1292   /* If the position is before the start, skip to the start, and adjust the
1293   length. If the length ends up negative, the substring is null because nothing
1294   can precede. This falls out naturally when the length is unset, meaning "all
1295   to the left". */
1296
1297   if (value1 < 0)
1298     {
1299     value2 += value1;
1300     if (value2 < 0) value2 = 0;
1301     value1 = 0;
1302     }
1303
1304   /* Otherwise an unset length => characters before value1 */
1305
1306   else if (value2 < 0)
1307     {
1308     value2 = value1;
1309     value1 = 0;
1310     }
1311   }
1312
1313 /* For a non-negative offset, if the starting position is past the end of the
1314 string, the result will be the null string. Otherwise, an unset length means
1315 "rest"; just set it to the maximum - it will be cut down below if necessary. */
1316
1317 else
1318   {
1319   if (value1 > sublen)
1320     {
1321     value1 = sublen;
1322     value2 = 0;
1323     }
1324   else if (value2 < 0) value2 = sublen;
1325   }
1326
1327 /* Cut the length down to the maximum possible for the offset value, and get
1328 the required characters. */
1329
1330 if (value1 + value2 > sublen) value2 = sublen - value1;
1331 *len = value2;
1332 return subject + value1;
1333 }
1334
1335
1336
1337
1338 /*************************************************
1339 *            Old-style hash of a string          *
1340 *************************************************/
1341
1342 /* Perform the ${hash expansion operation.
1343
1344 Arguments:
1345   subject     the input string (an expanded substring)
1346   value1      the length of the output string; if greater or equal to the
1347                 length of the input string, the input string is returned
1348   value2      the number of hash characters to use, or 26 if negative
1349   len         set to the length of the returned string
1350
1351 Returns:      pointer to the output string, or NULL if there is an error
1352 */
1353
1354 static uschar *
1355 compute_hash(uschar *subject, int value1, int value2, int *len)
1356 {
1357 int sublen = Ustrlen(subject);
1358
1359 if (value2 < 0) value2 = 26;
1360 else if (value2 > Ustrlen(hashcodes))
1361   {
1362   expand_string_message =
1363     string_sprintf("hash count \"%d\" too big", value2);
1364   return NULL;
1365   }
1366
1367 /* Calculate the hash text. We know it is shorter than the original string, so
1368 can safely place it in subject[] (we know that subject is always itself an
1369 expanded substring). */
1370
1371 if (value1 < sublen)
1372   {
1373   int c;
1374   int i = 0;
1375   int j = value1;
1376   while ((c = (subject[j])) != 0)
1377     {
1378     int shift = (c + j++) & 7;
1379     subject[i] ^= (c << shift) | (c >> (8-shift));
1380     if (++i >= value1) i = 0;
1381     }
1382   for (i = 0; i < value1; i++)
1383     subject[i] = hashcodes[(subject[i]) % value2];
1384   }
1385 else value1 = sublen;
1386
1387 *len = value1;
1388 return subject;
1389 }
1390
1391
1392
1393
1394 /*************************************************
1395 *             Numeric hash of a string           *
1396 *************************************************/
1397
1398 /* Perform the ${nhash expansion operation. The first characters of the
1399 string are treated as most important, and get the highest prime numbers.
1400
1401 Arguments:
1402   subject     the input string
1403   value1      the maximum value of the first part of the result
1404   value2      the maximum value of the second part of the result,
1405                 or negative to produce only a one-part result
1406   len         set to the length of the returned string
1407
1408 Returns:  pointer to the output string, or NULL if there is an error.
1409 */
1410
1411 static uschar *
1412 compute_nhash (uschar *subject, int value1, int value2, int *len)
1413 {
1414 uschar *s = subject;
1415 int i = 0;
1416 unsigned long int total = 0; /* no overflow */
1417
1418 while (*s != 0)
1419   {
1420   if (i == 0) i = sizeof(prime)/sizeof(int) - 1;
1421   total += prime[i--] * (unsigned int)(*s++);
1422   }
1423
1424 /* If value2 is unset, just compute one number */
1425
1426 if (value2 < 0)
1427   {
1428   s = string_sprintf("%d", total % value1);
1429   }
1430
1431 /* Otherwise do a div/mod hash */
1432
1433 else
1434   {
1435   total = total % (value1 * value2);
1436   s = string_sprintf("%d/%d", total/value2, total % value2);
1437   }
1438
1439 *len = Ustrlen(s);
1440 return s;
1441 }
1442
1443
1444
1445
1446
1447 /*************************************************
1448 *     Find the value of a header or headers      *
1449 *************************************************/
1450
1451 /* Multiple instances of the same header get concatenated, and this function
1452 can also return a concatenation of all the header lines. When concatenating
1453 specific headers that contain lists of addresses, a comma is inserted between
1454 them. Otherwise we use a straight concatenation. Because some messages can have
1455 pathologically large number of lines, there is a limit on the length that is
1456 returned. Also, to avoid massive store use which would result from using
1457 string_cat() as it copies and extends strings, we do a preliminary pass to find
1458 out exactly how much store will be needed. On "normal" messages this will be
1459 pretty trivial.
1460
1461 Arguments:
1462   name          the name of the header, without the leading $header_ or $h_,
1463                 or NULL if a concatenation of all headers is required
1464   exists_only   TRUE if called from a def: test; don't need to build a string;
1465                 just return a string that is not "" and not "0" if the header
1466                 exists
1467   newsize       return the size of memory block that was obtained; may be NULL
1468                 if exists_only is TRUE
1469   want_raw      TRUE if called for $rh_ or $rheader_ variables; no processing,
1470                 other than concatenating, will be done on the header. Also used
1471                 for $message_headers_raw.
1472   charset       name of charset to translate MIME words to; used only if
1473                 want_raw is false; if NULL, no translation is done (this is
1474                 used for $bh_ and $bheader_)
1475
1476 Returns:        NULL if the header does not exist, else a pointer to a new
1477                 store block
1478 */
1479
1480 static uschar *
1481 find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw,
1482   uschar *charset)
1483 {
1484 BOOL found = name == NULL;
1485 int comma = 0;
1486 int len = found? 0 : Ustrlen(name);
1487 int i;
1488 uschar *yield = NULL;
1489 uschar *ptr = NULL;
1490
1491 /* Loop for two passes - saves code repetition */
1492
1493 for (i = 0; i < 2; i++)
1494   {
1495   int size = 0;
1496   header_line *h;
1497
1498   for (h = header_list; size < header_insert_maxlen && h != NULL; h = h->next)
1499     {
1500     if (h->type != htype_old && h->text != NULL)  /* NULL => Received: placeholder */
1501       {
1502       if (name == NULL || (len <= h->slen && strncmpic(name, h->text, len) == 0))
1503         {
1504         int ilen;
1505         uschar *t;
1506
1507         if (exists_only) return US"1";      /* don't need actual string */
1508         found = TRUE;
1509         t = h->text + len;                  /* text to insert */
1510         if (!want_raw)                      /* unless wanted raw, */
1511           while (isspace(*t)) t++;          /* remove leading white space */
1512         ilen = h->slen - (t - h->text);     /* length to insert */
1513
1514         /* Unless wanted raw, remove trailing whitespace, including the
1515         newline. */
1516
1517         if (!want_raw)
1518           while (ilen > 0 && isspace(t[ilen-1])) ilen--;
1519
1520         /* Set comma = 1 if handling a single header and it's one of those
1521         that contains an address list, except when asked for raw headers. Only
1522         need to do this once. */
1523
1524         if (!want_raw && name != NULL && comma == 0 &&
1525             Ustrchr("BCFRST", h->type) != NULL)
1526           comma = 1;
1527
1528         /* First pass - compute total store needed; second pass - compute
1529         total store used, including this header. */
1530
1531         size += ilen + comma + 1;  /* +1 for the newline */
1532
1533         /* Second pass - concatentate the data, up to a maximum. Note that
1534         the loop stops when size hits the limit. */
1535
1536         if (i != 0)
1537           {
1538           if (size > header_insert_maxlen)
1539             {
1540             ilen -= size - header_insert_maxlen - 1;
1541             comma = 0;
1542             }
1543           Ustrncpy(ptr, t, ilen);
1544           ptr += ilen;
1545
1546           /* For a non-raw header, put in the comma if needed, then add
1547           back the newline we removed above, provided there was some text in
1548           the header. */
1549
1550           if (!want_raw && ilen > 0)
1551             {
1552             if (comma != 0) *ptr++ = ',';
1553             *ptr++ = '\n';
1554             }
1555           }
1556         }
1557       }
1558     }
1559
1560   /* At end of first pass, return NULL if no header found. Then truncate size
1561   if necessary, and get the buffer to hold the data, returning the buffer size.
1562   */
1563
1564   if (i == 0)
1565     {
1566     if (!found) return NULL;
1567     if (size > header_insert_maxlen) size = header_insert_maxlen;
1568     *newsize = size + 1;
1569     ptr = yield = store_get(*newsize);
1570     }
1571   }
1572
1573 /* That's all we do for raw header expansion. */
1574
1575 if (want_raw)
1576   {
1577   *ptr = 0;
1578   }
1579
1580 /* Otherwise, remove a final newline and a redundant added comma. Then we do
1581 RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2()
1582 function can return an error with decoded data if the charset translation
1583 fails. If decoding fails, it returns NULL. */
1584
1585 else
1586   {
1587   uschar *decoded, *error;
1588   if (ptr > yield && ptr[-1] == '\n') ptr--;
1589   if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--;
1590   *ptr = 0;
1591   decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL,
1592     newsize, &error);
1593   if (error != NULL)
1594     {
1595     DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n"
1596       "    input was: %s\n", error, yield);
1597     }
1598   if (decoded != NULL) yield = decoded;
1599   }
1600
1601 return yield;
1602 }
1603
1604
1605
1606
1607 /*************************************************
1608 *               Return list of recipients        *
1609 *************************************************/
1610 /* A recipients list is available only during system message filtering,
1611 during ACL processing after DATA, and while expanding pipe commands
1612 generated from a system filter, but not elsewhere. */
1613
1614 static uschar *
1615 fn_recipients(void)
1616 {
1617 if (!enable_dollar_recipients) return NULL; else
1618   {
1619   int size = 128;
1620   int ptr = 0;
1621   int i;
1622   uschar * s = store_get(size);
1623   for (i = 0; i < recipients_count; i++)
1624     {
1625     if (i != 0) s = string_cat(s, &size, &ptr, US", ", 2);
1626     s = string_cat(s, &size, &ptr, recipients_list[i].address,
1627       Ustrlen(recipients_list[i].address));
1628     }
1629   s[ptr] = 0;     /* string_cat() leaves room */
1630   return s;
1631   }
1632 }
1633
1634
1635 /*************************************************
1636 *               Find value of a variable         *
1637 *************************************************/
1638
1639 /* The table of variables is kept in alphabetic order, so we can search it
1640 using a binary chop. The "choplen" variable is nothing to do with the binary
1641 chop.
1642
1643 Arguments:
1644   name          the name of the variable being sought
1645   exists_only   TRUE if this is a def: test; passed on to find_header()
1646   skipping      TRUE => skip any processing evaluation; this is not the same as
1647                   exists_only because def: may test for values that are first
1648                   evaluated here
1649   newsize       pointer to an int which is initially zero; if the answer is in
1650                 a new memory buffer, *newsize is set to its size
1651
1652 Returns:        NULL if the variable does not exist, or
1653                 a pointer to the variable's contents, or
1654                 something non-NULL if exists_only is TRUE
1655 */
1656
1657 static uschar *
1658 find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize)
1659 {
1660 var_entry * vp;
1661 uschar *s, *domain;
1662 uschar **ss;
1663 void * val;
1664
1665 /* Handle ACL variables, whose names are of the form acl_cxxx or acl_mxxx.
1666 Originally, xxx had to be a number in the range 0-9 (later 0-19), but from
1667 release 4.64 onwards arbitrary names are permitted, as long as the first 5
1668 characters are acl_c or acl_m and the sixth is either a digit or an underscore
1669 (this gave backwards compatibility at the changeover). There may be built-in
1670 variables whose names start acl_ but they should never start in this way. This
1671 slightly messy specification is a consequence of the history, needless to say.
1672
1673 If an ACL variable does not exist, treat it as empty, unless strict_acl_vars is
1674 set, in which case give an error. */
1675
1676 if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) &&
1677      !isalpha(name[5]))
1678   {
1679   tree_node *node =
1680     tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4);
1681   return (node == NULL)? (strict_acl_vars? NULL : US"") : node->data.ptr;
1682   }
1683
1684 /* Handle $auth<n> variables. */
1685
1686 if (Ustrncmp(name, "auth", 4) == 0)
1687   {
1688   uschar *endptr;
1689   int n = Ustrtoul(name + 4, &endptr, 10);
1690   if (*endptr == 0 && n != 0 && n <= AUTH_VARS)
1691     return (auth_vars[n-1] == NULL)? US"" : auth_vars[n-1];
1692   }
1693
1694 /* For all other variables, search the table */
1695
1696 if (!(vp = find_var_ent(name)))
1697   return NULL;          /* Unknown variable name */
1698
1699 /* Found an existing variable. If in skipping state, the value isn't needed,
1700 and we want to avoid processing (such as looking up the host name). */
1701
1702 if (skipping)
1703   return US"";
1704
1705 val = vp->value;
1706 switch (vp->type)
1707   {
1708   case vtype_filter_int:
1709   if (!filter_running) return NULL;
1710   /* Fall through */
1711   /* VVVVVVVVVVVV */
1712   case vtype_int:
1713   sprintf(CS var_buffer, "%d", *(int *)(val)); /* Integer */
1714   return var_buffer;
1715
1716   case vtype_ino:
1717   sprintf(CS var_buffer, "%ld", (long int)(*(ino_t *)(val))); /* Inode */
1718   return var_buffer;
1719
1720   case vtype_gid:
1721   sprintf(CS var_buffer, "%ld", (long int)(*(gid_t *)(val))); /* gid */
1722   return var_buffer;
1723
1724   case vtype_uid:
1725   sprintf(CS var_buffer, "%ld", (long int)(*(uid_t *)(val))); /* uid */
1726   return var_buffer;
1727
1728   case vtype_bool:
1729   sprintf(CS var_buffer, "%s", *(BOOL *)(val) ? "yes" : "no"); /* bool */
1730   return var_buffer;
1731
1732   case vtype_stringptr:                      /* Pointer to string */
1733   s = *((uschar **)(val));
1734   return (s == NULL)? US"" : s;
1735
1736   case vtype_pid:
1737   sprintf(CS var_buffer, "%d", (int)getpid()); /* pid */
1738   return var_buffer;
1739
1740   case vtype_load_avg:
1741   sprintf(CS var_buffer, "%d", OS_GETLOADAVG()); /* load_average */
1742   return var_buffer;
1743
1744   case vtype_host_lookup:                    /* Lookup if not done so */
1745   if (sender_host_name == NULL && sender_host_address != NULL &&
1746       !host_lookup_failed && host_name_lookup() == OK)
1747     host_build_sender_fullhost();
1748   return (sender_host_name == NULL)? US"" : sender_host_name;
1749
1750   case vtype_localpart:                      /* Get local part from address */
1751   s = *((uschar **)(val));
1752   if (s == NULL) return US"";
1753   domain = Ustrrchr(s, '@');
1754   if (domain == NULL) return s;
1755   if (domain - s > sizeof(var_buffer) - 1)
1756     log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT
1757         " in string expansion", sizeof(var_buffer));
1758   Ustrncpy(var_buffer, s, domain - s);
1759   var_buffer[domain - s] = 0;
1760   return var_buffer;
1761
1762   case vtype_domain:                         /* Get domain from address */
1763   s = *((uschar **)(val));
1764   if (s == NULL) return US"";
1765   domain = Ustrrchr(s, '@');
1766   return (domain == NULL)? US"" : domain + 1;
1767
1768   case vtype_msgheaders:
1769   return find_header(NULL, exists_only, newsize, FALSE, NULL);
1770
1771   case vtype_msgheaders_raw:
1772   return find_header(NULL, exists_only, newsize, TRUE, NULL);
1773
1774   case vtype_msgbody:                        /* Pointer to msgbody string */
1775   case vtype_msgbody_end:                    /* Ditto, the end of the msg */
1776   ss = (uschar **)(val);
1777   if (*ss == NULL && deliver_datafile >= 0)  /* Read body when needed */
1778     {
1779     uschar *body;
1780     off_t start_offset = SPOOL_DATA_START_OFFSET;
1781     int len = message_body_visible;
1782     if (len > message_size) len = message_size;
1783     *ss = body = store_malloc(len+1);
1784     body[0] = 0;
1785     if (vp->type == vtype_msgbody_end)
1786       {
1787       struct stat statbuf;
1788       if (fstat(deliver_datafile, &statbuf) == 0)
1789         {
1790         start_offset = statbuf.st_size - len;
1791         if (start_offset < SPOOL_DATA_START_OFFSET)
1792           start_offset = SPOOL_DATA_START_OFFSET;
1793         }
1794       }
1795     lseek(deliver_datafile, start_offset, SEEK_SET);
1796     len = read(deliver_datafile, body, len);
1797     if (len > 0)
1798       {
1799       body[len] = 0;
1800       if (message_body_newlines)   /* Separate loops for efficiency */
1801         {
1802         while (len > 0)
1803           { if (body[--len] == 0) body[len] = ' '; }
1804         }
1805       else
1806         {
1807         while (len > 0)
1808           { if (body[--len] == '\n' || body[len] == 0) body[len] = ' '; }
1809         }
1810       }
1811     }
1812   return (*ss == NULL)? US"" : *ss;
1813
1814   case vtype_todbsdin:                       /* BSD inbox time of day */
1815   return tod_stamp(tod_bsdin);
1816
1817   case vtype_tode:                           /* Unix epoch time of day */
1818   return tod_stamp(tod_epoch);
1819
1820   case vtype_todel:                          /* Unix epoch/usec time of day */
1821   return tod_stamp(tod_epoch_l);
1822
1823   case vtype_todf:                           /* Full time of day */
1824   return tod_stamp(tod_full);
1825
1826   case vtype_todl:                           /* Log format time of day */
1827   return tod_stamp(tod_log_bare);            /* (without timezone) */
1828
1829   case vtype_todzone:                        /* Time zone offset only */
1830   return tod_stamp(tod_zone);
1831
1832   case vtype_todzulu:                        /* Zulu time */
1833   return tod_stamp(tod_zulu);
1834
1835   case vtype_todlf:                          /* Log file datestamp tod */
1836   return tod_stamp(tod_log_datestamp_daily);
1837
1838   case vtype_reply:                          /* Get reply address */
1839   s = find_header(US"reply-to:", exists_only, newsize, TRUE,
1840     headers_charset);
1841   if (s != NULL) while (isspace(*s)) s++;
1842   if (s == NULL || *s == 0)
1843     {
1844     *newsize = 0;                            /* For the *s==0 case */
1845     s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset);
1846     }
1847   if (s != NULL)
1848     {
1849     uschar *t;
1850     while (isspace(*s)) s++;
1851     for (t = s; *t != 0; t++) if (*t == '\n') *t = ' ';
1852     while (t > s && isspace(t[-1])) t--;
1853     *t = 0;
1854     }
1855   return (s == NULL)? US"" : s;
1856
1857   case vtype_string_func:
1858     {
1859     uschar * (*fn)() = val;
1860     return fn();
1861     }
1862
1863   case vtype_pspace:
1864     {
1865     int inodes;
1866     sprintf(CS var_buffer, "%d",
1867       receive_statvfs(val == (void *)TRUE, &inodes));
1868     }
1869   return var_buffer;
1870
1871   case vtype_pinodes:
1872     {
1873     int inodes;
1874     (void) receive_statvfs(val == (void *)TRUE, &inodes);
1875     sprintf(CS var_buffer, "%d", inodes);
1876     }
1877   return var_buffer;
1878
1879   case vtype_cert:
1880   return *(void **)val ? US"<cert>" : US"";
1881
1882   #ifndef DISABLE_DKIM
1883   case vtype_dkim:
1884   return dkim_exim_expand_query((int)(long)val);
1885   #endif
1886
1887   }
1888
1889 return NULL;  /* Unknown variable. Silences static checkers. */
1890 }
1891
1892
1893
1894
1895 void
1896 modify_variable(uschar *name, void * value)
1897 {
1898 var_entry * vp;
1899 if ((vp = find_var_ent(name))) vp->value = value;
1900 return;          /* Unknown variable name, fail silently */
1901 }
1902
1903
1904
1905
1906
1907 /*************************************************
1908 *           Read and expand substrings           *
1909 *************************************************/
1910
1911 /* This function is called to read and expand argument substrings for various
1912 expansion items. Some have a minimum requirement that is less than the maximum;
1913 in these cases, the first non-present one is set to NULL.
1914
1915 Arguments:
1916   sub        points to vector of pointers to set
1917   n          maximum number of substrings
1918   m          minimum required
1919   sptr       points to current string pointer
1920   skipping   the skipping flag
1921   check_end  if TRUE, check for final '}'
1922   name       name of item, for error message
1923   resetok    if not NULL, pointer to flag - write FALSE if unsafe to reset
1924              the store.
1925
1926 Returns:     0 OK; string pointer updated
1927              1 curly bracketing error (too few arguments)
1928              2 too many arguments (only if check_end is set); message set
1929              3 other error (expansion failure)
1930 */
1931
1932 static int
1933 read_subs(uschar **sub, int n, int m, uschar **sptr, BOOL skipping,
1934   BOOL check_end, uschar *name, BOOL *resetok)
1935 {
1936 int i;
1937 uschar *s = *sptr;
1938
1939 while (isspace(*s)) s++;
1940 for (i = 0; i < n; i++)
1941   {
1942   if (*s != '{')
1943     {
1944     if (i < m) return 1;
1945     sub[i] = NULL;
1946     break;
1947     }
1948   sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok);
1949   if (sub[i] == NULL) return 3;
1950   if (*s++ != '}') return 1;
1951   while (isspace(*s)) s++;
1952   }
1953 if (check_end && *s++ != '}')
1954   {
1955   if (s[-1] == '{')
1956     {
1957     expand_string_message = string_sprintf("Too many arguments for \"%s\" "
1958       "(max is %d)", name, n);
1959     return 2;
1960     }
1961   return 1;
1962   }
1963
1964 *sptr = s;
1965 return 0;
1966 }
1967
1968
1969
1970
1971 /*************************************************
1972 *     Elaborate message for bad variable         *
1973 *************************************************/
1974
1975 /* For the "unknown variable" message, take a look at the variable's name, and
1976 give additional information about possible ACL variables. The extra information
1977 is added on to expand_string_message.
1978
1979 Argument:   the name of the variable
1980 Returns:    nothing
1981 */
1982
1983 static void
1984 check_variable_error_message(uschar *name)
1985 {
1986 if (Ustrncmp(name, "acl_", 4) == 0)
1987   expand_string_message = string_sprintf("%s (%s)", expand_string_message,
1988     (name[4] == 'c' || name[4] == 'm')?
1989       (isalpha(name[5])?
1990         US"6th character of a user-defined ACL variable must be a digit or underscore" :
1991         US"strict_acl_vars is set"    /* Syntax is OK, it has to be this */
1992       ) :
1993       US"user-defined ACL variables must start acl_c or acl_m");
1994 }
1995
1996
1997
1998 /*
1999 Load args from sub array to globals, and call acl_check().
2000 Sub array will be corrupted on return.
2001
2002 Returns:       OK         access is granted by an ACCEPT verb
2003                DISCARD    access is granted by a DISCARD verb
2004                FAIL       access is denied
2005                FAIL_DROP  access is denied; drop the connection
2006                DEFER      can't tell at the moment
2007                ERROR      disaster
2008 */
2009 static int
2010 eval_acl(uschar ** sub, int nsub, uschar ** user_msgp)
2011 {
2012 int i;
2013 uschar *tmp;
2014 int sav_narg = acl_narg;
2015 int ret;
2016 extern int acl_where;
2017
2018 if(--nsub > sizeof(acl_arg)/sizeof(*acl_arg)) nsub = sizeof(acl_arg)/sizeof(*acl_arg);
2019 for (i = 0; i < nsub && sub[i+1]; i++)
2020   {
2021   tmp = acl_arg[i];
2022   acl_arg[i] = sub[i+1];        /* place callers args in the globals */
2023   sub[i+1] = tmp;               /* stash the old args using our caller's storage */
2024   }
2025 acl_narg = i;
2026 while (i < nsub)
2027   {
2028   sub[i+1] = acl_arg[i];
2029   acl_arg[i++] = NULL;
2030   }
2031
2032 DEBUG(D_expand)
2033   debug_printf("expanding: acl: %s  arg: %s%s\n",
2034     sub[0],
2035     acl_narg>0 ? acl_arg[0] : US"<none>",
2036     acl_narg>1 ? " +more"   : "");
2037
2038 ret = acl_eval(acl_where, sub[0], user_msgp, &tmp);
2039
2040 for (i = 0; i < nsub; i++)
2041   acl_arg[i] = sub[i+1];        /* restore old args */
2042 acl_narg = sav_narg;
2043
2044 return ret;
2045 }
2046
2047
2048
2049
2050 /*************************************************
2051 *        Read and evaluate a condition           *
2052 *************************************************/
2053
2054 /*
2055 Arguments:
2056   s        points to the start of the condition text
2057   resetok  points to a BOOL which is written false if it is unsafe to
2058            free memory. Certain condition types (acl) may have side-effect
2059            allocation which must be preserved.
2060   yield    points to a BOOL to hold the result of the condition test;
2061            if NULL, we are just reading through a condition that is
2062            part of an "or" combination to check syntax, or in a state
2063            where the answer isn't required
2064
2065 Returns:   a pointer to the first character after the condition, or
2066            NULL after an error
2067 */
2068
2069 static uschar *
2070 eval_condition(uschar *s, BOOL *resetok, BOOL *yield)
2071 {
2072 BOOL testfor = TRUE;
2073 BOOL tempcond, combined_cond;
2074 BOOL *subcondptr;
2075 BOOL sub2_honour_dollar = TRUE;
2076 int i, rc, cond_type, roffset;
2077 int_eximarith_t num[2];
2078 struct stat statbuf;
2079 uschar name[256];
2080 uschar *sub[10];
2081
2082 const pcre *re;
2083 const uschar *rerror;
2084
2085 for (;;)
2086   {
2087   while (isspace(*s)) s++;
2088   if (*s == '!') { testfor = !testfor; s++; } else break;
2089   }
2090
2091 /* Numeric comparisons are symbolic */
2092
2093 if (*s == '=' || *s == '>' || *s == '<')
2094   {
2095   int p = 0;
2096   name[p++] = *s++;
2097   if (*s == '=')
2098     {
2099     name[p++] = '=';
2100     s++;
2101     }
2102   name[p] = 0;
2103   }
2104
2105 /* All other conditions are named */
2106
2107 else s = read_name(name, 256, s, US"_");
2108
2109 /* If we haven't read a name, it means some non-alpha character is first. */
2110
2111 if (name[0] == 0)
2112   {
2113   expand_string_message = string_sprintf("condition name expected, "
2114     "but found \"%.16s\"", s);
2115   return NULL;
2116   }
2117
2118 /* Find which condition we are dealing with, and switch on it */
2119
2120 cond_type = chop_match(name, cond_table, sizeof(cond_table)/sizeof(uschar *));
2121 switch(cond_type)
2122   {
2123   /* def: tests for a non-empty variable, or for the existence of a header. If
2124   yield == NULL we are in a skipping state, and don't care about the answer. */
2125
2126   case ECOND_DEF:
2127   if (*s != ':')
2128     {
2129     expand_string_message = US"\":\" expected after \"def\"";
2130     return NULL;
2131     }
2132
2133   s = read_name(name, 256, s+1, US"_");
2134
2135   /* Test for a header's existence. If the name contains a closing brace
2136   character, this may be a user error where the terminating colon has been
2137   omitted. Set a flag to adjust a subsequent error message in this case. */
2138
2139   if (Ustrncmp(name, "h_", 2) == 0 ||
2140       Ustrncmp(name, "rh_", 3) == 0 ||
2141       Ustrncmp(name, "bh_", 3) == 0 ||
2142       Ustrncmp(name, "header_", 7) == 0 ||
2143       Ustrncmp(name, "rheader_", 8) == 0 ||
2144       Ustrncmp(name, "bheader_", 8) == 0)
2145     {
2146     s = read_header_name(name, 256, s);
2147     /* {-for-text-editors */
2148     if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
2149     if (yield != NULL) *yield =
2150       (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor;
2151     }
2152
2153   /* Test for a variable's having a non-empty value. A non-existent variable
2154   causes an expansion failure. */
2155
2156   else
2157     {
2158     uschar *value = find_variable(name, TRUE, yield == NULL, NULL);
2159     if (value == NULL)
2160       {
2161       expand_string_message = (name[0] == 0)?
2162         string_sprintf("variable name omitted after \"def:\"") :
2163         string_sprintf("unknown variable \"%s\" after \"def:\"", name);
2164       check_variable_error_message(name);
2165       return NULL;
2166       }
2167     if (yield != NULL) *yield = (value[0] != 0) == testfor;
2168     }
2169
2170   return s;
2171
2172
2173   /* first_delivery tests for first delivery attempt */
2174
2175   case ECOND_FIRST_DELIVERY:
2176   if (yield != NULL) *yield = deliver_firsttime == testfor;
2177   return s;
2178
2179
2180   /* queue_running tests for any process started by a queue runner */
2181
2182   case ECOND_QUEUE_RUNNING:
2183   if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor;
2184   return s;
2185
2186
2187   /* exists:  tests for file existence
2188        isip:  tests for any IP address
2189       isip4:  tests for an IPv4 address
2190       isip6:  tests for an IPv6 address
2191         pam:  does PAM authentication
2192      radius:  does RADIUS authentication
2193    ldapauth:  does LDAP authentication
2194     pwcheck:  does Cyrus SASL pwcheck authentication
2195   */
2196
2197   case ECOND_EXISTS:
2198   case ECOND_ISIP:
2199   case ECOND_ISIP4:
2200   case ECOND_ISIP6:
2201   case ECOND_PAM:
2202   case ECOND_RADIUS:
2203   case ECOND_LDAPAUTH:
2204   case ECOND_PWCHECK:
2205
2206   while (isspace(*s)) s++;
2207   if (*s != '{') goto COND_FAILED_CURLY_START;          /* }-for-text-editors */
2208
2209   sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok);
2210   if (sub[0] == NULL) return NULL;
2211   /* {-for-text-editors */
2212   if (*s++ != '}') goto COND_FAILED_CURLY_END;
2213
2214   if (yield == NULL) return s;   /* No need to run the test if skipping */
2215
2216   switch(cond_type)
2217     {
2218     case ECOND_EXISTS:
2219     if ((expand_forbid & RDO_EXISTS) != 0)
2220       {
2221       expand_string_message = US"File existence tests are not permitted";
2222       return NULL;
2223       }
2224     *yield = (Ustat(sub[0], &statbuf) == 0) == testfor;
2225     break;
2226
2227     case ECOND_ISIP:
2228     case ECOND_ISIP4:
2229     case ECOND_ISIP6:
2230     rc = string_is_ip_address(sub[0], NULL);
2231     *yield = ((cond_type == ECOND_ISIP)? (rc != 0) :
2232              (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor;
2233     break;
2234
2235     /* Various authentication tests - all optionally compiled */
2236
2237     case ECOND_PAM:
2238     #ifdef SUPPORT_PAM
2239     rc = auth_call_pam(sub[0], &expand_string_message);
2240     goto END_AUTH;
2241     #else
2242     goto COND_FAILED_NOT_COMPILED;
2243     #endif  /* SUPPORT_PAM */
2244
2245     case ECOND_RADIUS:
2246     #ifdef RADIUS_CONFIG_FILE
2247     rc = auth_call_radius(sub[0], &expand_string_message);
2248     goto END_AUTH;
2249     #else
2250     goto COND_FAILED_NOT_COMPILED;
2251     #endif  /* RADIUS_CONFIG_FILE */
2252
2253     case ECOND_LDAPAUTH:
2254     #ifdef LOOKUP_LDAP
2255       {
2256       /* Just to keep the interface the same */
2257       BOOL do_cache;
2258       int old_pool = store_pool;
2259       store_pool = POOL_SEARCH;
2260       rc = eldapauth_find((void *)(-1), NULL, sub[0], Ustrlen(sub[0]), NULL,
2261         &expand_string_message, &do_cache);
2262       store_pool = old_pool;
2263       }
2264     goto END_AUTH;
2265     #else
2266     goto COND_FAILED_NOT_COMPILED;
2267     #endif  /* LOOKUP_LDAP */
2268
2269     case ECOND_PWCHECK:
2270     #ifdef CYRUS_PWCHECK_SOCKET
2271     rc = auth_call_pwcheck(sub[0], &expand_string_message);
2272     goto END_AUTH;
2273     #else
2274     goto COND_FAILED_NOT_COMPILED;
2275     #endif  /* CYRUS_PWCHECK_SOCKET */
2276
2277     #if defined(SUPPORT_PAM) || defined(RADIUS_CONFIG_FILE) || \
2278         defined(LOOKUP_LDAP) || defined(CYRUS_PWCHECK_SOCKET)
2279     END_AUTH:
2280     if (rc == ERROR || rc == DEFER) return NULL;
2281     *yield = (rc == OK) == testfor;
2282     #endif
2283     }
2284   return s;
2285
2286
2287   /* call ACL (in a conditional context).  Accept true, deny false.
2288   Defer is a forced-fail.  Anything set by message= goes to $value.
2289   Up to ten parameters are used; we use the braces round the name+args
2290   like the saslauthd condition does, to permit a variable number of args.
2291   See also the expansion-item version EITEM_ACL and the traditional
2292   acl modifier ACLC_ACL.
2293   Since the ACL may allocate new global variables, tell our caller to not
2294   reclaim memory.
2295   */
2296
2297   case ECOND_ACL:
2298     /* ${if acl {{name}{arg1}{arg2}...}  {yes}{no}} */
2299     {
2300     uschar *user_msg;
2301     BOOL cond = FALSE;
2302     int size = 0;
2303     int ptr = 0;
2304
2305     while (isspace(*s)) s++;
2306     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /*}*/
2307
2308     switch(read_subs(sub, sizeof(sub)/sizeof(*sub), 1,
2309       &s, yield == NULL, TRUE, US"acl", resetok))
2310       {
2311       case 1: expand_string_message = US"too few arguments or bracketing "
2312         "error for acl";
2313       case 2:
2314       case 3: return NULL;
2315       }
2316
2317     *resetok = FALSE;
2318     if (yield != NULL) switch(eval_acl(sub, sizeof(sub)/sizeof(*sub), &user_msg))
2319         {
2320         case OK:
2321           cond = TRUE;
2322         case FAIL:
2323           lookup_value = NULL;
2324           if (user_msg)
2325             {
2326             lookup_value = string_cat(NULL, &size, &ptr, user_msg, Ustrlen(user_msg));
2327             lookup_value[ptr] = '\0';
2328             }
2329           *yield = cond == testfor;
2330           break;
2331
2332         case DEFER:
2333           expand_string_forcedfail = TRUE;
2334         default:
2335           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
2336           return NULL;
2337         }
2338     return s;
2339     }
2340
2341
2342   /* saslauthd: does Cyrus saslauthd authentication. Four parameters are used:
2343
2344      ${if saslauthd {{username}{password}{service}{realm}}  {yes}{no}}
2345
2346   However, the last two are optional. That is why the whole set is enclosed
2347   in their own set of braces. */
2348
2349   case ECOND_SASLAUTHD:
2350   #ifndef CYRUS_SASLAUTHD_SOCKET
2351   goto COND_FAILED_NOT_COMPILED;
2352   #else
2353   while (isspace(*s)) s++;
2354   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2355   switch(read_subs(sub, 4, 2, &s, yield == NULL, TRUE, US"saslauthd", resetok))
2356     {
2357     case 1: expand_string_message = US"too few arguments or bracketing "
2358       "error for saslauthd";
2359     case 2:
2360     case 3: return NULL;
2361     }
2362   if (sub[2] == NULL) sub[3] = NULL;  /* realm if no service */
2363   if (yield != NULL)
2364     {
2365     int rc;
2366     rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3],
2367       &expand_string_message);
2368     if (rc == ERROR || rc == DEFER) return NULL;
2369     *yield = (rc == OK) == testfor;
2370     }
2371   return s;
2372   #endif /* CYRUS_SASLAUTHD_SOCKET */
2373
2374
2375   /* symbolic operators for numeric and string comparison, and a number of
2376   other operators, all requiring two arguments.
2377
2378   crypteq:           encrypts plaintext and compares against an encrypted text,
2379                        using crypt(), crypt16(), MD5 or SHA-1
2380   inlist/inlisti:    checks if first argument is in the list of the second
2381   match:             does a regular expression match and sets up the numerical
2382                        variables if it succeeds
2383   match_address:     matches in an address list
2384   match_domain:      matches in a domain list
2385   match_ip:          matches a host list that is restricted to IP addresses
2386   match_local_part:  matches in a local part list
2387   */
2388
2389   case ECOND_MATCH_ADDRESS:
2390   case ECOND_MATCH_DOMAIN:
2391   case ECOND_MATCH_IP:
2392   case ECOND_MATCH_LOCAL_PART:
2393 #ifndef EXPAND_LISTMATCH_RHS
2394     sub2_honour_dollar = FALSE;
2395 #endif
2396     /* FALLTHROUGH */
2397
2398   case ECOND_CRYPTEQ:
2399   case ECOND_INLIST:
2400   case ECOND_INLISTI:
2401   case ECOND_MATCH:
2402
2403   case ECOND_NUM_L:     /* Numerical comparisons */
2404   case ECOND_NUM_LE:
2405   case ECOND_NUM_E:
2406   case ECOND_NUM_EE:
2407   case ECOND_NUM_G:
2408   case ECOND_NUM_GE:
2409
2410   case ECOND_STR_LT:    /* String comparisons */
2411   case ECOND_STR_LTI:
2412   case ECOND_STR_LE:
2413   case ECOND_STR_LEI:
2414   case ECOND_STR_EQ:
2415   case ECOND_STR_EQI:
2416   case ECOND_STR_GT:
2417   case ECOND_STR_GTI:
2418   case ECOND_STR_GE:
2419   case ECOND_STR_GEI:
2420
2421   for (i = 0; i < 2; i++)
2422     {
2423     /* Sometimes, we don't expand substrings; too many insecure configurations
2424     created using match_address{}{} and friends, where the second param
2425     includes information from untrustworthy sources. */
2426     BOOL honour_dollar = TRUE;
2427     if ((i > 0) && !sub2_honour_dollar)
2428       honour_dollar = FALSE;
2429
2430     while (isspace(*s)) s++;
2431     if (*s != '{')
2432       {
2433       if (i == 0) goto COND_FAILED_CURLY_START;
2434       expand_string_message = string_sprintf("missing 2nd string in {} "
2435         "after \"%s\"", name);
2436       return NULL;
2437       }
2438     sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL,
2439         honour_dollar, resetok);
2440     if (sub[i] == NULL) return NULL;
2441     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2442
2443     /* Convert to numerical if required; we know that the names of all the
2444     conditions that compare numbers do not start with a letter. This just saves
2445     checking for them individually. */
2446
2447     if (!isalpha(name[0]) && yield != NULL)
2448       {
2449       if (sub[i][0] == 0)
2450         {
2451         num[i] = 0;
2452         DEBUG(D_expand)
2453           debug_printf("empty string cast to zero for numerical comparison\n");
2454         }
2455       else
2456         {
2457         num[i] = expanded_string_integer(sub[i], FALSE);
2458         if (expand_string_message != NULL) return NULL;
2459         }
2460       }
2461     }
2462
2463   /* Result not required */
2464
2465   if (yield == NULL) return s;
2466
2467   /* Do an appropriate comparison */
2468
2469   switch(cond_type)
2470     {
2471     case ECOND_NUM_E:
2472     case ECOND_NUM_EE:
2473     tempcond = (num[0] == num[1]);
2474     break;
2475
2476     case ECOND_NUM_G:
2477     tempcond = (num[0] > num[1]);
2478     break;
2479
2480     case ECOND_NUM_GE:
2481     tempcond = (num[0] >= num[1]);
2482     break;
2483
2484     case ECOND_NUM_L:
2485     tempcond = (num[0] < num[1]);
2486     break;
2487
2488     case ECOND_NUM_LE:
2489     tempcond = (num[0] <= num[1]);
2490     break;
2491
2492     case ECOND_STR_LT:
2493     tempcond = (Ustrcmp(sub[0], sub[1]) < 0);
2494     break;
2495
2496     case ECOND_STR_LTI:
2497     tempcond = (strcmpic(sub[0], sub[1]) < 0);
2498     break;
2499
2500     case ECOND_STR_LE:
2501     tempcond = (Ustrcmp(sub[0], sub[1]) <= 0);
2502     break;
2503
2504     case ECOND_STR_LEI:
2505     tempcond = (strcmpic(sub[0], sub[1]) <= 0);
2506     break;
2507
2508     case ECOND_STR_EQ:
2509     tempcond = (Ustrcmp(sub[0], sub[1]) == 0);
2510     break;
2511
2512     case ECOND_STR_EQI:
2513     tempcond = (strcmpic(sub[0], sub[1]) == 0);
2514     break;
2515
2516     case ECOND_STR_GT:
2517     tempcond = (Ustrcmp(sub[0], sub[1]) > 0);
2518     break;
2519
2520     case ECOND_STR_GTI:
2521     tempcond = (strcmpic(sub[0], sub[1]) > 0);
2522     break;
2523
2524     case ECOND_STR_GE:
2525     tempcond = (Ustrcmp(sub[0], sub[1]) >= 0);
2526     break;
2527
2528     case ECOND_STR_GEI:
2529     tempcond = (strcmpic(sub[0], sub[1]) >= 0);
2530     break;
2531
2532     case ECOND_MATCH:   /* Regular expression match */
2533     re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
2534       NULL);
2535     if (re == NULL)
2536       {
2537       expand_string_message = string_sprintf("regular expression error in "
2538         "\"%s\": %s at offset %d", sub[1], rerror, roffset);
2539       return NULL;
2540       }
2541     tempcond = regex_match_and_setup(re, sub[0], 0, -1);
2542     break;
2543
2544     case ECOND_MATCH_ADDRESS:  /* Match in an address list */
2545     rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL);
2546     goto MATCHED_SOMETHING;
2547
2548     case ECOND_MATCH_DOMAIN:   /* Match in a domain list */
2549     rc = match_isinlist(sub[0], &(sub[1]), 0, &domainlist_anchor, NULL,
2550       MCL_DOMAIN + MCL_NOEXPAND, TRUE, NULL);
2551     goto MATCHED_SOMETHING;
2552
2553     case ECOND_MATCH_IP:       /* Match IP address in a host list */
2554     if (sub[0][0] != 0 && string_is_ip_address(sub[0], NULL) == 0)
2555       {
2556       expand_string_message = string_sprintf("\"%s\" is not an IP address",
2557         sub[0]);
2558       return NULL;
2559       }
2560     else
2561       {
2562       unsigned int *nullcache = NULL;
2563       check_host_block cb;
2564
2565       cb.host_name = US"";
2566       cb.host_address = sub[0];
2567
2568       /* If the host address starts off ::ffff: it is an IPv6 address in
2569       IPv4-compatible mode. Find the IPv4 part for checking against IPv4
2570       addresses. */
2571
2572       cb.host_ipv4 = (Ustrncmp(cb.host_address, "::ffff:", 7) == 0)?
2573         cb.host_address + 7 : cb.host_address;
2574
2575       rc = match_check_list(
2576              &sub[1],                   /* the list */
2577              0,                         /* separator character */
2578              &hostlist_anchor,          /* anchor pointer */
2579              &nullcache,                /* cache pointer */
2580              check_host,                /* function for testing */
2581              &cb,                       /* argument for function */
2582              MCL_HOST,                  /* type of check */
2583              sub[0],                    /* text for debugging */
2584              NULL);                     /* where to pass back data */
2585       }
2586     goto MATCHED_SOMETHING;
2587
2588     case ECOND_MATCH_LOCAL_PART:
2589     rc = match_isinlist(sub[0], &(sub[1]), 0, &localpartlist_anchor, NULL,
2590       MCL_LOCALPART + MCL_NOEXPAND, TRUE, NULL);
2591     /* Fall through */
2592     /* VVVVVVVVVVVV */
2593     MATCHED_SOMETHING:
2594     switch(rc)
2595       {
2596       case OK:
2597       tempcond = TRUE;
2598       break;
2599
2600       case FAIL:
2601       tempcond = FALSE;
2602       break;
2603
2604       case DEFER:
2605       expand_string_message = string_sprintf("unable to complete match "
2606         "against \"%s\": %s", sub[1], search_error_message);
2607       return NULL;
2608       }
2609
2610     break;
2611
2612     /* Various "encrypted" comparisons. If the second string starts with
2613     "{" then an encryption type is given. Default to crypt() or crypt16()
2614     (build-time choice). */
2615     /* }-for-text-editors */
2616
2617     case ECOND_CRYPTEQ:
2618     #ifndef SUPPORT_CRYPTEQ
2619     goto COND_FAILED_NOT_COMPILED;
2620     #else
2621     if (strncmpic(sub[1], US"{md5}", 5) == 0)
2622       {
2623       int sublen = Ustrlen(sub[1]+5);
2624       md5 base;
2625       uschar digest[16];
2626
2627       md5_start(&base);
2628       md5_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
2629
2630       /* If the length that we are comparing against is 24, the MD5 digest
2631       is expressed as a base64 string. This is the way LDAP does it. However,
2632       some other software uses a straightforward hex representation. We assume
2633       this if the length is 32. Other lengths fail. */
2634
2635       if (sublen == 24)
2636         {
2637         uschar *coded = auth_b64encode((uschar *)digest, 16);
2638         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
2639           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2640         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
2641         }
2642       else if (sublen == 32)
2643         {
2644         int i;
2645         uschar coded[36];
2646         for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2647         coded[32] = 0;
2648         DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n"
2649           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
2650         tempcond = (strcmpic(coded, sub[1]+5) == 0);
2651         }
2652       else
2653         {
2654         DEBUG(D_auth) debug_printf("crypteq: length for MD5 not 24 or 32: "
2655           "fail\n  crypted=%s\n", sub[1]+5);
2656         tempcond = FALSE;
2657         }
2658       }
2659
2660     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
2661       {
2662       int sublen = Ustrlen(sub[1]+6);
2663       sha1 base;
2664       uschar digest[20];
2665
2666       sha1_start(&base);
2667       sha1_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
2668
2669       /* If the length that we are comparing against is 28, assume the SHA1
2670       digest is expressed as a base64 string. If the length is 40, assume a
2671       straightforward hex representation. Other lengths fail. */
2672
2673       if (sublen == 28)
2674         {
2675         uschar *coded = auth_b64encode((uschar *)digest, 20);
2676         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
2677           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2678         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
2679         }
2680       else if (sublen == 40)
2681         {
2682         int i;
2683         uschar coded[44];
2684         for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]);
2685         coded[40] = 0;
2686         DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n"
2687           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
2688         tempcond = (strcmpic(coded, sub[1]+6) == 0);
2689         }
2690       else
2691         {
2692         DEBUG(D_auth) debug_printf("crypteq: length for SHA-1 not 28 or 40: "
2693           "fail\n  crypted=%s\n", sub[1]+6);
2694         tempcond = FALSE;
2695         }
2696       }
2697
2698     else   /* {crypt} or {crypt16} and non-{ at start */
2699            /* }-for-text-editors */
2700       {
2701       int which = 0;
2702       uschar *coded;
2703
2704       if (strncmpic(sub[1], US"{crypt}", 7) == 0)
2705         {
2706         sub[1] += 7;
2707         which = 1;
2708         }
2709       else if (strncmpic(sub[1], US"{crypt16}", 9) == 0)
2710         {
2711         sub[1] += 9;
2712         which = 2;
2713         }
2714       else if (sub[1][0] == '{')                /* }-for-text-editors */
2715         {
2716         expand_string_message = string_sprintf("unknown encryption mechanism "
2717           "in \"%s\"", sub[1]);
2718         return NULL;
2719         }
2720
2721       switch(which)
2722         {
2723         case 0:  coded = US DEFAULT_CRYPT(CS sub[0], CS sub[1]); break;
2724         case 1:  coded = US crypt(CS sub[0], CS sub[1]); break;
2725         default: coded = US crypt16(CS sub[0], CS sub[1]); break;
2726         }
2727
2728       #define STR(s) # s
2729       #define XSTR(s) STR(s)
2730       DEBUG(D_auth) debug_printf("crypteq: using %s()\n"
2731         "  subject=%s\n  crypted=%s\n",
2732         (which == 0)? XSTR(DEFAULT_CRYPT) : (which == 1)? "crypt" : "crypt16",
2733         coded, sub[1]);
2734       #undef STR
2735       #undef XSTR
2736
2737       /* If the encrypted string contains fewer than two characters (for the
2738       salt), force failure. Otherwise we get false positives: with an empty
2739       string the yield of crypt() is an empty string! */
2740
2741       tempcond = (Ustrlen(sub[1]) < 2)? FALSE :
2742         (Ustrcmp(coded, sub[1]) == 0);
2743       }
2744     break;
2745     #endif  /* SUPPORT_CRYPTEQ */
2746
2747     case ECOND_INLIST:
2748     case ECOND_INLISTI:
2749       {
2750       int sep = 0;
2751       uschar *save_iterate_item = iterate_item;
2752       int (*compare)(const uschar *, const uschar *);
2753
2754       tempcond = FALSE;
2755       if (cond_type == ECOND_INLISTI)
2756         compare = strcmpic;
2757       else
2758         compare = (int (*)(const uschar *, const uschar *)) strcmp;
2759
2760       while ((iterate_item = string_nextinlist(&sub[1], &sep, NULL, 0)) != NULL)
2761         if (compare(sub[0], iterate_item) == 0)
2762           {
2763           tempcond = TRUE;
2764           break;
2765           }
2766       iterate_item = save_iterate_item;
2767       }
2768
2769     }   /* Switch for comparison conditions */
2770
2771   *yield = tempcond == testfor;
2772   return s;    /* End of comparison conditions */
2773
2774
2775   /* and/or: computes logical and/or of several conditions */
2776
2777   case ECOND_AND:
2778   case ECOND_OR:
2779   subcondptr = (yield == NULL)? NULL : &tempcond;
2780   combined_cond = (cond_type == ECOND_AND);
2781
2782   while (isspace(*s)) s++;
2783   if (*s++ != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2784
2785   for (;;)
2786     {
2787     while (isspace(*s)) s++;
2788     /* {-for-text-editors */
2789     if (*s == '}') break;
2790     if (*s != '{')                                      /* }-for-text-editors */
2791       {
2792       expand_string_message = string_sprintf("each subcondition "
2793         "inside an \"%s{...}\" condition must be in its own {}", name);
2794       return NULL;
2795       }
2796
2797     if (!(s = eval_condition(s+1, resetok, subcondptr)))
2798       {
2799       expand_string_message = string_sprintf("%s inside \"%s{...}\" condition",
2800         expand_string_message, name);
2801       return NULL;
2802       }
2803     while (isspace(*s)) s++;
2804
2805     /* {-for-text-editors */
2806     if (*s++ != '}')
2807       {
2808       /* {-for-text-editors */
2809       expand_string_message = string_sprintf("missing } at end of condition "
2810         "inside \"%s\" group", name);
2811       return NULL;
2812       }
2813
2814     if (yield != NULL)
2815       {
2816       if (cond_type == ECOND_AND)
2817         {
2818         combined_cond &= tempcond;
2819         if (!combined_cond) subcondptr = NULL;  /* once false, don't */
2820         }                                       /* evaluate any more */
2821       else
2822         {
2823         combined_cond |= tempcond;
2824         if (combined_cond) subcondptr = NULL;   /* once true, don't */
2825         }                                       /* evaluate any more */
2826       }
2827     }
2828
2829   if (yield != NULL) *yield = (combined_cond == testfor);
2830   return ++s;
2831
2832
2833   /* forall/forany: iterates a condition with different values */
2834
2835   case ECOND_FORALL:
2836   case ECOND_FORANY:
2837     {
2838     int sep = 0;
2839     uschar *save_iterate_item = iterate_item;
2840
2841     while (isspace(*s)) s++;
2842     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2843     sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok);
2844     if (sub[0] == NULL) return NULL;
2845     /* {-for-text-editors */
2846     if (*s++ != '}') goto COND_FAILED_CURLY_END;
2847
2848     while (isspace(*s)) s++;
2849     if (*s++ != '{') goto COND_FAILED_CURLY_START;      /* }-for-text-editors */
2850
2851     sub[1] = s;
2852
2853     /* Call eval_condition once, with result discarded (as if scanning a
2854     "false" part). This allows us to find the end of the condition, because if
2855     the list it empty, we won't actually evaluate the condition for real. */
2856
2857     if (!(s = eval_condition(sub[1], resetok, NULL)))
2858       {
2859       expand_string_message = string_sprintf("%s inside \"%s\" condition",
2860         expand_string_message, name);
2861       return NULL;
2862       }
2863     while (isspace(*s)) s++;
2864
2865     /* {-for-text-editors */
2866     if (*s++ != '}')
2867       {
2868       /* {-for-text-editors */
2869       expand_string_message = string_sprintf("missing } at end of condition "
2870         "inside \"%s\"", name);
2871       return NULL;
2872       }
2873
2874     if (yield != NULL) *yield = !testfor;
2875     while ((iterate_item = string_nextinlist(&sub[0], &sep, NULL, 0)) != NULL)
2876       {
2877       DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
2878       if (!eval_condition(sub[1], resetok, &tempcond))
2879         {
2880         expand_string_message = string_sprintf("%s inside \"%s\" condition",
2881           expand_string_message, name);
2882         iterate_item = save_iterate_item;
2883         return NULL;
2884         }
2885       DEBUG(D_expand) debug_printf("%s: condition evaluated to %s\n", name,
2886         tempcond? "true":"false");
2887
2888       if (yield != NULL) *yield = (tempcond == testfor);
2889       if (tempcond == (cond_type == ECOND_FORANY)) break;
2890       }
2891
2892     iterate_item = save_iterate_item;
2893     return s;
2894     }
2895
2896
2897   /* The bool{} expansion condition maps a string to boolean.
2898   The values supported should match those supported by the ACL condition
2899   (acl.c, ACLC_CONDITION) so that we keep to a minimum the different ideas
2900   of true/false.  Note that Router "condition" rules have a different
2901   interpretation, where general data can be used and only a few values
2902   map to FALSE.
2903   Note that readconf.c boolean matching, for boolean configuration options,
2904   only matches true/yes/false/no.
2905   The bool_lax{} condition matches the Router logic, which is much more
2906   liberal. */
2907   case ECOND_BOOL:
2908   case ECOND_BOOL_LAX:
2909     {
2910     uschar *sub_arg[1];
2911     uschar *t, *t2;
2912     uschar *ourname;
2913     size_t len;
2914     BOOL boolvalue = FALSE;
2915     while (isspace(*s)) s++;
2916     if (*s != '{') goto COND_FAILED_CURLY_START;        /* }-for-text-editors */
2917     ourname = cond_type == ECOND_BOOL_LAX ? US"bool_lax" : US"bool";
2918     switch(read_subs(sub_arg, 1, 1, &s, yield == NULL, FALSE, ourname, resetok))
2919       {
2920       case 1: expand_string_message = string_sprintf(
2921                   "too few arguments or bracketing error for %s",
2922                   ourname);
2923       /*FALLTHROUGH*/
2924       case 2:
2925       case 3: return NULL;
2926       }
2927     t = sub_arg[0];
2928     while (isspace(*t)) t++;
2929     len = Ustrlen(t);
2930     if (len)
2931       {
2932       /* trailing whitespace: seems like a good idea to ignore it too */
2933       t2 = t + len - 1;
2934       while (isspace(*t2)) t2--;
2935       if (t2 != (t + len))
2936         {
2937         *++t2 = '\0';
2938         len = t2 - t;
2939         }
2940       }
2941     DEBUG(D_expand)
2942       debug_printf("considering %s: %s\n", ourname, len ? t : US"<empty>");
2943     /* logic for the lax case from expand_check_condition(), which also does
2944     expands, and the logic is both short and stable enough that there should
2945     be no maintenance burden from replicating it. */
2946     if (len == 0)
2947       boolvalue = FALSE;
2948     else if (*t == '-'
2949              ? Ustrspn(t+1, "0123456789") == len-1
2950              : Ustrspn(t,   "0123456789") == len)
2951       {
2952       boolvalue = (Uatoi(t) == 0) ? FALSE : TRUE;
2953       /* expand_check_condition only does a literal string "0" check */
2954       if ((cond_type == ECOND_BOOL_LAX) && (len > 1))
2955         boolvalue = TRUE;
2956       }
2957     else if (strcmpic(t, US"true") == 0 || strcmpic(t, US"yes") == 0)
2958       boolvalue = TRUE;
2959     else if (strcmpic(t, US"false") == 0 || strcmpic(t, US"no") == 0)
2960       boolvalue = FALSE;
2961     else if (cond_type == ECOND_BOOL_LAX)
2962       boolvalue = TRUE;
2963     else
2964       {
2965       expand_string_message = string_sprintf("unrecognised boolean "
2966        "value \"%s\"", t);
2967       return NULL;
2968       }
2969     if (yield != NULL) *yield = (boolvalue == testfor);
2970     return s;
2971     }
2972
2973   /* Unknown condition */
2974
2975   default:
2976   expand_string_message = string_sprintf("unknown condition \"%s\"", name);
2977   return NULL;
2978   }   /* End switch on condition type */
2979
2980 /* Missing braces at start and end of data */
2981
2982 COND_FAILED_CURLY_START:
2983 expand_string_message = string_sprintf("missing { after \"%s\"", name);
2984 return NULL;
2985
2986 COND_FAILED_CURLY_END:
2987 expand_string_message = string_sprintf("missing } at end of \"%s\" condition",
2988   name);
2989 return NULL;
2990
2991 /* A condition requires code that is not compiled */
2992
2993 #if !defined(SUPPORT_PAM) || !defined(RADIUS_CONFIG_FILE) || \
2994     !defined(LOOKUP_LDAP) || !defined(CYRUS_PWCHECK_SOCKET) || \
2995     !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET)
2996 COND_FAILED_NOT_COMPILED:
2997 expand_string_message = string_sprintf("support for \"%s\" not compiled",
2998   name);
2999 return NULL;
3000 #endif
3001 }
3002
3003
3004
3005
3006 /*************************************************
3007 *          Save numerical variables              *
3008 *************************************************/
3009
3010 /* This function is called from items such as "if" that want to preserve and
3011 restore the numbered variables.
3012
3013 Arguments:
3014   save_expand_string    points to an array of pointers to set
3015   save_expand_nlength   points to an array of ints for the lengths
3016
3017 Returns:                the value of expand max to save
3018 */
3019
3020 static int
3021 save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength)
3022 {
3023 int i;
3024 for (i = 0; i <= expand_nmax; i++)
3025   {
3026   save_expand_nstring[i] = expand_nstring[i];
3027   save_expand_nlength[i] = expand_nlength[i];
3028   }
3029 return expand_nmax;
3030 }
3031
3032
3033
3034 /*************************************************
3035 *           Restore numerical variables          *
3036 *************************************************/
3037
3038 /* This function restored saved values of numerical strings.
3039
3040 Arguments:
3041   save_expand_nmax      the number of strings to restore
3042   save_expand_string    points to an array of pointers
3043   save_expand_nlength   points to an array of ints
3044
3045 Returns:                nothing
3046 */
3047
3048 static void
3049 restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring,
3050   int *save_expand_nlength)
3051 {
3052 int i;
3053 expand_nmax = save_expand_nmax;
3054 for (i = 0; i <= expand_nmax; i++)
3055   {
3056   expand_nstring[i] = save_expand_nstring[i];
3057   expand_nlength[i] = save_expand_nlength[i];
3058   }
3059 }
3060
3061
3062
3063
3064
3065 /*************************************************
3066 *            Handle yes/no substrings            *
3067 *************************************************/
3068
3069 /* This function is used by ${if}, ${lookup} and ${extract} to handle the
3070 alternative substrings that depend on whether or not the condition was true,
3071 or the lookup or extraction succeeded. The substrings always have to be
3072 expanded, to check their syntax, but "skipping" is set when the result is not
3073 needed - this avoids unnecessary nested lookups.
3074
3075 Arguments:
3076   skipping       TRUE if we were skipping when this item was reached
3077   yes            TRUE if the first string is to be used, else use the second
3078   save_lookup    a value to put back into lookup_value before the 2nd expansion
3079   sptr           points to the input string pointer
3080   yieldptr       points to the output string pointer
3081   sizeptr        points to the output string size
3082   ptrptr         points to the output string pointer
3083   type           "lookup" or "if" or "extract" or "run", for error message
3084   resetok        if not NULL, pointer to flag - write FALSE if unsafe to reset
3085                 the store.
3086
3087 Returns:         0 OK; lookup_value has been reset to save_lookup
3088                  1 expansion failed
3089                  2 expansion failed because of bracketing error
3090 */
3091
3092 static int
3093 process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, uschar **sptr,
3094   uschar **yieldptr, int *sizeptr, int *ptrptr, uschar *type, BOOL *resetok)
3095 {
3096 int rc = 0;
3097 uschar *s = *sptr;    /* Local value */
3098 uschar *sub1, *sub2;
3099
3100 /* If there are no following strings, we substitute the contents of $value for
3101 lookups and for extractions in the success case. For the ${if item, the string
3102 "true" is substituted. In the fail case, nothing is substituted for all three
3103 items. */
3104
3105 while (isspace(*s)) s++;
3106 if (*s == '}')
3107   {
3108   if (type[0] == 'i')
3109     {
3110     if (yes) *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, US"true", 4);
3111     }
3112   else
3113     {
3114     if (yes && lookup_value != NULL)
3115       *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value,
3116         Ustrlen(lookup_value));
3117     lookup_value = save_lookup;
3118     }
3119   s++;
3120   goto RETURN;
3121   }
3122
3123 /* The first following string must be braced. */
3124
3125 if (*s++ != '{') goto FAILED_CURLY;
3126
3127 /* Expand the first substring. Forced failures are noticed only if we actually
3128 want this string. Set skipping in the call in the fail case (this will always
3129 be the case if we were already skipping). */
3130
3131 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
3132 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
3133 expand_string_forcedfail = FALSE;
3134 if (*s++ != '}') goto FAILED_CURLY;
3135
3136 /* If we want the first string, add it to the output */
3137
3138 if (yes)
3139   *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub1, Ustrlen(sub1));
3140
3141 /* If this is called from a lookup or an extract, we want to restore $value to
3142 what it was at the start of the item, so that it has this value during the
3143 second string expansion. For the call from "if" or "run" to this function,
3144 save_lookup is set to lookup_value, so that this statement does nothing. */
3145
3146 lookup_value = save_lookup;
3147
3148 /* There now follows either another substring, or "fail", or nothing. This
3149 time, forced failures are noticed only if we want the second string. We must
3150 set skipping in the nested call if we don't want this string, or if we were
3151 already skipping. */
3152
3153 while (isspace(*s)) s++;
3154 if (*s == '{')
3155   {
3156   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
3157   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
3158   expand_string_forcedfail = FALSE;
3159   if (*s++ != '}') goto FAILED_CURLY;
3160
3161   /* If we want the second string, add it to the output */
3162
3163   if (!yes)
3164     *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, sub2, Ustrlen(sub2));
3165   }
3166
3167 /* If there is no second string, but the word "fail" is present when the use of
3168 the second string is wanted, set a flag indicating it was a forced failure
3169 rather than a syntactic error. Swallow the terminating } in case this is nested
3170 inside another lookup or if or extract. */
3171
3172 else if (*s != '}')
3173   {
3174   uschar name[256];
3175   s = read_name(name, sizeof(name), s, US"_");
3176   if (Ustrcmp(name, "fail") == 0)
3177     {
3178     if (!yes && !skipping)
3179       {
3180       while (isspace(*s)) s++;
3181       if (*s++ != '}') goto FAILED_CURLY;
3182       expand_string_message =
3183         string_sprintf("\"%s\" failed and \"fail\" requested", type);
3184       expand_string_forcedfail = TRUE;
3185       goto FAILED;
3186       }
3187     }
3188   else
3189     {
3190     expand_string_message =
3191       string_sprintf("syntax error in \"%s\" item - \"fail\" expected", type);
3192     goto FAILED;
3193     }
3194   }
3195
3196 /* All we have to do now is to check on the final closing brace. */
3197
3198 while (isspace(*s)) s++;
3199 if (*s++ == '}') goto RETURN;
3200
3201 /* Get here if there is a bracketing failure */
3202
3203 FAILED_CURLY:
3204 rc++;
3205
3206 /* Get here for other failures */
3207
3208 FAILED:
3209 rc++;
3210
3211 /* Update the input pointer value before returning */
3212
3213 RETURN:
3214 *sptr = s;
3215 return rc;
3216 }
3217
3218
3219
3220
3221 /*************************************************
3222 *    Handle MD5 or SHA-1 computation for HMAC    *
3223 *************************************************/
3224
3225 /* These are some wrapping functions that enable the HMAC code to be a bit
3226 cleaner. A good compiler will spot the tail recursion.
3227
3228 Arguments:
3229   type         HMAC_MD5 or HMAC_SHA1
3230   remaining    are as for the cryptographic hash functions
3231
3232 Returns:       nothing
3233 */
3234
3235 static void
3236 chash_start(int type, void *base)
3237 {
3238 if (type == HMAC_MD5)
3239   md5_start((md5 *)base);
3240 else
3241   sha1_start((sha1 *)base);
3242 }
3243
3244 static void
3245 chash_mid(int type, void *base, uschar *string)
3246 {
3247 if (type == HMAC_MD5)
3248   md5_mid((md5 *)base, string);
3249 else
3250   sha1_mid((sha1 *)base, string);
3251 }
3252
3253 static void
3254 chash_end(int type, void *base, uschar *string, int length, uschar *digest)
3255 {
3256 if (type == HMAC_MD5)
3257   md5_end((md5 *)base, string, length, digest);
3258 else
3259   sha1_end((sha1 *)base, string, length, digest);
3260 }
3261
3262
3263
3264
3265
3266 /********************************************************
3267 * prvs: Get last three digits of days since Jan 1, 1970 *
3268 ********************************************************/
3269
3270 /* This is needed to implement the "prvs" BATV reverse
3271    path signing scheme
3272
3273 Argument: integer "days" offset to add or substract to
3274           or from the current number of days.
3275
3276 Returns:  pointer to string containing the last three
3277           digits of the number of days since Jan 1, 1970,
3278           modified by the offset argument, NULL if there
3279           was an error in the conversion.
3280
3281 */
3282
3283 static uschar *
3284 prvs_daystamp(int day_offset)
3285 {
3286 uschar *days = store_get(32);                /* Need at least 24 for cases */
3287 (void)string_format(days, 32, TIME_T_FMT,    /* where TIME_T_FMT is %lld */
3288   (time(NULL) + day_offset*86400)/86400);
3289 return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100";
3290 }
3291
3292
3293
3294 /********************************************************
3295 *   prvs: perform HMAC-SHA1 computation of prvs bits    *
3296 ********************************************************/
3297
3298 /* This is needed to implement the "prvs" BATV reverse
3299    path signing scheme
3300
3301 Arguments:
3302   address RFC2821 Address to use
3303       key The key to use (must be less than 64 characters
3304           in size)
3305   key_num Single-digit key number to use. Defaults to
3306           '0' when NULL.
3307
3308 Returns:  pointer to string containing the first three
3309           bytes of the final hash in hex format, NULL if
3310           there was an error in the process.
3311 */
3312
3313 static uschar *
3314 prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
3315 {
3316 uschar *hash_source, *p;
3317 int size = 0,offset = 0,i;
3318 sha1 sha1_base;
3319 void *use_base = &sha1_base;
3320 uschar innerhash[20];
3321 uschar finalhash[20];
3322 uschar innerkey[64];
3323 uschar outerkey[64];
3324 uschar *finalhash_hex = store_get(40);
3325
3326 if (key_num == NULL)
3327   key_num = US"0";
3328
3329 if (Ustrlen(key) > 64)
3330   return NULL;
3331
3332 hash_source = string_cat(NULL,&size,&offset,key_num,1);
3333 string_cat(hash_source,&size,&offset,daystamp,3);
3334 string_cat(hash_source,&size,&offset,address,Ustrlen(address));
3335 hash_source[offset] = '\0';
3336
3337 DEBUG(D_expand) debug_printf("prvs: hash source is '%s'\n", hash_source);
3338
3339 memset(innerkey, 0x36, 64);
3340 memset(outerkey, 0x5c, 64);
3341
3342 for (i = 0; i < Ustrlen(key); i++)
3343   {
3344   innerkey[i] ^= key[i];
3345   outerkey[i] ^= key[i];
3346   }
3347
3348 chash_start(HMAC_SHA1, use_base);
3349 chash_mid(HMAC_SHA1, use_base, innerkey);
3350 chash_end(HMAC_SHA1, use_base, hash_source, offset, innerhash);
3351
3352 chash_start(HMAC_SHA1, use_base);
3353 chash_mid(HMAC_SHA1, use_base, outerkey);
3354 chash_end(HMAC_SHA1, use_base, innerhash, 20, finalhash);
3355
3356 p = finalhash_hex;
3357 for (i = 0; i < 3; i++)
3358   {
3359   *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
3360   *p++ = hex_digits[finalhash[i] & 0x0f];
3361   }
3362 *p = '\0';
3363
3364 return finalhash_hex;
3365 }
3366
3367
3368
3369
3370 /*************************************************
3371 *        Join a file onto the output string      *
3372 *************************************************/
3373
3374 /* This is used for readfile and after a run expansion. It joins the contents
3375 of a file onto the output string, globally replacing newlines with a given
3376 string (optionally). The file is closed at the end.
3377
3378 Arguments:
3379   f            the FILE
3380   yield        pointer to the expandable string
3381   sizep        pointer to the current size
3382   ptrp         pointer to the current position
3383   eol          newline replacement string, or NULL
3384
3385 Returns:       new value of string pointer
3386 */
3387
3388 static uschar *
3389 cat_file(FILE *f, uschar *yield, int *sizep, int *ptrp, uschar *eol)
3390 {
3391 int eollen;
3392 uschar buffer[1024];
3393
3394 eollen = (eol == NULL)? 0 : Ustrlen(eol);
3395
3396 while (Ufgets(buffer, sizeof(buffer), f) != NULL)
3397   {
3398   int len = Ustrlen(buffer);
3399   if (eol != NULL && buffer[len-1] == '\n') len--;
3400   yield = string_cat(yield, sizep, ptrp, buffer, len);
3401   if (buffer[len] != 0)
3402     yield = string_cat(yield, sizep, ptrp, eol, eollen);
3403   }
3404
3405 if (yield != NULL) yield[*ptrp] = 0;
3406
3407 return yield;
3408 }
3409
3410
3411
3412
3413 /*************************************************
3414 *          Evaluate numeric expression           *
3415 *************************************************/
3416
3417 /* This is a set of mutually recursive functions that evaluate an arithmetic
3418 expression involving + - * / % & | ^ ~ << >> and parentheses. The only one of
3419 these functions that is called from elsewhere is eval_expr, whose interface is:
3420
3421 Arguments:
3422   sptr        pointer to the pointer to the string - gets updated
3423   decimal     TRUE if numbers are to be assumed decimal
3424   error       pointer to where to put an error message - must be NULL on input
3425   endket      TRUE if ')' must terminate - FALSE for external call
3426
3427 Returns:      on success: the value of the expression, with *error still NULL
3428               on failure: an undefined value, with *error = a message
3429 */
3430
3431 static int_eximarith_t eval_op_or(uschar **, BOOL, uschar **);
3432
3433
3434 static int_eximarith_t
3435 eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket)
3436 {
3437 uschar *s = *sptr;
3438 int_eximarith_t x = eval_op_or(&s, decimal, error);
3439 if (*error == NULL)
3440   {
3441   if (endket)
3442     {
3443     if (*s != ')')
3444       *error = US"expecting closing parenthesis";
3445     else
3446       while (isspace(*(++s)));
3447     }
3448   else if (*s != 0) *error = US"expecting operator";
3449   }
3450 *sptr = s;
3451 return x;
3452 }
3453
3454
3455 static int_eximarith_t
3456 eval_number(uschar **sptr, BOOL decimal, uschar **error)
3457 {
3458 register int c;
3459 int_eximarith_t n;
3460 uschar *s = *sptr;
3461 while (isspace(*s)) s++;
3462 c = *s;
3463 if (isdigit(c))
3464   {
3465   int count;
3466   (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count);
3467   s += count;
3468   switch (tolower(*s))
3469     {
3470     default: break;
3471     case 'k': n *= 1024; s++; break;
3472     case 'm': n *= 1024*1024; s++; break;
3473     case 'g': n *= 1024*1024*1024; s++; break;
3474     }
3475   while (isspace (*s)) s++;
3476   }
3477 else if (c == '(')
3478   {
3479   s++;
3480   n = eval_expr(&s, decimal, error, 1);
3481   }
3482 else
3483   {
3484   *error = US"expecting number or opening parenthesis";
3485   n = 0;
3486   }
3487 *sptr = s;
3488 return n;
3489 }
3490
3491
3492 static int_eximarith_t
3493 eval_op_unary(uschar **sptr, BOOL decimal, uschar **error)
3494 {
3495 uschar *s = *sptr;
3496 int_eximarith_t x;
3497 while (isspace(*s)) s++;
3498 if (*s == '+' || *s == '-' || *s == '~')
3499   {
3500   int op = *s++;
3501   x = eval_op_unary(&s, decimal, error);
3502   if (op == '-') x = -x;
3503     else if (op == '~') x = ~x;
3504   }
3505 else
3506   {
3507   x = eval_number(&s, decimal, error);
3508   }
3509 *sptr = s;
3510 return x;
3511 }
3512
3513
3514 static int_eximarith_t
3515 eval_op_mult(uschar **sptr, BOOL decimal, uschar **error)
3516 {
3517 uschar *s = *sptr;
3518 int_eximarith_t x = eval_op_unary(&s, decimal, error);
3519 if (*error == NULL)
3520   {
3521   while (*s == '*' || *s == '/' || *s == '%')
3522     {
3523     int op = *s++;
3524     int_eximarith_t y = eval_op_unary(&s, decimal, error);
3525     if (*error != NULL) break;
3526     /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give
3527      * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which
3528      * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64].  In fact, -N*-M where
3529      * -N*M is INT_MIN will yielf INT_MIN.
3530      * Since we don't support floating point, this is somewhat simpler.
3531      * Ideally, we'd return an error, but since we overflow for all other
3532      * arithmetic, consistency suggests otherwise, but what's the correct value
3533      * to use?  There is none.
3534      * The C standard guarantees overflow for unsigned arithmetic but signed
3535      * overflow invokes undefined behaviour; in practice, this is overflow
3536      * except for converting INT_MIN to INT_MAX+1.  We also can't guarantee
3537      * that long/longlong larger than int are available, or we could just work
3538      * with larger types.  We should consider whether to guarantee 32bit eval
3539      * and 64-bit working variables, with errors returned.  For now ...
3540      * So, the only SIGFPEs occur with a non-shrinking div/mod, thus -1; we
3541      * can just let the other invalid results occur otherwise, as they have
3542      * until now.  For this one case, we can coerce.
3543      */
3544     if (y == -1 && x == EXIM_ARITH_MIN && op != '*')
3545       {
3546       DEBUG(D_expand)
3547         debug_printf("Integer exception dodging: " PR_EXIM_ARITH "%c-1 coerced to " PR_EXIM_ARITH "\n",
3548             EXIM_ARITH_MIN, op, EXIM_ARITH_MAX);
3549       x = EXIM_ARITH_MAX;
3550       continue;
3551       }
3552     if (op == '*')
3553       x *= y;
3554     else
3555       {
3556       if (y == 0)
3557         {
3558         *error = (op == '/') ? US"divide by zero" : US"modulo by zero";
3559         x = 0;
3560         break;
3561         }
3562       if (op == '/')
3563         x /= y;
3564       else
3565         x %= y;
3566       }
3567     }
3568   }
3569 *sptr = s;
3570 return x;
3571 }
3572
3573
3574 static int_eximarith_t
3575 eval_op_sum(uschar **sptr, BOOL decimal, uschar **error)
3576 {
3577 uschar *s = *sptr;
3578 int_eximarith_t x = eval_op_mult(&s, decimal, error);
3579 if (*error == NULL)
3580   {
3581   while (*s == '+' || *s == '-')
3582     {
3583     int op = *s++;
3584     int_eximarith_t y = eval_op_mult(&s, decimal, error);
3585     if (*error != NULL) break;
3586     if (op == '+') x += y; else x -= y;
3587     }
3588   }
3589 *sptr = s;
3590 return x;
3591 }
3592
3593
3594 static int_eximarith_t
3595 eval_op_shift(uschar **sptr, BOOL decimal, uschar **error)
3596 {
3597 uschar *s = *sptr;
3598 int_eximarith_t x = eval_op_sum(&s, decimal, error);
3599 if (*error == NULL)
3600   {
3601   while ((*s == '<' || *s == '>') && s[1] == s[0])
3602     {
3603     int_eximarith_t y;
3604     int op = *s++;
3605     s++;
3606     y = eval_op_sum(&s, decimal, error);
3607     if (*error != NULL) break;
3608     if (op == '<') x <<= y; else x >>= y;
3609     }
3610   }
3611 *sptr = s;
3612 return x;
3613 }
3614
3615
3616 static int_eximarith_t
3617 eval_op_and(uschar **sptr, BOOL decimal, uschar **error)
3618 {
3619 uschar *s = *sptr;
3620 int_eximarith_t x = eval_op_shift(&s, decimal, error);
3621 if (*error == NULL)
3622   {
3623   while (*s == '&')
3624     {
3625     int_eximarith_t y;
3626     s++;
3627     y = eval_op_shift(&s, decimal, error);
3628     if (*error != NULL) break;
3629     x &= y;
3630     }
3631   }
3632 *sptr = s;
3633 return x;
3634 }
3635
3636
3637 static int_eximarith_t
3638 eval_op_xor(uschar **sptr, BOOL decimal, uschar **error)
3639 {
3640 uschar *s = *sptr;
3641 int_eximarith_t x = eval_op_and(&s, decimal, error);
3642 if (*error == NULL)
3643   {
3644   while (*s == '^')
3645     {
3646     int_eximarith_t y;
3647     s++;
3648     y = eval_op_and(&s, decimal, error);
3649     if (*error != NULL) break;
3650     x ^= y;
3651     }
3652   }
3653 *sptr = s;
3654 return x;
3655 }
3656
3657
3658 static int_eximarith_t
3659 eval_op_or(uschar **sptr, BOOL decimal, uschar **error)
3660 {
3661 uschar *s = *sptr;
3662 int_eximarith_t x = eval_op_xor(&s, decimal, error);
3663 if (*error == NULL)
3664   {
3665   while (*s == '|')
3666     {
3667     int_eximarith_t y;
3668     s++;
3669     y = eval_op_xor(&s, decimal, error);
3670     if (*error != NULL) break;
3671     x |= y;
3672     }
3673   }
3674 *sptr = s;
3675 return x;
3676 }
3677
3678
3679
3680 /*************************************************
3681 *                 Expand string                  *
3682 *************************************************/
3683
3684 /* Returns either an unchanged string, or the expanded string in stacking pool
3685 store. Interpreted sequences are:
3686
3687    \...                    normal escaping rules
3688    $name                   substitutes the variable
3689    ${name}                 ditto
3690    ${op:string}            operates on the expanded string value
3691    ${item{arg1}{arg2}...}  expands the args and then does the business
3692                              some literal args are not enclosed in {}
3693
3694 There are now far too many operators and item types to make it worth listing
3695 them here in detail any more.
3696
3697 We use an internal routine recursively to handle embedded substrings. The
3698 external function follows. The yield is NULL if the expansion failed, and there
3699 are two cases: if something collapsed syntactically, or if "fail" was given
3700 as the action on a lookup failure. These can be distinguised by looking at the
3701 variable expand_string_forcedfail, which is TRUE in the latter case.
3702
3703 The skipping flag is set true when expanding a substring that isn't actually
3704 going to be used (after "if" or "lookup") and it prevents lookups from
3705 happening lower down.
3706
3707 Store usage: At start, a store block of the length of the input plus 64
3708 is obtained. This is expanded as necessary by string_cat(), which might have to
3709 get a new block, or might be able to expand the original. At the end of the
3710 function we can release any store above that portion of the yield block that
3711 was actually used. In many cases this will be optimal.
3712
3713 However: if the first item in the expansion is a variable name or header name,
3714 we reset the store before processing it; if the result is in fresh store, we
3715 use that without copying. This is helpful for expanding strings like
3716 $message_headers which can get very long.
3717
3718 There's a problem if a ${dlfunc item has side-effects that cause allocation,
3719 since resetting the store at the end of the expansion will free store that was
3720 allocated by the plugin code as well as the slop after the expanded string. So
3721 we skip any resets if ${dlfunc } has been used. The same applies for ${acl }
3722 and, given the acl condition, ${if }. This is an unfortunate consequence of
3723 string expansion becoming too powerful.
3724
3725 Arguments:
3726   string         the string to be expanded
3727   ket_ends       true if expansion is to stop at }
3728   left           if not NULL, a pointer to the first character after the
3729                  expansion is placed here (typically used with ket_ends)
3730   skipping       TRUE for recursive calls when the value isn't actually going
3731                  to be used (to allow for optimisation)
3732   honour_dollar  TRUE if $ is to be expanded,
3733                  FALSE if it's just another character
3734   resetok_p      if not NULL, pointer to flag - write FALSE if unsafe to reset
3735                  the store.
3736
3737 Returns:         NULL if expansion fails:
3738                    expand_string_forcedfail is set TRUE if failure was forced
3739                    expand_string_message contains a textual error message
3740                  a pointer to the expanded string on success
3741 */
3742
3743 static uschar *
3744 expand_string_internal(uschar *string, BOOL ket_ends, uschar **left,
3745   BOOL skipping, BOOL honour_dollar, BOOL *resetok_p)
3746 {
3747 int ptr = 0;
3748 int size = Ustrlen(string)+ 64;
3749 int item_type;
3750 uschar *yield = store_get(size);
3751 uschar *s = string;
3752 uschar *save_expand_nstring[EXPAND_MAXN+1];
3753 int save_expand_nlength[EXPAND_MAXN+1];
3754 BOOL resetok = TRUE;
3755
3756 expand_string_forcedfail = FALSE;
3757 expand_string_message = US"";
3758
3759 while (*s != 0)
3760   {
3761   uschar *value;
3762   uschar name[256];
3763
3764   /* \ escapes the next character, which must exist, or else
3765   the expansion fails. There's a special escape, \N, which causes
3766   copying of the subject verbatim up to the next \N. Otherwise,
3767   the escapes are the standard set. */
3768
3769   if (*s == '\\')
3770     {
3771     if (s[1] == 0)
3772       {
3773       expand_string_message = US"\\ at end of string";
3774       goto EXPAND_FAILED;
3775       }
3776
3777     if (s[1] == 'N')
3778       {
3779       uschar *t = s + 2;
3780       for (s = t; *s != 0; s++) if (*s == '\\' && s[1] == 'N') break;
3781       yield = string_cat(yield, &size, &ptr, t, s - t);
3782       if (*s != 0) s += 2;
3783       }
3784
3785     else
3786       {
3787       uschar ch[1];
3788       ch[0] = string_interpret_escape(&s);
3789       s++;
3790       yield = string_cat(yield, &size, &ptr, ch, 1);
3791       }
3792
3793     continue;
3794     }
3795
3796   /*{*/
3797   /* Anything other than $ is just copied verbatim, unless we are
3798   looking for a terminating } character. */
3799
3800   /*{*/
3801   if (ket_ends && *s == '}') break;
3802
3803   if (*s != '$' || !honour_dollar)
3804     {
3805     yield = string_cat(yield, &size, &ptr, s++, 1);
3806     continue;
3807     }
3808
3809   /* No { after the $ - must be a plain name or a number for string
3810   match variable. There has to be a fudge for variables that are the
3811   names of header fields preceded by "$header_" because header field
3812   names can contain any printing characters except space and colon.
3813   For those that don't like typing this much, "$h_" is a synonym for
3814   "$header_". A non-existent header yields a NULL value; nothing is
3815   inserted. */  /*}*/
3816
3817   if (isalpha((*(++s))))
3818     {
3819     int len;
3820     int newsize = 0;
3821
3822     s = read_name(name, sizeof(name), s, US"_");
3823
3824     /* If this is the first thing to be expanded, release the pre-allocated
3825     buffer. */
3826
3827     if (ptr == 0 && yield != NULL)
3828       {
3829       if (resetok) store_reset(yield);
3830       yield = NULL;
3831       size = 0;
3832       }
3833
3834     /* Header */
3835
3836     if (Ustrncmp(name, "h_", 2) == 0 ||
3837         Ustrncmp(name, "rh_", 3) == 0 ||
3838         Ustrncmp(name, "bh_", 3) == 0 ||
3839         Ustrncmp(name, "header_", 7) == 0 ||
3840         Ustrncmp(name, "rheader_", 8) == 0 ||
3841         Ustrncmp(name, "bheader_", 8) == 0)
3842       {
3843       BOOL want_raw = (name[0] == 'r')? TRUE : FALSE;
3844       uschar *charset = (name[0] == 'b')? NULL : headers_charset;
3845       s = read_header_name(name, sizeof(name), s);
3846       value = find_header(name, FALSE, &newsize, want_raw, charset);
3847
3848       /* If we didn't find the header, and the header contains a closing brace
3849       character, this may be a user error where the terminating colon
3850       has been omitted. Set a flag to adjust the error message in this case.
3851       But there is no error here - nothing gets inserted. */
3852
3853       if (value == NULL)
3854         {
3855         if (Ustrchr(name, '}') != NULL) malformed_header = TRUE;
3856         continue;
3857         }
3858       }
3859
3860     /* Variable */
3861
3862     else
3863       {
3864       value = find_variable(name, FALSE, skipping, &newsize);
3865       if (value == NULL)
3866         {
3867         expand_string_message =
3868           string_sprintf("unknown variable name \"%s\"", name);
3869           check_variable_error_message(name);
3870         goto EXPAND_FAILED;
3871         }
3872       }
3873
3874     /* If the data is known to be in a new buffer, newsize will be set to the
3875     size of that buffer. If this is the first thing in an expansion string,
3876     yield will be NULL; just point it at the new store instead of copying. Many
3877     expansion strings contain just one reference, so this is a useful
3878     optimization, especially for humungous headers. */
3879
3880     len = Ustrlen(value);
3881     if (yield == NULL && newsize != 0)
3882       {
3883       yield = value;
3884       size = newsize;
3885       ptr = len;
3886       }
3887     else yield = string_cat(yield, &size, &ptr, value, len);
3888
3889     continue;
3890     }
3891
3892   if (isdigit(*s))
3893     {
3894     int n;
3895     s = read_number(&n, s);
3896     if (n >= 0 && n <= expand_nmax)
3897       yield = string_cat(yield, &size, &ptr, expand_nstring[n],
3898         expand_nlength[n]);
3899     continue;
3900     }
3901
3902   /* Otherwise, if there's no '{' after $ it's an error. */             /*}*/
3903
3904   if (*s != '{')                                                        /*}*/
3905     {
3906     expand_string_message = US"$ not followed by letter, digit, or {";  /*}*/
3907     goto EXPAND_FAILED;
3908     }
3909
3910   /* After { there can be various things, but they all start with
3911   an initial word, except for a number for a string match variable. */
3912
3913   if (isdigit((*(++s))))
3914     {
3915     int n;
3916     s = read_number(&n, s);             /*{*/
3917     if (*s++ != '}')
3918       {                                 /*{*/
3919       expand_string_message = US"} expected after number";
3920       goto EXPAND_FAILED;
3921       }
3922     if (n >= 0 && n <= expand_nmax)
3923       yield = string_cat(yield, &size, &ptr, expand_nstring[n],
3924         expand_nlength[n]);
3925     continue;
3926     }
3927
3928   if (!isalpha(*s))
3929     {
3930     expand_string_message = US"letter or digit expected after ${";      /*}*/
3931     goto EXPAND_FAILED;
3932     }
3933
3934   /* Allow "-" in names to cater for substrings with negative
3935   arguments. Since we are checking for known names after { this is
3936   OK. */
3937
3938   s = read_name(name, sizeof(name), s, US"_-");
3939   item_type = chop_match(name, item_table, sizeof(item_table)/sizeof(uschar *));
3940
3941   switch(item_type)
3942     {
3943     /* Call an ACL from an expansion.  We feed data in via $acl_arg1 - $acl_arg9.
3944     If the ACL returns accept or reject we return content set by "message ="
3945     There is currently no limit on recursion; this would have us call
3946     acl_check_internal() directly and get a current level from somewhere.
3947     See also the acl expansion condition ECOND_ACL and the traditional
3948     acl modifier ACLC_ACL.
3949     Assume that the function has side-effects on the store that must be preserved.
3950     */
3951
3952     case EITEM_ACL:
3953       /* ${acl {name} {arg1}{arg2}...} */
3954       {
3955       uschar *sub[10];  /* name + arg1-arg9 (which must match number of acl_arg[]) */
3956       uschar *user_msg;
3957
3958       switch(read_subs(sub, 10, 1, &s, skipping, TRUE, US"acl", &resetok))
3959         {
3960         case 1: goto EXPAND_FAILED_CURLY;
3961         case 2:
3962         case 3: goto EXPAND_FAILED;
3963         }
3964       if (skipping) continue;
3965
3966       resetok = FALSE;
3967       switch(eval_acl(sub, sizeof(sub)/sizeof(*sub), &user_msg))
3968         {
3969         case OK:
3970         case FAIL:
3971           DEBUG(D_expand)
3972             debug_printf("acl expansion yield: %s\n", user_msg);
3973           if (user_msg)
3974             yield = string_cat(yield, &size, &ptr, user_msg, Ustrlen(user_msg));
3975           continue;
3976
3977         case DEFER:
3978           expand_string_forcedfail = TRUE;
3979         default:
3980           expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]);
3981           goto EXPAND_FAILED;
3982         }
3983       }
3984
3985     /* Handle conditionals - preserve the values of the numerical expansion
3986     variables in case they get changed by a regular expression match in the
3987     condition. If not, they retain their external settings. At the end
3988     of this "if" section, they get restored to their previous values. */
3989
3990     case EITEM_IF:
3991       {
3992       BOOL cond = FALSE;
3993       uschar *next_s;
3994       int save_expand_nmax =
3995         save_expand_strings(save_expand_nstring, save_expand_nlength);
3996
3997       while (isspace(*s)) s++;
3998       next_s = eval_condition(s, &resetok, skipping? NULL : &cond);
3999       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
4000
4001       DEBUG(D_expand)
4002         debug_printf("condition: %.*s\n   result: %s\n", (int)(next_s - s), s,
4003           cond? "true" : "false");
4004
4005       s = next_s;
4006
4007       /* The handling of "yes" and "no" result strings is now in a separate
4008       function that is also used by ${lookup} and ${extract} and ${run}. */
4009
4010       switch(process_yesno(
4011                skipping,                     /* were previously skipping */
4012                cond,                         /* success/failure indicator */
4013                lookup_value,                 /* value to reset for string2 */
4014                &s,                           /* input pointer */
4015                &yield,                       /* output pointer */
4016                &size,                        /* output size */
4017                &ptr,                         /* output current point */
4018                US"if",                       /* condition type */
4019                &resetok))
4020         {
4021         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4022         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4023         }
4024
4025       /* Restore external setting of expansion variables for continuation
4026       at this level. */
4027
4028       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4029         save_expand_nlength);
4030       continue;
4031       }
4032
4033     /* Handle database lookups unless locked out. If "skipping" is TRUE, we are
4034     expanding an internal string that isn't actually going to be used. All we
4035     need to do is check the syntax, so don't do a lookup at all. Preserve the
4036     values of the numerical expansion variables in case they get changed by a
4037     partial lookup. If not, they retain their external settings. At the end
4038     of this "lookup" section, they get restored to their previous values. */
4039
4040     case EITEM_LOOKUP:
4041       {
4042       int stype, partial, affixlen, starflags;
4043       int expand_setup = 0;
4044       int nameptr = 0;
4045       uschar *key, *filename, *affix;
4046       uschar *save_lookup_value = lookup_value;
4047       int save_expand_nmax =
4048         save_expand_strings(save_expand_nstring, save_expand_nlength);
4049
4050       if ((expand_forbid & RDO_LOOKUP) != 0)
4051         {
4052         expand_string_message = US"lookup expansions are not permitted";
4053         goto EXPAND_FAILED;
4054         }
4055
4056       /* Get the key we are to look up for single-key+file style lookups.
4057       Otherwise set the key NULL pro-tem. */
4058
4059       while (isspace(*s)) s++;
4060       if (*s == '{')                                    /*}*/
4061         {
4062         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4063         if (key == NULL) goto EXPAND_FAILED;            /*{*/
4064         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4065         while (isspace(*s)) s++;
4066         }
4067       else key = NULL;
4068
4069       /* Find out the type of database */
4070
4071       if (!isalpha(*s))
4072         {
4073         expand_string_message = US"missing lookup type";
4074         goto EXPAND_FAILED;
4075         }
4076
4077       /* The type is a string that may contain special characters of various
4078       kinds. Allow everything except space or { to appear; the actual content
4079       is checked by search_findtype_partial. */         /*}*/
4080
4081       while (*s != 0 && *s != '{' && !isspace(*s))      /*}*/
4082         {
4083         if (nameptr < sizeof(name) - 1) name[nameptr++] = *s;
4084         s++;
4085         }
4086       name[nameptr] = 0;
4087       while (isspace(*s)) s++;
4088
4089       /* Now check for the individual search type and any partial or default
4090       options. Only those types that are actually in the binary are valid. */
4091
4092       stype = search_findtype_partial(name, &partial, &affix, &affixlen,
4093         &starflags);
4094       if (stype < 0)
4095         {
4096         expand_string_message = search_error_message;
4097         goto EXPAND_FAILED;
4098         }
4099
4100       /* Check that a key was provided for those lookup types that need it,
4101       and was not supplied for those that use the query style. */
4102
4103       if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery))
4104         {
4105         if (key == NULL)
4106           {
4107           expand_string_message = string_sprintf("missing {key} for single-"
4108             "key \"%s\" lookup", name);
4109           goto EXPAND_FAILED;
4110           }
4111         }
4112       else
4113         {
4114         if (key != NULL)
4115           {
4116           expand_string_message = string_sprintf("a single key was given for "
4117             "lookup type \"%s\", which is not a single-key lookup type", name);
4118           goto EXPAND_FAILED;
4119           }
4120         }
4121
4122       /* Get the next string in brackets and expand it. It is the file name for
4123       single-key+file lookups, and the whole query otherwise. In the case of
4124       queries that also require a file name (e.g. sqlite), the file name comes
4125       first. */
4126
4127       if (*s != '{') goto EXPAND_FAILED_CURLY;
4128       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4129       if (filename == NULL) goto EXPAND_FAILED;
4130       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4131       while (isspace(*s)) s++;
4132
4133       /* If this isn't a single-key+file lookup, re-arrange the variables
4134       to be appropriate for the search_ functions. For query-style lookups,
4135       there is just a "key", and no file name. For the special query-style +
4136       file types, the query (i.e. "key") starts with a file name. */
4137
4138       if (key == NULL)
4139         {
4140         while (isspace(*filename)) filename++;
4141         key = filename;
4142
4143         if (mac_islookup(stype, lookup_querystyle))
4144           {
4145           filename = NULL;
4146           }
4147         else
4148           {
4149           if (*filename != '/')
4150             {
4151             expand_string_message = string_sprintf(
4152               "absolute file name expected for \"%s\" lookup", name);
4153             goto EXPAND_FAILED;
4154             }
4155           while (*key != 0 && !isspace(*key)) key++;
4156           if (*key != 0) *key++ = 0;
4157           }
4158         }
4159
4160       /* If skipping, don't do the next bit - just lookup_value == NULL, as if
4161       the entry was not found. Note that there is no search_close() function.
4162       Files are left open in case of re-use. At suitable places in higher logic,
4163       search_tidyup() is called to tidy all open files. This can save opening
4164       the same file several times. However, files may also get closed when
4165       others are opened, if too many are open at once. The rule is that a
4166       handle should not be used after a second search_open().
4167
4168       Request that a partial search sets up $1 and maybe $2 by passing
4169       expand_setup containing zero. If its value changes, reset expand_nmax,
4170       since new variables will have been set. Note that at the end of this
4171       "lookup" section, the old numeric variables are restored. */
4172
4173       if (skipping)
4174         lookup_value = NULL;
4175       else
4176         {
4177         void *handle = search_open(filename, stype, 0, NULL, NULL);
4178         if (handle == NULL)
4179           {
4180           expand_string_message = search_error_message;
4181           goto EXPAND_FAILED;
4182           }
4183         lookup_value = search_find(handle, filename, key, partial, affix,
4184           affixlen, starflags, &expand_setup);
4185         if (search_find_defer)
4186           {
4187           expand_string_message =
4188             string_sprintf("lookup of \"%s\" gave DEFER: %s",
4189               string_printing2(key, FALSE), search_error_message);
4190           goto EXPAND_FAILED;
4191           }
4192         if (expand_setup > 0) expand_nmax = expand_setup;
4193         }
4194
4195       /* The handling of "yes" and "no" result strings is now in a separate
4196       function that is also used by ${if} and ${extract}. */
4197
4198       switch(process_yesno(
4199                skipping,                     /* were previously skipping */
4200                lookup_value != NULL,         /* success/failure indicator */
4201                save_lookup_value,            /* value to reset for string2 */
4202                &s,                           /* input pointer */
4203                &yield,                       /* output pointer */
4204                &size,                        /* output size */
4205                &ptr,                         /* output current point */
4206                US"lookup",                   /* condition type */
4207                &resetok))
4208         {
4209         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4210         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4211         }
4212
4213       /* Restore external setting of expansion variables for carrying on
4214       at this level, and continue. */
4215
4216       restore_expand_strings(save_expand_nmax, save_expand_nstring,
4217         save_expand_nlength);
4218       continue;
4219       }
4220
4221     /* If Perl support is configured, handle calling embedded perl subroutines,
4222     unless locked out at this time. Syntax is ${perl{sub}} or ${perl{sub}{arg}}
4223     or ${perl{sub}{arg1}{arg2}} or up to a maximum of EXIM_PERL_MAX_ARGS
4224     arguments (defined below). */
4225
4226     #define EXIM_PERL_MAX_ARGS 8
4227
4228     case EITEM_PERL:
4229     #ifndef EXIM_PERL
4230     expand_string_message = US"\"${perl\" encountered, but this facility "      /*}*/
4231       "is not included in this binary";
4232     goto EXPAND_FAILED;
4233
4234     #else   /* EXIM_PERL */
4235       {
4236       uschar *sub_arg[EXIM_PERL_MAX_ARGS + 2];
4237       uschar *new_yield;
4238
4239       if ((expand_forbid & RDO_PERL) != 0)
4240         {
4241         expand_string_message = US"Perl calls are not permitted";
4242         goto EXPAND_FAILED;
4243         }
4244
4245       switch(read_subs(sub_arg, EXIM_PERL_MAX_ARGS + 1, 1, &s, skipping, TRUE,
4246            US"perl", &resetok))
4247         {
4248         case 1: goto EXPAND_FAILED_CURLY;
4249         case 2:
4250         case 3: goto EXPAND_FAILED;
4251         }
4252
4253       /* If skipping, we don't actually do anything */
4254
4255       if (skipping) continue;
4256
4257       /* Start the interpreter if necessary */
4258
4259       if (!opt_perl_started)
4260         {
4261         uschar *initerror;
4262         if (opt_perl_startup == NULL)
4263           {
4264           expand_string_message = US"A setting of perl_startup is needed when "
4265             "using the Perl interpreter";
4266           goto EXPAND_FAILED;
4267           }
4268         DEBUG(D_any) debug_printf("Starting Perl interpreter\n");
4269         initerror = init_perl(opt_perl_startup);
4270         if (initerror != NULL)
4271           {
4272           expand_string_message =
4273             string_sprintf("error in perl_startup code: %s\n", initerror);
4274           goto EXPAND_FAILED;
4275           }
4276         opt_perl_started = TRUE;
4277         }
4278
4279       /* Call the function */
4280
4281       sub_arg[EXIM_PERL_MAX_ARGS + 1] = NULL;
4282       new_yield = call_perl_cat(yield, &size, &ptr, &expand_string_message,
4283         sub_arg[0], sub_arg + 1);
4284
4285       /* NULL yield indicates failure; if the message pointer has been set to
4286       NULL, the yield was undef, indicating a forced failure. Otherwise the
4287       message will indicate some kind of Perl error. */
4288
4289       if (new_yield == NULL)
4290         {
4291         if (expand_string_message == NULL)
4292           {
4293           expand_string_message =
4294             string_sprintf("Perl subroutine \"%s\" returned undef to force "
4295               "failure", sub_arg[0]);
4296           expand_string_forcedfail = TRUE;
4297           }
4298         goto EXPAND_FAILED;
4299         }
4300
4301       /* Yield succeeded. Ensure forcedfail is unset, just in case it got
4302       set during a callback from Perl. */
4303
4304       expand_string_forcedfail = FALSE;
4305       yield = new_yield;
4306       continue;
4307       }
4308     #endif /* EXIM_PERL */
4309
4310     /* Transform email address to "prvs" scheme to use
4311        as BATV-signed return path */
4312
4313     case EITEM_PRVS:
4314       {
4315       uschar *sub_arg[3];
4316       uschar *p,*domain;
4317
4318       switch(read_subs(sub_arg, 3, 2, &s, skipping, TRUE, US"prvs", &resetok))
4319         {
4320         case 1: goto EXPAND_FAILED_CURLY;
4321         case 2:
4322         case 3: goto EXPAND_FAILED;
4323         }
4324
4325       /* If skipping, we don't actually do anything */
4326       if (skipping) continue;
4327
4328       /* sub_arg[0] is the address */
4329       domain = Ustrrchr(sub_arg[0],'@');
4330       if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) )
4331         {
4332         expand_string_message = US"prvs first argument must be a qualified email address";
4333         goto EXPAND_FAILED;
4334         }
4335
4336       /* Calculate the hash. The second argument must be a single-digit
4337       key number, or unset. */
4338
4339       if (sub_arg[2] != NULL &&
4340           (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0))
4341         {
4342         expand_string_message = US"prvs second argument must be a single digit";
4343         goto EXPAND_FAILED;
4344         }
4345
4346       p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7));
4347       if (p == NULL)
4348         {
4349         expand_string_message = US"prvs hmac-sha1 conversion failed";
4350         goto EXPAND_FAILED;
4351         }
4352
4353       /* Now separate the domain from the local part */
4354       *domain++ = '\0';
4355
4356       yield = string_cat(yield,&size,&ptr,US"prvs=",5);
4357       string_cat(yield,&size,&ptr,(sub_arg[2] != NULL) ? sub_arg[2] : US"0", 1);
4358       string_cat(yield,&size,&ptr,prvs_daystamp(7),3);
4359       string_cat(yield,&size,&ptr,p,6);
4360       string_cat(yield,&size,&ptr,US"=",1);
4361       string_cat(yield,&size,&ptr,sub_arg[0],Ustrlen(sub_arg[0]));
4362       string_cat(yield,&size,&ptr,US"@",1);
4363       string_cat(yield,&size,&ptr,domain,Ustrlen(domain));
4364
4365       continue;
4366       }
4367
4368     /* Check a prvs-encoded address for validity */
4369
4370     case EITEM_PRVSCHECK:
4371       {
4372       uschar *sub_arg[3];
4373       int mysize = 0, myptr = 0;
4374       const pcre *re;
4375       uschar *p;
4376
4377       /* TF: Ugliness: We want to expand parameter 1 first, then set
4378          up expansion variables that are used in the expansion of
4379          parameter 2. So we clone the string for the first
4380          expansion, where we only expand parameter 1.
4381
4382          PH: Actually, that isn't necessary. The read_subs() function is
4383          designed to work this way for the ${if and ${lookup expansions. I've
4384          tidied the code.
4385       */
4386
4387       /* Reset expansion variables */
4388       prvscheck_result = NULL;
4389       prvscheck_address = NULL;
4390       prvscheck_keynum = NULL;
4391
4392       switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4393         {
4394         case 1: goto EXPAND_FAILED_CURLY;
4395         case 2:
4396         case 3: goto EXPAND_FAILED;
4397         }
4398
4399       re = regex_must_compile(US"^prvs\\=([0-9])([0-9]{3})([A-F0-9]{6})\\=(.+)\\@(.+)$",
4400                               TRUE,FALSE);
4401
4402       if (regex_match_and_setup(re,sub_arg[0],0,-1))
4403         {
4404         uschar *local_part = string_copyn(expand_nstring[4],expand_nlength[4]);
4405         uschar *key_num = string_copyn(expand_nstring[1],expand_nlength[1]);
4406         uschar *daystamp = string_copyn(expand_nstring[2],expand_nlength[2]);
4407         uschar *hash = string_copyn(expand_nstring[3],expand_nlength[3]);
4408         uschar *domain = string_copyn(expand_nstring[5],expand_nlength[5]);
4409
4410         DEBUG(D_expand) debug_printf("prvscheck localpart: %s\n", local_part);
4411         DEBUG(D_expand) debug_printf("prvscheck key number: %s\n", key_num);
4412         DEBUG(D_expand) debug_printf("prvscheck daystamp: %s\n", daystamp);
4413         DEBUG(D_expand) debug_printf("prvscheck hash: %s\n", hash);
4414         DEBUG(D_expand) debug_printf("prvscheck domain: %s\n", domain);
4415
4416         /* Set up expansion variables */
4417         prvscheck_address = string_cat(NULL, &mysize, &myptr, local_part, Ustrlen(local_part));
4418         string_cat(prvscheck_address,&mysize,&myptr,US"@",1);
4419         string_cat(prvscheck_address,&mysize,&myptr,domain,Ustrlen(domain));
4420         prvscheck_address[myptr] = '\0';
4421         prvscheck_keynum = string_copy(key_num);
4422
4423         /* Now expand the second argument */
4424         switch(read_subs(sub_arg, 1, 1, &s, skipping, FALSE, US"prvs", &resetok))
4425           {
4426           case 1: goto EXPAND_FAILED_CURLY;
4427           case 2:
4428           case 3: goto EXPAND_FAILED;
4429           }
4430
4431         /* Now we have the key and can check the address. */
4432
4433         p = prvs_hmac_sha1(prvscheck_address, sub_arg[0], prvscheck_keynum,
4434           daystamp);
4435
4436         if (p == NULL)
4437           {
4438           expand_string_message = US"hmac-sha1 conversion failed";
4439           goto EXPAND_FAILED;
4440           }
4441
4442         DEBUG(D_expand) debug_printf("prvscheck: received hash is %s\n", hash);
4443         DEBUG(D_expand) debug_printf("prvscheck:      own hash is %s\n", p);
4444
4445         if (Ustrcmp(p,hash) == 0)
4446           {
4447           /* Success, valid BATV address. Now check the expiry date. */
4448           uschar *now = prvs_daystamp(0);
4449           unsigned int inow = 0,iexpire = 1;
4450
4451           (void)sscanf(CS now,"%u",&inow);
4452           (void)sscanf(CS daystamp,"%u",&iexpire);
4453
4454           /* When "iexpire" is < 7, a "flip" has occured.
4455              Adjust "inow" accordingly. */
4456           if ( (iexpire < 7) && (inow >= 993) ) inow = 0;
4457
4458           if (iexpire >= inow)
4459             {
4460             prvscheck_result = US"1";
4461             DEBUG(D_expand) debug_printf("prvscheck: success, $pvrs_result set to 1\n");
4462             }
4463             else
4464             {
4465             prvscheck_result = NULL;
4466             DEBUG(D_expand) debug_printf("prvscheck: signature expired, $pvrs_result unset\n");
4467             }
4468           }
4469         else
4470           {
4471           prvscheck_result = NULL;
4472           DEBUG(D_expand) debug_printf("prvscheck: hash failure, $pvrs_result unset\n");
4473           }
4474
4475         /* Now expand the final argument. We leave this till now so that
4476         it can include $prvscheck_result. */
4477
4478         switch(read_subs(sub_arg, 1, 0, &s, skipping, TRUE, US"prvs", &resetok))
4479           {
4480           case 1: goto EXPAND_FAILED_CURLY;
4481           case 2:
4482           case 3: goto EXPAND_FAILED;
4483           }
4484
4485         if (sub_arg[0] == NULL || *sub_arg[0] == '\0')
4486           yield = string_cat(yield,&size,&ptr,prvscheck_address,Ustrlen(prvscheck_address));
4487         else
4488           yield = string_cat(yield,&size,&ptr,sub_arg[0],Ustrlen(sub_arg[0]));
4489
4490         /* Reset the "internal" variables afterwards, because they are in
4491         dynamic store that will be reclaimed if the expansion succeeded. */
4492
4493         prvscheck_address = NULL;
4494         prvscheck_keynum = NULL;
4495         }
4496       else
4497         {
4498         /* Does not look like a prvs encoded address, return the empty string.
4499            We need to make sure all subs are expanded first, so as to skip over
4500            the entire item. */
4501
4502         switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"prvs", &resetok))
4503           {
4504           case 1: goto EXPAND_FAILED_CURLY;
4505           case 2:
4506           case 3: goto EXPAND_FAILED;
4507           }
4508         }
4509
4510       continue;
4511       }
4512
4513     /* Handle "readfile" to insert an entire file */
4514
4515     case EITEM_READFILE:
4516       {
4517       FILE *f;
4518       uschar *sub_arg[2];
4519
4520       if ((expand_forbid & RDO_READFILE) != 0)
4521         {
4522         expand_string_message = US"file insertions are not permitted";
4523         goto EXPAND_FAILED;
4524         }
4525
4526       switch(read_subs(sub_arg, 2, 1, &s, skipping, TRUE, US"readfile", &resetok))
4527         {
4528         case 1: goto EXPAND_FAILED_CURLY;
4529         case 2:
4530         case 3: goto EXPAND_FAILED;
4531         }
4532
4533       /* If skipping, we don't actually do anything */
4534
4535       if (skipping) continue;
4536
4537       /* Open the file and read it */
4538
4539       f = Ufopen(sub_arg[0], "rb");
4540       if (f == NULL)
4541         {
4542         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
4543         goto EXPAND_FAILED;
4544         }
4545
4546       yield = cat_file(f, yield, &size, &ptr, sub_arg[1]);
4547       (void)fclose(f);
4548       continue;
4549       }
4550
4551     /* Handle "readsocket" to insert data from a Unix domain socket */
4552
4553     case EITEM_READSOCK:
4554       {
4555       int fd;
4556       int timeout = 5;
4557       int save_ptr = ptr;
4558       FILE *f;
4559       struct sockaddr_un sockun;         /* don't call this "sun" ! */
4560       uschar *arg;
4561       uschar *sub_arg[4];
4562
4563       if ((expand_forbid & RDO_READSOCK) != 0)
4564         {
4565         expand_string_message = US"socket insertions are not permitted";
4566         goto EXPAND_FAILED;
4567         }
4568
4569       /* Read up to 4 arguments, but don't do the end of item check afterwards,
4570       because there may be a string for expansion on failure. */
4571
4572       switch(read_subs(sub_arg, 4, 2, &s, skipping, FALSE, US"readsocket", &resetok))
4573         {
4574         case 1: goto EXPAND_FAILED_CURLY;
4575         case 2:                             /* Won't occur: no end check */
4576         case 3: goto EXPAND_FAILED;
4577         }
4578
4579       /* Sort out timeout, if given */
4580
4581       if (sub_arg[2] != NULL)
4582         {
4583         timeout = readconf_readtime(sub_arg[2], 0, FALSE);
4584         if (timeout < 0)
4585           {
4586           expand_string_message = string_sprintf("bad time value %s",
4587             sub_arg[2]);
4588           goto EXPAND_FAILED;
4589           }
4590         }
4591       else sub_arg[3] = NULL;                     /* No eol if no timeout */
4592
4593       /* If skipping, we don't actually do anything. Otherwise, arrange to
4594       connect to either an IP or a Unix socket. */
4595
4596       if (!skipping)
4597         {
4598         /* Handle an IP (internet) domain */
4599
4600         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
4601           {
4602           int port;
4603           uschar *server_name = sub_arg[0] + 5;
4604           uschar *port_name = Ustrrchr(server_name, ':');
4605
4606           /* Sort out the port */
4607
4608           if (port_name == NULL)
4609             {
4610             expand_string_message =
4611               string_sprintf("missing port for readsocket %s", sub_arg[0]);
4612             goto EXPAND_FAILED;
4613             }
4614           *port_name++ = 0;           /* Terminate server name */
4615
4616           if (isdigit(*port_name))
4617             {
4618             uschar *end;
4619             port = Ustrtol(port_name, &end, 0);
4620             if (end != port_name + Ustrlen(port_name))
4621               {
4622               expand_string_message =
4623                 string_sprintf("invalid port number %s", port_name);
4624               goto EXPAND_FAILED;
4625               }
4626             }
4627           else
4628             {
4629             struct servent *service_info = getservbyname(CS port_name, "tcp");
4630             if (service_info == NULL)
4631               {
4632               expand_string_message = string_sprintf("unknown port \"%s\"",
4633                 port_name);
4634               goto EXPAND_FAILED;
4635               }
4636             port = ntohs(service_info->s_port);
4637             }
4638
4639           if ((fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
4640                   timeout, NULL, &expand_string_message)) < 0)
4641               goto SOCK_FAIL;
4642           }
4643
4644         /* Handle a Unix domain socket */
4645
4646         else
4647           {
4648           int rc;
4649           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
4650             {
4651             expand_string_message = string_sprintf("failed to create socket: %s",
4652               strerror(errno));
4653             goto SOCK_FAIL;
4654             }
4655
4656           sockun.sun_family = AF_UNIX;
4657           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
4658             sub_arg[0]);
4659
4660           sigalrm_seen = FALSE;
4661           alarm(timeout);
4662           rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun));
4663           alarm(0);
4664           if (sigalrm_seen)
4665             {
4666             expand_string_message = US "socket connect timed out";
4667             goto SOCK_FAIL;
4668             }
4669           if (rc < 0)
4670             {
4671             expand_string_message = string_sprintf("failed to connect to socket "
4672               "%s: %s", sub_arg[0], strerror(errno));
4673             goto SOCK_FAIL;
4674             }
4675           }
4676
4677         DEBUG(D_expand) debug_printf("connected to socket %s\n", sub_arg[0]);
4678
4679         /* Allow sequencing of test actions */
4680         if (running_in_test_harness) millisleep(100);
4681
4682         /* Write the request string, if not empty */
4683
4684         if (sub_arg[1][0] != 0)
4685           {
4686           int len = Ustrlen(sub_arg[1]);
4687           DEBUG(D_expand) debug_printf("writing \"%s\" to socket\n",
4688             sub_arg[1]);
4689           if (write(fd, sub_arg[1], len) != len)
4690             {
4691             expand_string_message = string_sprintf("request write to socket "
4692               "failed: %s", strerror(errno));
4693             goto SOCK_FAIL;
4694             }
4695           }
4696
4697         /* Shut down the sending side of the socket. This helps some servers to
4698         recognise that it is their turn to do some work. Just in case some
4699         system doesn't have this function, make it conditional. */
4700
4701         #ifdef SHUT_WR
4702         shutdown(fd, SHUT_WR);
4703         #endif
4704
4705         if (running_in_test_harness) millisleep(100);
4706
4707         /* Now we need to read from the socket, under a timeout. The function
4708         that reads a file can be used. */
4709
4710         f = fdopen(fd, "rb");
4711         sigalrm_seen = FALSE;
4712         alarm(timeout);
4713         yield = cat_file(f, yield, &size, &ptr, sub_arg[3]);
4714         alarm(0);
4715         (void)fclose(f);
4716
4717         /* After a timeout, we restore the pointer in the result, that is,
4718         make sure we add nothing from the socket. */
4719
4720         if (sigalrm_seen)
4721           {
4722           ptr = save_ptr;
4723           expand_string_message = US "socket read timed out";
4724           goto SOCK_FAIL;
4725           }
4726         }
4727
4728       /* The whole thing has worked (or we were skipping). If there is a
4729       failure string following, we need to skip it. */
4730
4731       if (*s == '{')
4732         {
4733         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
4734           goto EXPAND_FAILED;
4735         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4736         while (isspace(*s)) s++;
4737         }
4738       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4739       continue;
4740
4741       /* Come here on failure to create socket, connect socket, write to the
4742       socket, or timeout on reading. If another substring follows, expand and
4743       use it. Otherwise, those conditions give expand errors. */
4744
4745       SOCK_FAIL:
4746       if (*s != '{') goto EXPAND_FAILED;
4747       DEBUG(D_any) debug_printf("%s\n", expand_string_message);
4748       arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok);
4749       if (arg == NULL) goto EXPAND_FAILED;
4750       yield = string_cat(yield, &size, &ptr, arg, Ustrlen(arg));
4751       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4752       while (isspace(*s)) s++;
4753       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4754       continue;
4755       }
4756
4757     /* Handle "run" to execute a program. */
4758
4759     case EITEM_RUN:
4760       {
4761       FILE *f;
4762       uschar *arg;
4763       uschar **argv;
4764       pid_t pid;
4765       int fd_in, fd_out;
4766       int lsize = 0;
4767       int lptr = 0;
4768
4769       if ((expand_forbid & RDO_RUN) != 0)
4770         {
4771         expand_string_message = US"running a command is not permitted";
4772         goto EXPAND_FAILED;
4773         }
4774
4775       while (isspace(*s)) s++;
4776       if (*s != '{') goto EXPAND_FAILED_CURLY;
4777       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
4778       if (arg == NULL) goto EXPAND_FAILED;
4779       while (isspace(*s)) s++;
4780       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
4781
4782       if (skipping)   /* Just pretend it worked when we're skipping */
4783         {
4784         runrc = 0;
4785         }
4786       else
4787         {
4788         if (!transport_set_up_command(&argv,    /* anchor for arg list */
4789             arg,                                /* raw command */
4790             FALSE,                              /* don't expand the arguments */
4791             0,                                  /* not relevant when... */
4792             NULL,                               /* no transporting address */
4793             US"${run} expansion",               /* for error messages */
4794             &expand_string_message))            /* where to put error message */
4795           {
4796           goto EXPAND_FAILED;
4797           }
4798
4799         /* Create the child process, making it a group leader. */
4800
4801         pid = child_open(argv, NULL, 0077, &fd_in, &fd_out, TRUE);
4802
4803         if (pid < 0)
4804           {
4805           expand_string_message =
4806             string_sprintf("couldn't create child process: %s", strerror(errno));
4807           goto EXPAND_FAILED;
4808           }
4809
4810         /* Nothing is written to the standard input. */
4811
4812         (void)close(fd_in);
4813
4814         /* Read the pipe to get the command's output into $value (which is kept
4815         in lookup_value). Read during execution, so that if the output exceeds
4816         the OS pipe buffer limit, we don't block forever. */
4817
4818         f = fdopen(fd_out, "rb");
4819         sigalrm_seen = FALSE;
4820         alarm(60);
4821         lookup_value = cat_file(f, lookup_value, &lsize, &lptr, NULL);
4822         alarm(0);
4823         (void)fclose(f);
4824
4825         /* Wait for the process to finish, applying the timeout, and inspect its
4826         return code for serious disasters. Simple non-zero returns are passed on.
4827         */
4828
4829         if (sigalrm_seen == TRUE || (runrc = child_close(pid, 30)) < 0)
4830           {
4831           if (sigalrm_seen == TRUE || runrc == -256)
4832             {
4833             expand_string_message = string_sprintf("command timed out");
4834             killpg(pid, SIGKILL);       /* Kill the whole process group */
4835             }
4836
4837           else if (runrc == -257)
4838             expand_string_message = string_sprintf("wait() failed: %s",
4839               strerror(errno));
4840
4841           else
4842             expand_string_message = string_sprintf("command killed by signal %d",
4843               -runrc);
4844
4845           goto EXPAND_FAILED;
4846           }
4847         }
4848
4849       /* Process the yes/no strings; $value may be useful in both cases */
4850
4851       switch(process_yesno(
4852                skipping,                     /* were previously skipping */
4853                runrc == 0,                   /* success/failure indicator */
4854                lookup_value,                 /* value to reset for string2 */
4855                &s,                           /* input pointer */
4856                &yield,                       /* output pointer */
4857                &size,                        /* output size */
4858                &ptr,                         /* output current point */
4859                US"run",                      /* condition type */
4860                &resetok))
4861         {
4862         case 1: goto EXPAND_FAILED;          /* when all is well, the */
4863         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
4864         }
4865
4866       continue;
4867       }
4868
4869     /* Handle character translation for "tr" */
4870
4871     case EITEM_TR:
4872       {
4873       int oldptr = ptr;
4874       int o2m;
4875       uschar *sub[3];
4876
4877       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"tr", &resetok))
4878         {
4879         case 1: goto EXPAND_FAILED_CURLY;
4880         case 2:
4881         case 3: goto EXPAND_FAILED;
4882         }
4883
4884       yield = string_cat(yield, &size, &ptr, sub[0], Ustrlen(sub[0]));
4885       o2m = Ustrlen(sub[2]) - 1;
4886
4887       if (o2m >= 0) for (; oldptr < ptr; oldptr++)
4888         {
4889         uschar *m = Ustrrchr(sub[1], yield[oldptr]);
4890         if (m != NULL)
4891           {
4892           int o = m - sub[1];
4893           yield[oldptr] = sub[2][(o < o2m)? o : o2m];
4894           }
4895         }
4896
4897       continue;
4898       }
4899
4900     /* Handle "hash", "length", "nhash", and "substr" when they are given with
4901     expanded arguments. */
4902
4903     case EITEM_HASH:
4904     case EITEM_LENGTH:
4905     case EITEM_NHASH:
4906     case EITEM_SUBSTR:
4907       {
4908       int i;
4909       int len;
4910       uschar *ret;
4911       int val[2] = { 0, -1 };
4912       uschar *sub[3];
4913
4914       /* "length" takes only 2 arguments whereas the others take 2 or 3.
4915       Ensure that sub[2] is set in the ${length } case. */
4916
4917       sub[2] = NULL;
4918       switch(read_subs(sub, (item_type == EITEM_LENGTH)? 2:3, 2, &s, skipping,
4919              TRUE, name, &resetok))
4920         {
4921         case 1: goto EXPAND_FAILED_CURLY;
4922         case 2:
4923         case 3: goto EXPAND_FAILED;
4924         }
4925
4926       /* Juggle the arguments if there are only two of them: always move the
4927       string to the last position and make ${length{n}{str}} equivalent to
4928       ${substr{0}{n}{str}}. See the defaults for val[] above. */
4929
4930       if (sub[2] == NULL)
4931         {
4932         sub[2] = sub[1];
4933         sub[1] = NULL;
4934         if (item_type == EITEM_LENGTH)
4935           {
4936           sub[1] = sub[0];
4937           sub[0] = NULL;
4938           }
4939         }
4940
4941       for (i = 0; i < 2; i++)
4942         {
4943         if (sub[i] == NULL) continue;
4944         val[i] = (int)Ustrtol(sub[i], &ret, 10);
4945         if (*ret != 0 || (i != 0 && val[i] < 0))
4946           {
4947           expand_string_message = string_sprintf("\"%s\" is not a%s number "
4948             "(in \"%s\" expansion)", sub[i], (i != 0)? " positive" : "", name);
4949           goto EXPAND_FAILED;
4950           }
4951         }
4952
4953       ret =
4954         (item_type == EITEM_HASH)?
4955           compute_hash(sub[2], val[0], val[1], &len) :
4956         (item_type == EITEM_NHASH)?
4957           compute_nhash(sub[2], val[0], val[1], &len) :
4958           extract_substr(sub[2], val[0], val[1], &len);
4959
4960       if (ret == NULL) goto EXPAND_FAILED;
4961       yield = string_cat(yield, &size, &ptr, ret, len);
4962       continue;
4963       }
4964
4965     /* Handle HMAC computation: ${hmac{<algorithm>}{<secret>}{<text>}}
4966     This code originally contributed by Steve Haslam. It currently supports
4967     the use of MD5 and SHA-1 hashes.
4968
4969     We need some workspace that is large enough to handle all the supported
4970     hash types. Use macros to set the sizes rather than be too elaborate. */
4971
4972     #define MAX_HASHLEN      20
4973     #define MAX_HASHBLOCKLEN 64
4974
4975     case EITEM_HMAC:
4976       {
4977       uschar *sub[3];
4978       md5 md5_base;
4979       sha1 sha1_base;
4980       void *use_base;
4981       int type, i;
4982       int hashlen;      /* Number of octets for the hash algorithm's output */
4983       int hashblocklen; /* Number of octets the hash algorithm processes */
4984       uschar *keyptr, *p;
4985       unsigned int keylen;
4986
4987       uschar keyhash[MAX_HASHLEN];
4988       uschar innerhash[MAX_HASHLEN];
4989       uschar finalhash[MAX_HASHLEN];
4990       uschar finalhash_hex[2*MAX_HASHLEN];
4991       uschar innerkey[MAX_HASHBLOCKLEN];
4992       uschar outerkey[MAX_HASHBLOCKLEN];
4993
4994       switch (read_subs(sub, 3, 3, &s, skipping, TRUE, name, &resetok))
4995         {
4996         case 1: goto EXPAND_FAILED_CURLY;
4997         case 2:
4998         case 3: goto EXPAND_FAILED;
4999         }
5000
5001       if (Ustrcmp(sub[0], "md5") == 0)
5002         {
5003         type = HMAC_MD5;
5004         use_base = &md5_base;
5005         hashlen = 16;
5006         hashblocklen = 64;
5007         }
5008       else if (Ustrcmp(sub[0], "sha1") == 0)
5009         {
5010         type = HMAC_SHA1;
5011         use_base = &sha1_base;
5012         hashlen = 20;
5013         hashblocklen = 64;
5014         }
5015       else
5016         {
5017         expand_string_message =
5018           string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
5019         goto EXPAND_FAILED;
5020         }
5021
5022       keyptr = sub[1];
5023       keylen = Ustrlen(keyptr);
5024
5025       /* If the key is longer than the hash block length, then hash the key
5026       first */
5027
5028       if (keylen > hashblocklen)
5029         {
5030         chash_start(type, use_base);
5031         chash_end(type, use_base, keyptr, keylen, keyhash);
5032         keyptr = keyhash;
5033         keylen = hashlen;
5034         }
5035
5036       /* Now make the inner and outer key values */
5037
5038       memset(innerkey, 0x36, hashblocklen);
5039       memset(outerkey, 0x5c, hashblocklen);
5040
5041       for (i = 0; i < keylen; i++)
5042         {
5043         innerkey[i] ^= keyptr[i];
5044         outerkey[i] ^= keyptr[i];
5045         }
5046
5047       /* Now do the hashes */
5048
5049       chash_start(type, use_base);
5050       chash_mid(type, use_base, innerkey);
5051       chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
5052
5053       chash_start(type, use_base);
5054       chash_mid(type, use_base, outerkey);
5055       chash_end(type, use_base, innerhash, hashlen, finalhash);
5056
5057       /* Encode the final hash as a hex string */
5058
5059       p = finalhash_hex;
5060       for (i = 0; i < hashlen; i++)
5061         {
5062         *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
5063         *p++ = hex_digits[finalhash[i] & 0x0f];
5064         }
5065
5066       DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%.*s)=%.*s\n", sub[0],
5067         (int)keylen, keyptr, Ustrlen(sub[2]), sub[2], hashlen*2, finalhash_hex);
5068
5069       yield = string_cat(yield, &size, &ptr, finalhash_hex, hashlen*2);
5070       }
5071
5072     continue;
5073
5074     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
5075     We have to save the numerical variables and restore them afterwards. */
5076
5077     case EITEM_SG:
5078       {
5079       const pcre *re;
5080       int moffset, moffsetextra, slen;
5081       int roffset;
5082       int emptyopt;
5083       const uschar *rerror;
5084       uschar *subject;
5085       uschar *sub[3];
5086       int save_expand_nmax =
5087         save_expand_strings(save_expand_nstring, save_expand_nlength);
5088
5089       switch(read_subs(sub, 3, 3, &s, skipping, TRUE, US"sg", &resetok))
5090         {
5091         case 1: goto EXPAND_FAILED_CURLY;
5092         case 2:
5093         case 3: goto EXPAND_FAILED;
5094         }
5095
5096       /* Compile the regular expression */
5097
5098       re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset,
5099         NULL);
5100
5101       if (re == NULL)
5102         {
5103         expand_string_message = string_sprintf("regular expression error in "
5104           "\"%s\": %s at offset %d", sub[1], rerror, roffset);
5105         goto EXPAND_FAILED;
5106         }
5107
5108       /* Now run a loop to do the substitutions as often as necessary. It ends
5109       when there are no more matches. Take care over matches of the null string;
5110       do the same thing as Perl does. */
5111
5112       subject = sub[0];
5113       slen = Ustrlen(sub[0]);
5114       moffset = moffsetextra = 0;
5115       emptyopt = 0;
5116
5117       for (;;)
5118         {
5119         int ovector[3*(EXPAND_MAXN+1)];
5120         int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra,
5121           PCRE_EOPT | emptyopt, ovector, sizeof(ovector)/sizeof(int));
5122         int nn;
5123         uschar *insert;
5124
5125         /* No match - if we previously set PCRE_NOTEMPTY after a null match, this
5126         is not necessarily the end. We want to repeat the match from one
5127         character further along, but leaving the basic offset the same (for
5128         copying below). We can't be at the end of the string - that was checked
5129         before setting PCRE_NOTEMPTY. If PCRE_NOTEMPTY is not set, we are
5130         finished; copy the remaining string and end the loop. */
5131
5132         if (n < 0)
5133           {
5134           if (emptyopt != 0)
5135             {
5136             moffsetextra = 1;
5137             emptyopt = 0;
5138             continue;
5139             }
5140           yield = string_cat(yield, &size, &ptr, subject+moffset, slen-moffset);
5141           break;
5142           }
5143
5144         /* Match - set up for expanding the replacement. */
5145
5146         if (n == 0) n = EXPAND_MAXN + 1;
5147         expand_nmax = 0;
5148         for (nn = 0; nn < n*2; nn += 2)
5149           {
5150           expand_nstring[expand_nmax] = subject + ovector[nn];
5151           expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
5152           }
5153         expand_nmax--;
5154
5155         /* Copy the characters before the match, plus the expanded insertion. */
5156
5157         yield = string_cat(yield, &size, &ptr, subject + moffset,
5158           ovector[0] - moffset);
5159         insert = expand_string(sub[2]);
5160         if (insert == NULL) goto EXPAND_FAILED;
5161         yield = string_cat(yield, &size, &ptr, insert, Ustrlen(insert));
5162
5163         moffset = ovector[1];
5164         moffsetextra = 0;
5165         emptyopt = 0;
5166
5167         /* If we have matched an empty string, first check to see if we are at
5168         the end of the subject. If so, the loop is over. Otherwise, mimic
5169         what Perl's /g options does. This turns out to be rather cunning. First
5170         we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match a non-empty
5171         string at the same point. If this fails (picked up above) we advance to
5172         the next character. */
5173
5174         if (ovector[0] == ovector[1])
5175           {
5176           if (ovector[0] == slen) break;
5177           emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED;
5178           }
5179         }
5180
5181       /* All done - restore numerical variables. */
5182
5183       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5184         save_expand_nlength);
5185       continue;
5186       }
5187
5188     /* Handle keyed and numbered substring extraction. If the first argument
5189     consists entirely of digits, then a numerical extraction is assumed. */
5190
5191     case EITEM_EXTRACT:
5192       {
5193       int i;
5194       int j = 2;
5195       int field_number = 1;
5196       BOOL field_number_set = FALSE;
5197       uschar *save_lookup_value = lookup_value;
5198       uschar *sub[3];
5199       int save_expand_nmax =
5200         save_expand_strings(save_expand_nstring, save_expand_nlength);
5201
5202       /* Read the arguments */
5203
5204       for (i = 0; i < j; i++)
5205         {
5206         while (isspace(*s)) s++;
5207         if (*s == '{')                                          /*}*/
5208           {
5209           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5210           if (sub[i] == NULL) goto EXPAND_FAILED;               /*{*/
5211           if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5212
5213           /* After removal of leading and trailing white space, the first
5214           argument must not be empty; if it consists entirely of digits
5215           (optionally preceded by a minus sign), this is a numerical
5216           extraction, and we expect 3 arguments. */
5217
5218           if (i == 0)
5219             {
5220             int len;
5221             int x = 0;
5222             uschar *p = sub[0];
5223
5224             while (isspace(*p)) p++;
5225             sub[0] = p;
5226
5227             len = Ustrlen(p);
5228             while (len > 0 && isspace(p[len-1])) len--;
5229             p[len] = 0;
5230
5231             if (*p == 0 && !skipping)
5232               {
5233               expand_string_message = US"first argument of \"extract\" must "
5234                 "not be empty";
5235               goto EXPAND_FAILED;
5236               }
5237
5238             if (*p == '-')
5239               {
5240               field_number = -1;
5241               p++;
5242               }
5243             while (*p != 0 && isdigit(*p)) x = x * 10 + *p++ - '0';
5244             if (*p == 0)
5245               {
5246               field_number *= x;
5247               j = 3;               /* Need 3 args */
5248               field_number_set = TRUE;
5249               }
5250             }
5251           }
5252         else goto EXPAND_FAILED_CURLY;
5253         }
5254
5255       /* Extract either the numbered or the keyed substring into $value. If
5256       skipping, just pretend the extraction failed. */
5257
5258       lookup_value = skipping? NULL : field_number_set?
5259         expand_gettokened(field_number, sub[1], sub[2]) :
5260         expand_getkeyed(sub[0], sub[1]);
5261
5262       /* If no string follows, $value gets substituted; otherwise there can
5263       be yes/no strings, as for lookup or if. */
5264
5265       switch(process_yesno(
5266                skipping,                     /* were previously skipping */
5267                lookup_value != NULL,         /* success/failure indicator */
5268                save_lookup_value,            /* value to reset for string2 */
5269                &s,                           /* input pointer */
5270                &yield,                       /* output pointer */
5271                &size,                        /* output size */
5272                &ptr,                         /* output current point */
5273                US"extract",                  /* condition type */
5274                &resetok))
5275         {
5276         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5277         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5278         }
5279
5280       /* All done - restore numerical variables. */
5281
5282       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5283         save_expand_nlength);
5284
5285       continue;
5286       }
5287
5288     /* return the Nth item from a list */
5289
5290     case EITEM_LISTEXTRACT:
5291       {
5292       int i;
5293       int field_number = 1;
5294       uschar *save_lookup_value = lookup_value;
5295       uschar *sub[2];
5296       int save_expand_nmax =
5297         save_expand_strings(save_expand_nstring, save_expand_nlength);
5298
5299       /* Read the field & list arguments */
5300
5301       for (i = 0; i < 2; i++)
5302         {
5303         while (isspace(*s)) s++;
5304         if (*s != '{')                                  /*}*/
5305           goto EXPAND_FAILED_CURLY;
5306
5307         sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5308         if (!sub[i])     goto EXPAND_FAILED;            /*{*/
5309         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5310
5311         /* After removal of leading and trailing white space, the first
5312         argument must be numeric and nonempty. */
5313
5314         if (i == 0)
5315           {
5316           int len;
5317           int x = 0;
5318           uschar *p = sub[0];
5319
5320           while (isspace(*p)) p++;
5321           sub[0] = p;
5322
5323           len = Ustrlen(p);
5324           while (len > 0 && isspace(p[len-1])) len--;
5325           p[len] = 0;
5326
5327           if (!*p && !skipping)
5328             {
5329             expand_string_message = US"first argument of \"listextract\" must "
5330               "not be empty";
5331             goto EXPAND_FAILED;
5332             }
5333
5334           if (*p == '-')
5335             {
5336             field_number = -1;
5337             p++;
5338             }
5339           while (*p && isdigit(*p)) x = x * 10 + *p++ - '0';
5340           if (*p)
5341             {
5342             expand_string_message = US"first argument of \"listextract\" must "
5343               "be numeric";
5344             goto EXPAND_FAILED;
5345             }
5346           field_number *= x;
5347           }
5348         }
5349
5350       /* Extract the numbered element into $value. If
5351       skipping, just pretend the extraction failed. */
5352
5353       lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]);
5354
5355       /* If no string follows, $value gets substituted; otherwise there can
5356       be yes/no strings, as for lookup or if. */
5357
5358       switch(process_yesno(
5359                skipping,                     /* were previously skipping */
5360                lookup_value != NULL,         /* success/failure indicator */
5361                save_lookup_value,            /* value to reset for string2 */
5362                &s,                           /* input pointer */
5363                &yield,                       /* output pointer */
5364                &size,                        /* output size */
5365                &ptr,                         /* output current point */
5366                US"extract",                  /* condition type */
5367                &resetok))
5368         {
5369         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5370         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5371         }
5372
5373       /* All done - restore numerical variables. */
5374
5375       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5376         save_expand_nlength);
5377
5378       continue;
5379       }
5380
5381 #ifdef SUPPORT_TLS
5382     case EITEM_CERTEXTRACT:
5383       {
5384       uschar *save_lookup_value = lookup_value;
5385       uschar *sub[2];
5386       int save_expand_nmax =
5387         save_expand_strings(save_expand_nstring, save_expand_nlength);
5388
5389       /* Read the field argument */
5390       while (isspace(*s)) s++;
5391       if (*s != '{')                                    /*}*/
5392         goto EXPAND_FAILED_CURLY;
5393       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5394       if (!sub[0])     goto EXPAND_FAILED;              /*{*/
5395       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5396       /* strip spaces fore & aft */
5397       {
5398       int len;
5399       uschar *p = sub[0];
5400
5401       while (isspace(*p)) p++;
5402       sub[0] = p;
5403
5404       len = Ustrlen(p);
5405       while (len > 0 && isspace(p[len-1])) len--;
5406       p[len] = 0;
5407       }
5408
5409       /* inspect the cert argument */
5410       while (isspace(*s)) s++;
5411       if (*s != '{')                                    /*}*/
5412         goto EXPAND_FAILED_CURLY;
5413       if (*++s != '$')
5414         {
5415         expand_string_message = US"second argument of \"certextract\" must "
5416           "be a certificate variable";
5417         goto EXPAND_FAILED;
5418         }
5419       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
5420       if (!sub[1])     goto EXPAND_FAILED;              /*{*/
5421       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5422
5423       if (skipping)
5424         lookup_value = NULL;
5425       else
5426         {
5427         lookup_value = expand_getcertele(sub[0], sub[1]);
5428         if (*expand_string_message) goto EXPAND_FAILED;
5429         }
5430       switch(process_yesno(
5431                skipping,                     /* were previously skipping */
5432                lookup_value != NULL,         /* success/failure indicator */
5433                save_lookup_value,            /* value to reset for string2 */
5434                &s,                           /* input pointer */
5435                &yield,                       /* output pointer */
5436                &size,                        /* output size */
5437                &ptr,                         /* output current point */
5438                US"extract",                  /* condition type */
5439                &resetok))
5440         {
5441         case 1: goto EXPAND_FAILED;          /* when all is well, the */
5442         case 2: goto EXPAND_FAILED_CURLY;    /* returned value is 0 */
5443         }
5444
5445       restore_expand_strings(save_expand_nmax, save_expand_nstring,
5446         save_expand_nlength);
5447       continue;
5448       }
5449 #endif  /*SUPPORT_TLS*/
5450
5451     /* Handle list operations */
5452
5453     case EITEM_FILTER:
5454     case EITEM_MAP:
5455     case EITEM_REDUCE:
5456       {
5457       int sep = 0;
5458       int save_ptr = ptr;
5459       uschar outsep[2] = { '\0', '\0' };
5460       uschar *list, *expr, *temp;
5461       uschar *save_iterate_item = iterate_item;
5462       uschar *save_lookup_value = lookup_value;
5463
5464       while (isspace(*s)) s++;
5465       if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5466
5467       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5468       if (list == NULL) goto EXPAND_FAILED;
5469       if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5470
5471       if (item_type == EITEM_REDUCE)
5472         {
5473         while (isspace(*s)) s++;
5474         if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5475         temp = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
5476         if (temp == NULL) goto EXPAND_FAILED;
5477         lookup_value = temp;
5478         if (*s++ != '}') goto EXPAND_FAILED_CURLY;
5479         }
5480
5481       while (isspace(*s)) s++;
5482       if (*s++ != '{') goto EXPAND_FAILED_CURLY;
5483
5484       expr = s;
5485
5486       /* For EITEM_FILTER, call eval_condition once, with result discarded (as
5487       if scanning a "false" part). This allows us to find the end of the
5488       condition, because if the list is empty, we won't actually evaluate the
5489       condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using
5490       the normal internal expansion function. */
5491
5492       if (item_type == EITEM_FILTER)
5493         {
5494         temp = eval_condition(expr, &resetok, NULL);
5495         if (temp != NULL) s = temp;
5496         }
5497       else
5498         {
5499         temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
5500         }
5501
5502       if (temp == NULL)
5503         {
5504         expand_string_message = string_sprintf("%s inside \"%s\" item",
5505           expand_string_message, name);
5506         goto EXPAND_FAILED;
5507         }
5508
5509       while (isspace(*s)) s++;
5510       if (*s++ != '}')
5511         {                                               /*{*/
5512         expand_string_message = string_sprintf("missing } at end of condition "
5513           "or expression inside \"%s\"", name);
5514         goto EXPAND_FAILED;
5515         }
5516
5517       while (isspace(*s)) s++;                          /*{*/
5518       if (*s++ != '}')
5519         {                                               /*{*/
5520         expand_string_message = string_sprintf("missing } at end of \"%s\"",
5521           name);
5522         goto EXPAND_FAILED;
5523         }
5524
5525       /* If we are skipping, we can now just move on to the next item. When
5526       processing for real, we perform the iteration. */
5527
5528       if (skipping) continue;
5529       while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL)
5530         {
5531         *outsep = (uschar)sep;      /* Separator as a string */
5532
5533         DEBUG(D_expand) debug_printf("%s: $item = \"%s\"\n", name, iterate_item);
5534
5535         if (item_type == EITEM_FILTER)
5536           {
5537           BOOL condresult;
5538           if (eval_condition(expr, &resetok, &condresult) == NULL)
5539             {
5540             iterate_item = save_iterate_item;
5541             lookup_value = save_lookup_value;
5542             expand_string_message = string_sprintf("%s inside \"%s\" condition",
5543               expand_string_message, name);
5544             goto EXPAND_FAILED;
5545             }
5546           DEBUG(D_expand) debug_printf("%s: condition is %s\n", name,
5547             condresult? "true":"false");
5548           if (condresult)
5549             temp = iterate_item;    /* TRUE => include this item */
5550           else
5551             continue;               /* FALSE => skip this item */
5552           }
5553
5554         /* EITEM_MAP and EITEM_REDUCE */
5555
5556         else
5557           {
5558           temp = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok);
5559           if (temp == NULL)
5560             {
5561             iterate_item = save_iterate_item;
5562             expand_string_message = string_sprintf("%s inside \"%s\" item",
5563               expand_string_message, name);
5564             goto EXPAND_FAILED;
5565             }
5566           if (item_type == EITEM_REDUCE)
5567             {
5568             lookup_value = temp;      /* Update the value of $value */
5569             continue;                 /* and continue the iteration */
5570             }
5571           }
5572
5573         /* We reach here for FILTER if the condition is true, always for MAP,
5574         and never for REDUCE. The value in "temp" is to be added to the output
5575         list that is being created, ensuring that any occurrences of the
5576         separator character are doubled. Unless we are dealing with the first
5577         item of the output list, add in a space if the new item begins with the
5578         separator character, or is an empty string. */
5579
5580         if (ptr != save_ptr && (temp[0] == *outsep || temp[0] == 0))
5581           yield = string_cat(yield, &size, &ptr, US" ", 1);
5582
5583         /* Add the string in "temp" to the output list that we are building,
5584         This is done in chunks by searching for the separator character. */
5585
5586         for (;;)
5587           {
5588           size_t seglen = Ustrcspn(temp, outsep);
5589             yield = string_cat(yield, &size, &ptr, temp, seglen + 1);
5590
5591           /* If we got to the end of the string we output one character
5592           too many; backup and end the loop. Otherwise arrange to double the
5593           separator. */
5594
5595           if (temp[seglen] == '\0') { ptr--; break; }
5596           yield = string_cat(yield, &size, &ptr, outsep, 1);
5597           temp += seglen + 1;
5598           }
5599
5600         /* Output a separator after the string: we will remove the redundant
5601         final one at the end. */
5602
5603         yield = string_cat(yield, &size, &ptr, outsep, 1);
5604         }   /* End of iteration over the list loop */
5605
5606       /* REDUCE has generated no output above: output the final value of
5607       $value. */
5608
5609       if (item_type == EITEM_REDUCE)
5610         {
5611         yield = string_cat(yield, &size, &ptr, lookup_value,
5612           Ustrlen(lookup_value));
5613         lookup_value = save_lookup_value;  /* Restore $value */
5614         }
5615
5616       /* FILTER and MAP generate lists: if they have generated anything, remove
5617       the redundant final separator. Even though an empty item at the end of a
5618       list does not count, this is tidier. */
5619
5620       else if (ptr != save_ptr) ptr--;
5621
5622       /* Restore preserved $item */
5623
5624       iterate_item = save_iterate_item;
5625       continue;
5626       }
5627
5628
5629     /* If ${dlfunc } support is configured, handle calling dynamically-loaded
5630     functions, unless locked out at this time. Syntax is ${dlfunc{file}{func}}
5631     or ${dlfunc{file}{func}{arg}} or ${dlfunc{file}{func}{arg1}{arg2}} or up to
5632     a maximum of EXPAND_DLFUNC_MAX_ARGS arguments (defined below). */
5633
5634     #define EXPAND_DLFUNC_MAX_ARGS 8
5635
5636     case EITEM_DLFUNC:
5637     #ifndef EXPAND_DLFUNC
5638     expand_string_message = US"\"${dlfunc\" encountered, but this facility "    /*}*/
5639       "is not included in this binary";
5640     goto EXPAND_FAILED;
5641
5642     #else   /* EXPAND_DLFUNC */
5643       {
5644       tree_node *t;
5645       exim_dlfunc_t *func;
5646       uschar *result;
5647       int status, argc;
5648       uschar *argv[EXPAND_DLFUNC_MAX_ARGS + 3];
5649
5650       if ((expand_forbid & RDO_DLFUNC) != 0)
5651         {
5652         expand_string_message =
5653           US"dynamically-loaded functions are not permitted";
5654         goto EXPAND_FAILED;
5655         }
5656
5657       switch(read_subs(argv, EXPAND_DLFUNC_MAX_ARGS + 2, 2, &s, skipping,
5658            TRUE, US"dlfunc", &resetok))
5659         {
5660         case 1: goto EXPAND_FAILED_CURLY;
5661         case 2:
5662         case 3: goto EXPAND_FAILED;
5663         }
5664
5665       /* If skipping, we don't actually do anything */
5666
5667       if (skipping) continue;
5668
5669       /* Look up the dynamically loaded object handle in the tree. If it isn't
5670       found, dlopen() the file and put the handle in the tree for next time. */
5671
5672       t = tree_search(dlobj_anchor, argv[0]);
5673       if (t == NULL)
5674         {
5675         void *handle = dlopen(CS argv[0], RTLD_LAZY);
5676         if (handle == NULL)
5677           {
5678           expand_string_message = string_sprintf("dlopen \"%s\" failed: %s",
5679             argv[0], dlerror());
5680           log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
5681           goto EXPAND_FAILED;
5682           }
5683         t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]));
5684         Ustrcpy(t->name, argv[0]);
5685         t->data.ptr = handle;
5686         (void)tree_insertnode(&dlobj_anchor, t);
5687         }
5688
5689       /* Having obtained the dynamically loaded object handle, look up the
5690       function pointer. */
5691
5692       func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]);
5693       if (func == NULL)
5694         {
5695         expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: "
5696           "%s", argv[1], argv[0], dlerror());
5697         log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message);
5698         goto EXPAND_FAILED;
5699         }
5700
5701       /* Call the function and work out what to do with the result. If it
5702       returns OK, we have a replacement string; if it returns DEFER then
5703       expansion has failed in a non-forced manner; if it returns FAIL then
5704       failure was forced; if it returns ERROR or any other value there's a
5705       problem, so panic slightly. In any case, assume that the function has
5706       side-effects on the store that must be preserved. */
5707
5708       resetok = FALSE;
5709       result = NULL;
5710       for (argc = 0; argv[argc] != NULL; argc++);
5711       status = func(&result, argc - 2, &argv[2]);
5712       if(status == OK)
5713         {
5714         if (result == NULL) result = US"";
5715         yield = string_cat(yield, &size, &ptr, result, Ustrlen(result));
5716         continue;
5717         }
5718       else
5719         {
5720         expand_string_message = result == NULL ? US"(no message)" : result;
5721         if(status == FAIL_FORCED) expand_string_forcedfail = TRUE;
5722           else if(status != FAIL)
5723             log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s",
5724               argv[0], argv[1], status, expand_string_message);
5725         goto EXPAND_FAILED;
5726         }
5727       }
5728     #endif /* EXPAND_DLFUNC */
5729     }   /* EITEM_* switch */
5730
5731   /* Control reaches here if the name is not recognized as one of the more
5732   complicated expansion items. Check for the "operator" syntax (name terminated
5733   by a colon). Some of the operators have arguments, separated by _ from the
5734   name. */
5735
5736   if (*s == ':')
5737     {
5738     int c;
5739     uschar *arg = NULL;
5740     uschar *sub;
5741     var_entry *vp = NULL;
5742
5743     /* Owing to an historical mis-design, an underscore may be part of the
5744     operator name, or it may introduce arguments.  We therefore first scan the
5745     table of names that contain underscores. If there is no match, we cut off
5746     the arguments and then scan the main table. */
5747
5748     if ((c = chop_match(name, op_table_underscore,
5749         sizeof(op_table_underscore)/sizeof(uschar *))) < 0)
5750       {
5751       arg = Ustrchr(name, '_');
5752       if (arg != NULL) *arg = 0;
5753       c = chop_match(name, op_table_main,
5754         sizeof(op_table_main)/sizeof(uschar *));
5755       if (c >= 0) c += sizeof(op_table_underscore)/sizeof(uschar *);
5756       if (arg != NULL) *arg++ = '_';   /* Put back for error messages */
5757       }
5758
5759     /* Deal specially with operators that might take a certificate variable
5760     as we do not want to do the usual expansion. For most, expand the string.*/
5761     switch(c)
5762       {
5763 #ifdef SUPPORT_TLS
5764       case EOP_MD5:
5765       case EOP_SHA1:
5766       case EOP_SHA256:
5767         if (s[1] == '$')
5768           {
5769           uschar * s1 = s;
5770           sub = expand_string_internal(s+2, TRUE, &s1, skipping,
5771                   FALSE, &resetok);
5772           if (!sub)       goto EXPAND_FAILED;           /*{*/
5773           if (*s1 != '}') goto EXPAND_FAILED_CURLY;
5774           if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
5775             {
5776             s = s1+1;
5777             break;
5778             }
5779           vp = NULL;
5780           }
5781         /*FALLTHROUGH*/
5782 #endif
5783       default:
5784         sub = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
5785         if (!sub) goto EXPAND_FAILED;
5786         s++;
5787         break;
5788       }
5789
5790     /* If we are skipping, we don't need to perform the operation at all.
5791     This matters for operations like "mask", because the data may not be
5792     in the correct format when skipping. For example, the expression may test
5793     for the existence of $sender_host_address before trying to mask it. For
5794     other operations, doing them may not fail, but it is a waste of time. */
5795
5796     if (skipping && c >= 0) continue;
5797
5798     /* Otherwise, switch on the operator type */
5799
5800     switch(c)
5801       {
5802       case EOP_BASE62:
5803         {
5804         uschar *t;
5805         unsigned long int n = Ustrtoul(sub, &t, 10);
5806         if (*t != 0)
5807           {
5808           expand_string_message = string_sprintf("argument for base62 "
5809             "operator is \"%s\", which is not a decimal number", sub);
5810           goto EXPAND_FAILED;
5811           }
5812         t = string_base62(n);
5813         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
5814         continue;
5815         }
5816
5817       /* Note that for Darwin and Cygwin, BASE_62 actually has the value 36 */
5818
5819       case EOP_BASE62D:
5820         {
5821         uschar buf[16];
5822         uschar *tt = sub;
5823         unsigned long int n = 0;
5824         while (*tt != 0)
5825           {
5826           uschar *t = Ustrchr(base62_chars, *tt++);
5827           if (t == NULL)
5828             {
5829             expand_string_message = string_sprintf("argument for base62d "
5830               "operator is \"%s\", which is not a base %d number", sub,
5831               BASE_62);
5832             goto EXPAND_FAILED;
5833             }
5834           n = n * BASE_62 + (t - base62_chars);
5835           }
5836         (void)sprintf(CS buf, "%ld", n);
5837         yield = string_cat(yield, &size, &ptr, buf, Ustrlen(buf));
5838         continue;
5839         }
5840
5841       case EOP_EXPAND:
5842         {
5843         uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok);
5844         if (expanded == NULL)
5845           {
5846           expand_string_message =
5847             string_sprintf("internal expansion of \"%s\" failed: %s", sub,
5848               expand_string_message);
5849           goto EXPAND_FAILED;
5850           }
5851         yield = string_cat(yield, &size, &ptr, expanded, Ustrlen(expanded));
5852         continue;
5853         }
5854
5855       case EOP_LC:
5856         {
5857         int count = 0;
5858         uschar *t = sub - 1;
5859         while (*(++t) != 0) { *t = tolower(*t); count++; }
5860         yield = string_cat(yield, &size, &ptr, sub, count);
5861         continue;
5862         }
5863
5864       case EOP_UC:
5865         {
5866         int count = 0;
5867         uschar *t = sub - 1;
5868         while (*(++t) != 0) { *t = toupper(*t); count++; }
5869         yield = string_cat(yield, &size, &ptr, sub, count);
5870         continue;
5871         }
5872
5873       case EOP_MD5:
5874 #ifdef SUPPORT_TLS
5875         if (vp && *(void **)vp->value)
5876           {
5877           uschar * cp = tls_cert_fprt_md5(*(void **)vp->value);
5878           yield = string_cat(yield, &size, &ptr, cp, Ustrlen(cp));
5879           }
5880         else
5881 #endif
5882           {
5883           md5 base;
5884           uschar digest[16];
5885           int j;
5886           char st[33];
5887           md5_start(&base);
5888           md5_end(&base, sub, Ustrlen(sub), digest);
5889           for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]);
5890           yield = string_cat(yield, &size, &ptr, US st, (int)strlen(st));
5891           }
5892         continue;
5893
5894       case EOP_SHA1:
5895 #ifdef SUPPORT_TLS
5896         if (vp && *(void **)vp->value)
5897           {
5898           uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value);
5899           yield = string_cat(yield, &size, &ptr, cp, Ustrlen(cp));
5900           }
5901         else
5902 #endif
5903           {
5904           sha1 base;
5905           uschar digest[20];
5906           int j;
5907           char st[41];
5908           sha1_start(&base);
5909           sha1_end(&base, sub, Ustrlen(sub), digest);
5910           for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
5911           yield = string_cat(yield, &size, &ptr, US st, (int)strlen(st));
5912           }
5913         continue;
5914
5915       case EOP_SHA256:
5916 #ifdef SUPPORT_TLS
5917         if (vp && *(void **)vp->value)
5918           {
5919           uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value);
5920           yield = string_cat(yield, &size, &ptr, cp, (int)Ustrlen(cp));
5921           }
5922         else
5923 #endif
5924           expand_string_message = US"sha256 only supported for certificates";
5925         continue;
5926
5927       /* Convert hex encoding to base64 encoding */
5928
5929       case EOP_HEX2B64:
5930         {
5931         int c = 0;
5932         int b = -1;
5933         uschar *in = sub;
5934         uschar *out = sub;
5935         uschar *enc;
5936
5937         for (enc = sub; *enc != 0; enc++)
5938           {
5939           if (!isxdigit(*enc))
5940             {
5941             expand_string_message = string_sprintf("\"%s\" is not a hex "
5942               "string", sub);
5943             goto EXPAND_FAILED;
5944             }
5945           c++;
5946           }
5947
5948         if ((c & 1) != 0)
5949           {
5950           expand_string_message = string_sprintf("\"%s\" contains an odd "
5951             "number of characters", sub);
5952           goto EXPAND_FAILED;
5953           }
5954
5955         while ((c = *in++) != 0)
5956           {
5957           if (isdigit(c)) c -= '0';
5958           else c = toupper(c) - 'A' + 10;
5959           if (b == -1)
5960             {
5961             b = c << 4;
5962             }
5963           else
5964             {
5965             *out++ = b | c;
5966             b = -1;
5967             }
5968           }
5969
5970         enc = auth_b64encode(sub, out - sub);
5971         yield = string_cat(yield, &size, &ptr, enc, Ustrlen(enc));
5972         continue;
5973         }
5974
5975       /* Convert octets outside 0x21..0x7E to \xXX form */
5976
5977       case EOP_HEXQUOTE:
5978         {
5979         uschar *t = sub - 1;
5980         while (*(++t) != 0)
5981           {
5982           if (*t < 0x21 || 0x7E < *t)
5983             yield = string_cat(yield, &size, &ptr,
5984               string_sprintf("\\x%02x", *t), 4);
5985           else
5986             yield = string_cat(yield, &size, &ptr, t, 1);
5987           }
5988         continue;
5989         }
5990
5991       /* count the number of list elements */
5992
5993       case EOP_LISTCOUNT:
5994         {
5995         int cnt = 0;
5996         int sep = 0;
5997         uschar * cp;
5998         uschar buffer[256];
5999
6000         while (string_nextinlist(&sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++;
6001         cp = string_sprintf("%d", cnt);
6002         yield = string_cat(yield, &size, &ptr, cp, Ustrlen(cp));
6003         continue;
6004         }
6005
6006       /* expand a named list given the name */
6007       /* handles nested named lists; requotes as colon-sep list */
6008
6009       case EOP_LISTNAMED:
6010         {
6011         tree_node *t = NULL;
6012         uschar * list;
6013         int sep = 0;
6014         uschar * item;
6015         uschar * suffix = US"";
6016         BOOL needsep = FALSE;
6017         uschar buffer[256];
6018
6019         if (*sub == '+') sub++;
6020         if (arg == NULL)        /* no-argument version */
6021           {
6022           if (!(t = tree_search(addresslist_anchor, sub)) &&
6023               !(t = tree_search(domainlist_anchor,  sub)) &&
6024               !(t = tree_search(hostlist_anchor,    sub)))
6025             t = tree_search(localpartlist_anchor, sub);
6026           }
6027         else switch(*arg)       /* specific list-type version */
6028           {
6029           case 'a': t = tree_search(addresslist_anchor,   sub); suffix = US"_a"; break;
6030           case 'd': t = tree_search(domainlist_anchor,    sub); suffix = US"_d"; break;
6031           case 'h': t = tree_search(hostlist_anchor,      sub); suffix = US"_h"; break;
6032           case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break;
6033           default:
6034             expand_string_message = string_sprintf("bad suffix on \"list\" operator");
6035             goto EXPAND_FAILED;
6036           }
6037
6038         if(!t)
6039           {
6040           expand_string_message = string_sprintf("\"%s\" is not a %snamed list",
6041             sub, !arg?""
6042               : *arg=='a'?"address "
6043               : *arg=='d'?"domain "
6044               : *arg=='h'?"host "
6045               : *arg=='l'?"localpart "
6046               : 0);
6047           goto EXPAND_FAILED;
6048           }
6049
6050         list = ((namedlist_block *)(t->data.ptr))->string;
6051
6052         while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) != NULL)
6053           {
6054           uschar * buf = US" : ";
6055           if (needsep)
6056             yield = string_cat(yield, &size, &ptr, buf, 3);
6057           else
6058             needsep = TRUE;
6059
6060           if (*item == '+')     /* list item is itself a named list */
6061             {
6062             uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item);
6063             item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok);
6064             }
6065           else if (sep != ':')  /* item from non-colon-sep list, re-quote for colon list-separator */
6066             {
6067             char * cp;
6068             char tok[3];
6069             tok[0] = sep; tok[1] = ':'; tok[2] = 0;
6070             while ((cp= strpbrk((const char *)item, tok)))
6071               {
6072               yield = string_cat(yield, &size, &ptr, item, cp-(char *)item);
6073               if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */
6074                 {
6075                 yield = string_cat(yield, &size, &ptr, US"::", 2);
6076                 item = (uschar *)cp;
6077                 }
6078               else              /* sep in item; should already be doubled; emit once */
6079                 {
6080                 yield = string_cat(yield, &size, &ptr, (uschar *)tok, 1);
6081                 if (*cp == sep) cp++;
6082                 item = (uschar *)cp;
6083                 }
6084               }
6085             }
6086           yield = string_cat(yield, &size, &ptr, item, Ustrlen(item));
6087           }
6088         continue;
6089         }
6090
6091       /* mask applies a mask to an IP address; for example the result of
6092       ${mask:131.111.10.206/28} is 131.111.10.192/28. */
6093
6094       case EOP_MASK:
6095         {
6096         int count;
6097         uschar *endptr;
6098         int binary[4];
6099         int mask, maskoffset;
6100         int type = string_is_ip_address(sub, &maskoffset);
6101         uschar buffer[64];
6102
6103         if (type == 0)
6104           {
6105           expand_string_message = string_sprintf("\"%s\" is not an IP address",
6106            sub);
6107           goto EXPAND_FAILED;
6108           }
6109
6110         if (maskoffset == 0)
6111           {
6112           expand_string_message = string_sprintf("missing mask value in \"%s\"",
6113             sub);
6114           goto EXPAND_FAILED;
6115           }
6116
6117         mask = Ustrtol(sub + maskoffset + 1, &endptr, 10);
6118
6119         if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128))
6120           {
6121           expand_string_message = string_sprintf("mask value too big in \"%s\"",
6122             sub);
6123           goto EXPAND_FAILED;
6124           }
6125
6126         /* Convert the address to binary integer(s) and apply the mask */
6127
6128         sub[maskoffset] = 0;
6129         count = host_aton(sub, binary);
6130         host_mask(count, binary, mask);
6131
6132         /* Convert to masked textual format and add to output. */
6133
6134         yield = string_cat(yield, &size, &ptr, buffer,
6135           host_nmtoa(count, binary, mask, buffer, '.'));
6136         continue;
6137         }
6138
6139       case EOP_ADDRESS:
6140       case EOP_LOCAL_PART:
6141       case EOP_DOMAIN:
6142         {
6143         uschar *error;
6144         int start, end, domain;
6145         uschar *t = parse_extract_address(sub, &error, &start, &end, &domain,
6146           FALSE);
6147         if (t != NULL)
6148           {
6149           if (c != EOP_DOMAIN)
6150             {
6151             if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1;
6152             yield = string_cat(yield, &size, &ptr, sub+start, end-start);
6153             }
6154           else if (domain != 0)
6155             {
6156             domain += start;
6157             yield = string_cat(yield, &size, &ptr, sub+domain, end-domain);
6158             }
6159           }
6160         continue;
6161         }
6162
6163       case EOP_ADDRESSES:
6164         {
6165         uschar outsep[2] = { ':', '\0' };
6166         uschar *address, *error;
6167         int save_ptr = ptr;
6168         int start, end, domain;  /* Not really used */
6169
6170         while (isspace(*sub)) sub++;
6171         if (*sub == '>') { *outsep = *++sub; ++sub; }
6172         parse_allow_group = TRUE;
6173
6174         for (;;)
6175           {
6176           uschar *p = parse_find_address_end(sub, FALSE);
6177           uschar saveend = *p;
6178           *p = '\0';
6179           address = parse_extract_address(sub, &error, &start, &end, &domain,
6180             FALSE);
6181           *p = saveend;
6182
6183           /* Add the address to the output list that we are building. This is
6184           done in chunks by searching for the separator character. At the
6185           start, unless we are dealing with the first address of the output
6186           list, add in a space if the new address begins with the separator
6187           character, or is an empty string. */
6188
6189           if (address != NULL)
6190             {
6191             if (ptr != save_ptr && address[0] == *outsep)
6192               yield = string_cat(yield, &size, &ptr, US" ", 1);
6193
6194             for (;;)
6195               {
6196               size_t seglen = Ustrcspn(address, outsep);
6197               yield = string_cat(yield, &size, &ptr, address, seglen + 1);
6198
6199               /* If we got to the end of the string we output one character
6200               too many. */
6201
6202               if (address[seglen] == '\0') { ptr--; break; }
6203               yield = string_cat(yield, &size, &ptr, outsep, 1);
6204               address += seglen + 1;
6205               }
6206
6207             /* Output a separator after the string: we will remove the
6208             redundant final one at the end. */
6209
6210             yield = string_cat(yield, &size, &ptr, outsep, 1);
6211             }
6212
6213           if (saveend == '\0') break;
6214           sub = p + 1;
6215           }
6216
6217         /* If we have generated anything, remove the redundant final
6218         separator. */
6219
6220         if (ptr != save_ptr) ptr--;
6221         parse_allow_group = FALSE;
6222         continue;
6223         }
6224
6225
6226       /* quote puts a string in quotes if it is empty or contains anything
6227       other than alphamerics, underscore, dot, or hyphen.
6228
6229       quote_local_part puts a string in quotes if RFC 2821/2822 requires it to
6230       be quoted in order to be a valid local part.
6231
6232       In both cases, newlines and carriage returns are converted into \n and \r
6233       respectively */
6234
6235       case EOP_QUOTE:
6236       case EOP_QUOTE_LOCAL_PART:
6237       if (arg == NULL)
6238         {
6239         BOOL needs_quote = (*sub == 0);      /* TRUE for empty string */
6240         uschar *t = sub - 1;
6241
6242         if (c == EOP_QUOTE)
6243           {
6244           while (!needs_quote && *(++t) != 0)
6245             needs_quote = !isalnum(*t) && !strchr("_-.", *t);
6246           }
6247         else  /* EOP_QUOTE_LOCAL_PART */
6248           {
6249           while (!needs_quote && *(++t) != 0)
6250             needs_quote = !isalnum(*t) &&
6251               strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL &&
6252               (*t != '.' || t == sub || t[1] == 0);
6253           }
6254
6255         if (needs_quote)
6256           {
6257           yield = string_cat(yield, &size, &ptr, US"\"", 1);
6258           t = sub - 1;
6259           while (*(++t) != 0)
6260             {
6261             if (*t == '\n')
6262               yield = string_cat(yield, &size, &ptr, US"\\n", 2);
6263             else if (*t == '\r')
6264               yield = string_cat(yield, &size, &ptr, US"\\r", 2);
6265             else
6266               {
6267               if (*t == '\\' || *t == '"')
6268                 yield = string_cat(yield, &size, &ptr, US"\\", 1);
6269               yield = string_cat(yield, &size, &ptr, t, 1);
6270               }
6271             }
6272           yield = string_cat(yield, &size, &ptr, US"\"", 1);
6273           }
6274         else yield = string_cat(yield, &size, &ptr, sub, Ustrlen(sub));
6275         continue;
6276         }
6277
6278       /* quote_lookuptype does lookup-specific quoting */
6279
6280       else
6281         {
6282         int n;
6283         uschar *opt = Ustrchr(arg, '_');
6284
6285         if (opt != NULL) *opt++ = 0;
6286
6287         n = search_findtype(arg, Ustrlen(arg));
6288         if (n < 0)
6289           {
6290           expand_string_message = search_error_message;
6291           goto EXPAND_FAILED;
6292           }
6293
6294         if (lookup_list[n]->quote != NULL)
6295           sub = (lookup_list[n]->quote)(sub, opt);
6296         else if (opt != NULL) sub = NULL;
6297
6298         if (sub == NULL)
6299           {
6300           expand_string_message = string_sprintf(
6301             "\"%s\" unrecognized after \"${quote_%s\"",
6302             opt, arg);
6303           goto EXPAND_FAILED;
6304           }
6305
6306         yield = string_cat(yield, &size, &ptr, sub, Ustrlen(sub));
6307         continue;
6308         }
6309
6310       /* rx quote sticks in \ before any non-alphameric character so that
6311       the insertion works in a regular expression. */
6312
6313       case EOP_RXQUOTE:
6314         {
6315         uschar *t = sub - 1;
6316         while (*(++t) != 0)
6317           {
6318           if (!isalnum(*t))
6319             yield = string_cat(yield, &size, &ptr, US"\\", 1);
6320           yield = string_cat(yield, &size, &ptr, t, 1);
6321           }
6322         continue;
6323         }
6324
6325       /* RFC 2047 encodes, assuming headers_charset (default ISO 8859-1) as
6326       prescribed by the RFC, if there are characters that need to be encoded */
6327
6328       case EOP_RFC2047:
6329         {
6330         uschar buffer[2048];
6331         uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset,
6332           buffer, sizeof(buffer), FALSE);
6333         yield = string_cat(yield, &size, &ptr, string, Ustrlen(string));
6334         continue;
6335         }
6336
6337       /* RFC 2047 decode */
6338
6339       case EOP_RFC2047D:
6340         {
6341         int len;
6342         uschar *error;
6343         uschar *decoded = rfc2047_decode(sub, check_rfc2047_length,
6344           headers_charset, '?', &len, &error);
6345         if (error != NULL)
6346           {
6347           expand_string_message = error;
6348           goto EXPAND_FAILED;
6349           }
6350         yield = string_cat(yield, &size, &ptr, decoded, len);
6351         continue;
6352         }
6353
6354       /* from_utf8 converts UTF-8 to 8859-1, turning non-existent chars into
6355       underscores */
6356
6357       case EOP_FROM_UTF8:
6358         {
6359         while (*sub != 0)
6360           {
6361           int c;
6362           uschar buff[4];
6363           GETUTF8INC(c, sub);
6364           if (c > 255) c = '_';
6365           buff[0] = c;
6366           yield = string_cat(yield, &size, &ptr, buff, 1);
6367           }
6368         continue;
6369         }
6370
6371           /* replace illegal UTF-8 sequences by replacement character  */
6372
6373       #define UTF8_REPLACEMENT_CHAR US"?"
6374
6375       case EOP_UTF8CLEAN:
6376         {
6377         int seq_len = 0, index = 0;
6378         int bytes_left = 0;
6379         uschar seq_buff[4];                     /* accumulate utf-8 here */
6380
6381         while (*sub != 0)
6382           {
6383           int complete;
6384           long codepoint = 0;
6385           uschar c;
6386
6387           complete = 0;
6388           c = *sub++;
6389           if (bytes_left)
6390             {
6391             if ((c & 0xc0) != 0x80)
6392               {
6393                     /* wrong continuation byte; invalidate all bytes */
6394               complete = 1; /* error */
6395               }
6396             else
6397               {
6398               codepoint = (codepoint << 6) | (c & 0x3f);
6399               seq_buff[index++] = c;
6400               if (--bytes_left == 0)            /* codepoint complete */
6401                 {
6402                 if(codepoint > 0x10FFFF)        /* is it too large? */
6403                   complete = -1;        /* error */
6404                 else
6405                   {             /* finished; output utf-8 sequence */
6406                   yield = string_cat(yield, &size, &ptr, seq_buff, seq_len);
6407                   index = 0;
6408                   }
6409                 }
6410               }
6411             }
6412           else  /* no bytes left: new sequence */
6413             {
6414             if((c & 0x80) == 0) /* 1-byte sequence, US-ASCII, keep it */
6415               {
6416               yield = string_cat(yield, &size, &ptr, &c, 1);
6417               continue;
6418               }
6419             if((c & 0xe0) == 0xc0)              /* 2-byte sequence */
6420               {
6421               if(c == 0xc0 || c == 0xc1)        /* 0xc0 and 0xc1 are illegal */
6422                 complete = -1;
6423               else
6424                 {
6425                   bytes_left = 1;
6426                   codepoint = c & 0x1f;
6427                 }
6428               }
6429             else if((c & 0xf0) == 0xe0)         /* 3-byte sequence */
6430               {
6431               bytes_left = 2;
6432               codepoint = c & 0x0f;
6433               }
6434             else if((c & 0xf8) == 0xf0)         /* 4-byte sequence */
6435               {
6436               bytes_left = 3;
6437               codepoint = c & 0x07;
6438               }
6439             else        /* invalid or too long (RFC3629 allows only 4 bytes) */
6440               complete = -1;
6441
6442             seq_buff[index++] = c;
6443             seq_len = bytes_left + 1;
6444             }           /* if(bytes_left) */
6445
6446           if (complete != 0)
6447             {
6448             bytes_left = index = 0;
6449             yield = string_cat(yield, &size, &ptr, UTF8_REPLACEMENT_CHAR, 1);
6450             }
6451           if ((complete == 1) && ((c & 0x80) == 0))
6452             { /* ASCII character follows incomplete sequence */
6453               yield = string_cat(yield, &size, &ptr, &c, 1);
6454             }
6455           }
6456         continue;
6457         }
6458
6459       /* escape turns all non-printing characters into escape sequences. */
6460
6461       case EOP_ESCAPE:
6462         {
6463         uschar *t = string_printing(sub);
6464         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
6465         continue;
6466         }
6467
6468       /* Handle numeric expression evaluation */
6469
6470       case EOP_EVAL:
6471       case EOP_EVAL10:
6472         {
6473         uschar *save_sub = sub;
6474         uschar *error = NULL;
6475         int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE);
6476         if (error != NULL)
6477           {
6478           expand_string_message = string_sprintf("error in expression "
6479             "evaluation: %s (after processing \"%.*s\")", error, sub-save_sub,
6480               save_sub);
6481           goto EXPAND_FAILED;
6482           }
6483         sprintf(CS var_buffer, PR_EXIM_ARITH, n);
6484         yield = string_cat(yield, &size, &ptr, var_buffer, Ustrlen(var_buffer));
6485         continue;
6486         }
6487
6488       /* Handle time period formating */
6489
6490       case EOP_TIME_EVAL:
6491         {
6492         int n = readconf_readtime(sub, 0, FALSE);
6493         if (n < 0)
6494           {
6495           expand_string_message = string_sprintf("string \"%s\" is not an "
6496             "Exim time interval in \"%s\" operator", sub, name);
6497           goto EXPAND_FAILED;
6498           }
6499         sprintf(CS var_buffer, "%d", n);
6500         yield = string_cat(yield, &size, &ptr, var_buffer, Ustrlen(var_buffer));
6501         continue;
6502         }
6503
6504       case EOP_TIME_INTERVAL:
6505         {
6506         int n;
6507         uschar *t = read_number(&n, sub);
6508         if (*t != 0) /* Not A Number*/
6509           {
6510           expand_string_message = string_sprintf("string \"%s\" is not a "
6511             "positive number in \"%s\" operator", sub, name);
6512           goto EXPAND_FAILED;
6513           }
6514         t = readconf_printtime(n);
6515         yield = string_cat(yield, &size, &ptr, t, Ustrlen(t));
6516         continue;
6517         }
6518
6519       /* Convert string to base64 encoding */
6520
6521       case EOP_STR2B64:
6522         {
6523         uschar *encstr = auth_b64encode(sub, Ustrlen(sub));
6524         yield = string_cat(yield, &size, &ptr, encstr, Ustrlen(encstr));
6525         continue;
6526         }
6527
6528       /* strlen returns the length of the string */
6529
6530       case EOP_STRLEN:
6531         {
6532         uschar buff[24];
6533         (void)sprintf(CS buff, "%d", Ustrlen(sub));
6534         yield = string_cat(yield, &size, &ptr, buff, Ustrlen(buff));
6535         continue;
6536         }
6537
6538       /* length_n or l_n takes just the first n characters or the whole string,
6539       whichever is the shorter;
6540
6541       substr_m_n, and s_m_n take n characters from offset m; negative m take
6542       from the end; l_n is synonymous with s_0_n. If n is omitted in substr it
6543       takes the rest, either to the right or to the left.
6544
6545       hash_n or h_n makes a hash of length n from the string, yielding n
6546       characters from the set a-z; hash_n_m makes a hash of length n, but
6547       uses m characters from the set a-zA-Z0-9.
6548
6549       nhash_n returns a single number between 0 and n-1 (in text form), while
6550       nhash_n_m returns a div/mod hash as two numbers "a/b". The first lies
6551       between 0 and n-1 and the second between 0 and m-1. */
6552
6553       case EOP_LENGTH:
6554       case EOP_L:
6555       case EOP_SUBSTR:
6556       case EOP_S:
6557       case EOP_HASH:
6558       case EOP_H:
6559       case EOP_NHASH:
6560       case EOP_NH:
6561         {
6562         int sign = 1;
6563         int value1 = 0;
6564         int value2 = -1;
6565         int *pn;
6566         int len;
6567         uschar *ret;
6568
6569         if (arg == NULL)
6570           {
6571           expand_string_message = string_sprintf("missing values after %s",
6572             name);
6573           goto EXPAND_FAILED;
6574           }
6575
6576         /* "length" has only one argument, effectively being synonymous with
6577         substr_0_n. */
6578
6579         if (c == EOP_LENGTH || c == EOP_L)
6580           {
6581           pn = &value2;
6582           value2 = 0;
6583           }
6584
6585         /* The others have one or two arguments; for "substr" the first may be
6586         negative. The second being negative means "not supplied". */
6587
6588         else
6589           {
6590           pn = &value1;
6591           if (name[0] == 's' && *arg == '-') { sign = -1; arg++; }
6592           }
6593
6594         /* Read up to two numbers, separated by underscores */
6595
6596         ret = arg;
6597         while (*arg != 0)
6598           {
6599           if (arg != ret && *arg == '_' && pn == &value1)
6600             {
6601             pn = &value2;
6602             value2 = 0;
6603             if (arg[1] != 0) arg++;
6604             }
6605           else if (!isdigit(*arg))
6606             {
6607             expand_string_message =
6608               string_sprintf("non-digit after underscore in \"%s\"", name);
6609             goto EXPAND_FAILED;
6610             }
6611           else *pn = (*pn)*10 + *arg++ - '0';
6612           }
6613         value1 *= sign;
6614
6615         /* Perform the required operation */
6616
6617         ret =
6618           (c == EOP_HASH || c == EOP_H)?
6619              compute_hash(sub, value1, value2, &len) :
6620           (c == EOP_NHASH || c == EOP_NH)?
6621              compute_nhash(sub, value1, value2, &len) :
6622              extract_substr(sub, value1, value2, &len);
6623
6624         if (ret == NULL) goto EXPAND_FAILED;
6625         yield = string_cat(yield, &size, &ptr, ret, len);
6626         continue;
6627         }
6628
6629       /* Stat a path */
6630
6631       case EOP_STAT:
6632         {
6633         uschar *s;
6634         uschar smode[12];
6635         uschar **modetable[3];
6636         int i;
6637         mode_t mode;
6638         struct stat st;
6639
6640         if ((expand_forbid & RDO_EXISTS) != 0)
6641           {
6642           expand_string_message = US"Use of the stat() expansion is not permitted";
6643           goto EXPAND_FAILED;
6644           }
6645
6646         if (stat(CS sub, &st) < 0)
6647           {
6648           expand_string_message = string_sprintf("stat(%s) failed: %s",
6649             sub, strerror(errno));
6650           goto EXPAND_FAILED;
6651           }
6652         mode = st.st_mode;
6653         switch (mode & S_IFMT)
6654           {
6655           case S_IFIFO: smode[0] = 'p'; break;
6656           case S_IFCHR: smode[0] = 'c'; break;
6657           case S_IFDIR: smode[0] = 'd'; break;
6658           case S_IFBLK: smode[0] = 'b'; break;
6659           case S_IFREG: smode[0] = '-'; break;
6660           default: smode[0] = '?'; break;
6661           }
6662
6663         modetable[0] = ((mode & 01000) == 0)? mtable_normal : mtable_sticky;
6664         modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid;
6665         modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid;
6666
6667         for (i = 0; i < 3; i++)
6668           {
6669           memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3);
6670           mode >>= 3;
6671           }
6672
6673         smode[10] = 0;
6674         s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld "
6675           "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld",
6676           (long)(st.st_mode & 077777), smode, (long)st.st_ino,
6677           (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid,
6678           (long)st.st_gid, st.st_size, (long)st.st_atime,
6679           (long)st.st_mtime, (long)st.st_ctime);
6680         yield = string_cat(yield, &size, &ptr, s, Ustrlen(s));
6681         continue;
6682         }
6683
6684       /* vaguely random number less than N */
6685
6686       case EOP_RANDINT:
6687         {
6688         int_eximarith_t max;
6689         uschar *s;
6690
6691         max = expanded_string_integer(sub, TRUE);
6692         if (expand_string_message != NULL)
6693           goto EXPAND_FAILED;
6694         s = string_sprintf("%d", vaguely_random_number((int)max));
6695         yield = string_cat(yield, &size, &ptr, s, Ustrlen(s));
6696         continue;
6697         }
6698
6699       /* Reverse IP, including IPv6 to dotted-nibble */
6700
6701       case EOP_REVERSE_IP:
6702         {
6703         int family, maskptr;
6704         uschar reversed[128];
6705
6706         family = string_is_ip_address(sub, &maskptr);
6707         if (family == 0)
6708           {
6709           expand_string_message = string_sprintf(
6710               "reverse_ip() not given an IP address [%s]", sub);
6711           goto EXPAND_FAILED;
6712           }
6713         invert_address(reversed, sub);
6714         yield = string_cat(yield, &size, &ptr, reversed, Ustrlen(reversed));
6715         continue;
6716         }
6717
6718       /* Unknown operator */
6719
6720       default:
6721       expand_string_message =
6722         string_sprintf("unknown expansion operator \"%s\"", name);
6723       goto EXPAND_FAILED;
6724       }
6725     }
6726
6727   /* Handle a plain name. If this is the first thing in the expansion, release
6728   the pre-allocated buffer. If the result data is known to be in a new buffer,
6729   newsize will be set to the size of that buffer, and we can just point at that
6730   store instead of copying. Many expansion strings contain just one reference,
6731   so this is a useful optimization, especially for humungous headers
6732   ($message_headers). */
6733                                                 /*{*/
6734   if (*s++ == '}')
6735     {
6736     int len;
6737     int newsize = 0;
6738     if (ptr == 0)
6739       {
6740       if (resetok) store_reset(yield);
6741       yield = NULL;
6742       size = 0;
6743       }
6744     value = find_variable(name, FALSE, skipping, &newsize);
6745     if (value == NULL)
6746       {
6747       expand_string_message =
6748         string_sprintf("unknown variable in \"${%s}\"", name);
6749       check_variable_error_message(name);
6750       goto EXPAND_FAILED;
6751       }
6752     len = Ustrlen(value);
6753     if (yield == NULL && newsize != 0)
6754       {
6755       yield = value;
6756       size = newsize;
6757       ptr = len;
6758       }
6759     else yield = string_cat(yield, &size, &ptr, value, len);
6760     continue;
6761     }
6762
6763   /* Else there's something wrong */
6764
6765   expand_string_message =
6766     string_sprintf("\"${%s\" is not a known operator (or a } is missing "
6767     "in a variable reference)", name);
6768   goto EXPAND_FAILED;
6769   }
6770
6771 /* If we hit the end of the string when ket_ends is set, there is a missing
6772 terminating brace. */
6773
6774 if (ket_ends && *s == 0)
6775   {
6776   expand_string_message = malformed_header?
6777     US"missing } at end of string - could be header name not terminated by colon"
6778     :
6779     US"missing } at end of string";
6780   goto EXPAND_FAILED;
6781   }
6782
6783 /* Expansion succeeded; yield may still be NULL here if nothing was actually
6784 added to the string. If so, set up an empty string. Add a terminating zero. If
6785 left != NULL, return a pointer to the terminator. */
6786
6787 if (yield == NULL) yield = store_get(1);
6788 yield[ptr] = 0;
6789 if (left != NULL) *left = s;
6790
6791 /* Any stacking store that was used above the final string is no longer needed.
6792 In many cases the final string will be the first one that was got and so there
6793 will be optimal store usage. */
6794
6795 if (resetok) store_reset(yield + ptr + 1);
6796 else if (resetok_p) *resetok_p = FALSE;
6797
6798 DEBUG(D_expand)
6799   {
6800   debug_printf("expanding: %.*s\n   result: %s\n", (int)(s - string), string,
6801     yield);
6802   if (skipping) debug_printf("skipping: result is not used\n");
6803   }
6804 return yield;
6805
6806 /* This is the failure exit: easiest to program with a goto. We still need
6807 to update the pointer to the terminator, for cases of nested calls with "fail".
6808 */
6809
6810 EXPAND_FAILED_CURLY:
6811 expand_string_message = malformed_header?
6812   US"missing or misplaced { or } - could be header name not terminated by colon"
6813   :
6814   US"missing or misplaced { or }";
6815
6816 /* At one point, Exim reset the store to yield (if yield was not NULL), but
6817 that is a bad idea, because expand_string_message is in dynamic store. */
6818
6819 EXPAND_FAILED:
6820 if (left != NULL) *left = s;
6821 DEBUG(D_expand)
6822   {
6823   debug_printf("failed to expand: %s\n", string);
6824   debug_printf("   error message: %s\n", expand_string_message);
6825   if (expand_string_forcedfail) debug_printf("failure was forced\n");
6826   }
6827 if (resetok_p) *resetok_p = resetok;
6828 return NULL;
6829 }
6830
6831
6832 /* This is the external function call. Do a quick check for any expansion
6833 metacharacters, and if there are none, just return the input string.
6834
6835 Argument: the string to be expanded
6836 Returns:  the expanded string, or NULL if expansion failed; if failure was
6837           due to a lookup deferring, search_find_defer will be TRUE
6838 */
6839
6840 uschar *
6841 expand_string(uschar *string)
6842 {
6843 search_find_defer = FALSE;
6844 malformed_header = FALSE;
6845 return (Ustrpbrk(string, "$\\") == NULL)? string :
6846   expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL);
6847 }
6848
6849
6850
6851 /*************************************************
6852 *              Expand and copy                   *
6853 *************************************************/
6854
6855 /* Now and again we want to expand a string and be sure that the result is in a
6856 new bit of store. This function does that.
6857
6858 Argument: the string to be expanded
6859 Returns:  the expanded string, always in a new bit of store, or NULL
6860 */
6861
6862 uschar *
6863 expand_string_copy(uschar *string)
6864 {
6865 uschar *yield = expand_string(string);
6866 if (yield == string) yield = string_copy(string);
6867 return yield;
6868 }
6869
6870
6871
6872 /*************************************************
6873 *        Expand and interpret as an integer      *
6874 *************************************************/
6875
6876 /* Expand a string, and convert the result into an integer.
6877
6878 Arguments:
6879   string  the string to be expanded
6880   isplus  TRUE if a non-negative number is expected
6881
6882 Returns:  the integer value, or
6883           -1 for an expansion error               ) in both cases, message in
6884           -2 for an integer interpretation error  ) expand_string_message
6885           expand_string_message is set NULL for an OK integer
6886 */
6887
6888 int_eximarith_t
6889 expand_string_integer(uschar *string, BOOL isplus)
6890 {
6891 return expanded_string_integer(expand_string(string), isplus);
6892 }
6893
6894
6895 /*************************************************
6896  *         Interpret string as an integer        *
6897  *************************************************/
6898
6899 /* Convert a string (that has already been expanded) into an integer.
6900
6901 This function is used inside the expansion code.
6902
6903 Arguments:
6904   s       the string to be expanded
6905   isplus  TRUE if a non-negative number is expected
6906
6907 Returns:  the integer value, or
6908           -1 if string is NULL (which implies an expansion error)
6909           -2 for an integer interpretation error
6910           expand_string_message is set NULL for an OK integer
6911 */
6912
6913 static int_eximarith_t
6914 expanded_string_integer(uschar *s, BOOL isplus)
6915 {
6916 int_eximarith_t value;
6917 uschar *msg = US"invalid integer \"%s\"";
6918 uschar *endptr;
6919
6920 /* If expansion failed, expand_string_message will be set. */
6921
6922 if (s == NULL) return -1;
6923
6924 /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno
6925 to ERANGE. When there isn't an overflow, errno is not changed, at least on some
6926 systems, so we set it zero ourselves. */
6927
6928 errno = 0;
6929 expand_string_message = NULL;               /* Indicates no error */
6930
6931 /* Before Exim 4.64, strings consisting entirely of whitespace compared
6932 equal to 0.  Unfortunately, people actually relied upon that, so preserve
6933 the behaviour explicitly.  Stripping leading whitespace is a harmless
6934 noop change since strtol skips it anyway (provided that there is a number
6935 to find at all). */
6936 if (isspace(*s))
6937   {
6938   while (isspace(*s)) ++s;
6939   if (*s == '\0')
6940     {
6941       DEBUG(D_expand)
6942        debug_printf("treating blank string as number 0\n");
6943       return 0;
6944     }
6945   }
6946
6947 value = strtoll(CS s, CSS &endptr, 10);
6948
6949 if (endptr == s)
6950   {
6951   msg = US"integer expected but \"%s\" found";
6952   }
6953 else if (value < 0 && isplus)
6954   {
6955   msg = US"non-negative integer expected but \"%s\" found";
6956   }
6957 else
6958   {
6959   switch (tolower(*endptr))
6960     {
6961     default:
6962       break;
6963     case 'k':
6964       if (value > EXIM_ARITH_MAX/1024 || value < EXIM_ARITH_MIN/1024) errno = ERANGE;
6965       else value *= 1024;
6966       endptr++;
6967       break;
6968     case 'm':
6969       if (value > EXIM_ARITH_MAX/(1024*1024) || value < EXIM_ARITH_MIN/(1024*1024)) errno = ERANGE;
6970       else value *= 1024*1024;
6971       endptr++;
6972       break;
6973     case 'g':
6974       if (value > EXIM_ARITH_MAX/(1024*1024*1024) || value < EXIM_ARITH_MIN/(1024*1024*1024)) errno = ERANGE;
6975       else value *= 1024*1024*1024;
6976       endptr++;
6977       break;
6978     }
6979   if (errno == ERANGE)
6980     msg = US"absolute value of integer \"%s\" is too large (overflow)";
6981   else
6982     {
6983     while (isspace(*endptr)) endptr++;
6984     if (*endptr == 0) return value;
6985     }
6986   }
6987
6988 expand_string_message = string_sprintf(CS msg, s);
6989 return -2;
6990 }
6991
6992
6993 /*************************************************
6994 **************************************************
6995 *             Stand-alone test program           *
6996 **************************************************
6997 *************************************************/
6998
6999 #ifdef STAND_ALONE
7000
7001
7002 BOOL
7003 regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup)
7004 {
7005 int ovector[3*(EXPAND_MAXN+1)];
7006 int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options,
7007   ovector, sizeof(ovector)/sizeof(int));
7008 BOOL yield = n >= 0;
7009 if (n == 0) n = EXPAND_MAXN + 1;
7010 if (yield)
7011   {
7012   int nn;
7013   expand_nmax = (setup < 0)? 0 : setup + 1;
7014   for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2)
7015     {
7016     expand_nstring[expand_nmax] = subject + ovector[nn];
7017     expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn];
7018     }
7019   expand_nmax--;
7020   }
7021 return yield;
7022 }
7023
7024
7025 int main(int argc, uschar **argv)
7026 {
7027 int i;
7028 uschar buffer[1024];
7029
7030 debug_selector = D_v;
7031 debug_file = stderr;
7032 debug_fd = fileno(debug_file);
7033 big_buffer = malloc(big_buffer_size);
7034
7035 for (i = 1; i < argc; i++)
7036   {
7037   if (argv[i][0] == '+')
7038     {
7039     debug_trace_memory = 2;
7040     argv[i]++;
7041     }
7042   if (isdigit(argv[i][0]))
7043     debug_selector = Ustrtol(argv[i], NULL, 0);
7044   else
7045     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
7046         Ustrlen(argv[i]))
7047       {
7048       #ifdef LOOKUP_LDAP
7049       eldap_default_servers = argv[i];
7050       #endif
7051       #ifdef LOOKUP_MYSQL
7052       mysql_servers = argv[i];
7053       #endif
7054       #ifdef LOOKUP_PGSQL
7055       pgsql_servers = argv[i];
7056       #endif
7057       #ifdef EXPERIMENTAL_REDIS
7058       redis_servers = argv[i];
7059       #endif
7060       }
7061   #ifdef EXIM_PERL
7062   else opt_perl_startup = argv[i];
7063   #endif
7064   }
7065
7066 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
7067
7068 expand_nstring[1] = US"string 1....";
7069 expand_nlength[1] = 8;
7070 expand_nmax = 1;
7071
7072 #ifdef EXIM_PERL
7073 if (opt_perl_startup != NULL)
7074   {
7075   uschar *errstr;
7076   printf("Starting Perl interpreter\n");
7077   errstr = init_perl(opt_perl_startup);
7078   if (errstr != NULL)
7079     {
7080     printf("** error in perl_startup code: %s\n", errstr);
7081     return EXIT_FAILURE;
7082     }
7083   }
7084 #endif /* EXIM_PERL */
7085
7086 while (fgets(buffer, sizeof(buffer), stdin) != NULL)
7087   {
7088   void *reset_point = store_get(0);
7089   uschar *yield = expand_string(buffer);
7090   if (yield != NULL)
7091     {
7092     printf("%s\n", yield);
7093     store_reset(reset_point);
7094     }
7095   else
7096     {
7097     if (search_find_defer) printf("search_find deferred\n");
7098     printf("Failed: %s\n", expand_string_message);
7099     if (expand_string_forcedfail) printf("Forced failure\n");
7100     printf("\n");
7101     }
7102   }
7103
7104 search_tidyup();
7105
7106 return 0;
7107 }
7108
7109 #endif
7110
7111 /* vi: aw ai sw=2
7112 */
7113 /* End of expand.c */