X-Git-Url: https://vcs.fsf.org/?a=blobdiff_plain;f=src%2Fsrc%2Fexpand.c;h=cfde23610acafb6d200cfd3934179e802f45ac67;hb=5dc309a45b3f266afbe1b8ccc9a066b0f76650a3;hp=c8d1ffc7d88c38c4087171b15d3655e097292bd0;hpb=c2f669a4994192344613569e198c7b503d46d45e;p=exim.git

diff --git a/src/src/expand.c b/src/src/expand.c
index c8d1ffc7d..cfde23610 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -199,12 +199,15 @@ enum {
 static uschar *op_table_main[] = {
   US"address",
   US"addresses",
+  US"base32",
+  US"base32d",
   US"base62",
   US"base62d",
   US"base64",
   US"base64d",
   US"domain",
   US"escape",
+  US"escape8bit",
   US"eval",
   US"eval10",
   US"expand",
@@ -231,6 +234,7 @@ static uschar *op_table_main[] = {
   US"s",
   US"sha1",
   US"sha256",
+  US"sha3",
   US"stat",
   US"str2b64",
   US"strlen",
@@ -241,12 +245,15 @@ static uschar *op_table_main[] = {
 enum {
   EOP_ADDRESS =  nelem(op_table_underscore),
   EOP_ADDRESSES,
+  EOP_BASE32,
+  EOP_BASE32D,
   EOP_BASE62,
   EOP_BASE62D,
   EOP_BASE64,
   EOP_BASE64D,
   EOP_DOMAIN,
   EOP_ESCAPE,
+  EOP_ESCAPE8BIT,
   EOP_EVAL,
   EOP_EVAL10,
   EOP_EXPAND,
@@ -273,6 +280,7 @@ enum {
   EOP_S,
   EOP_SHA1,
   EOP_SHA256,
+  EOP_SHA3,
   EOP_STAT,
   EOP_STR2B64,
   EOP_STRLEN,
@@ -625,6 +633,7 @@ static var_entry var_table[] = {
   { "prvscheck_result",    vtype_stringptr,   &prvscheck_result },
   { "qualify_domain",      vtype_stringptr,   &qualify_domain_sender },
   { "qualify_recipient",   vtype_stringptr,   &qualify_domain_recipient },
+  { "queue_name",          vtype_stringptr,   &queue_name },
   { "rcpt_count",          vtype_int,         &rcpt_count },
   { "rcpt_defer_count",    vtype_int,         &rcpt_defer_count },
   { "rcpt_fail_count",     vtype_int,         &rcpt_fail_count },
@@ -835,6 +844,9 @@ static int utf8_table2[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
     }
 
 
+
+static uschar * base32_chars = US"abcdefghijklmnopqrstuvwxyz234567";
+
 /*************************************************
 *           Binary chop search on a table        *
 *************************************************/
@@ -1991,12 +2003,17 @@ for (i = 0; i < n; i++)
   {
   if (*s != '{')
     {
-    if (i < m) return 1;
+    if (i < m)
+      {
+      expand_string_message = string_sprintf("Not enough arguments for '%s' "
+	"(min is %d)", name, m);
+      return 1;
+      }
     sub[i] = NULL;
     break;
     }
-  sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok);
-  if (sub[i] == NULL) return 3;
+  if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, resetok)))
+    return 3;
   if (*s++ != '}') return 1;
   while (isspace(*s)) s++;
   }
@@ -2004,10 +2021,11 @@ if (check_end && *s++ != '}')
   {
   if (s[-1] == '{')
     {
-    expand_string_message = string_sprintf("Too many arguments for \"%s\" "
+    expand_string_message = string_sprintf("Too many arguments for '%s' "
       "(max is %d)", name, n);
     return 2;
     }
+  expand_string_message = string_sprintf("missing '}' after '%s'", name);
   return 1;
   }
 
@@ -2500,7 +2518,6 @@ switch(cond_type)
     checking for them individually. */
 
     if (!isalpha(name[0]) && yield != NULL)
-      {
       if (sub[i][0] == 0)
         {
         num[i] = 0;
@@ -2512,7 +2529,6 @@ switch(cond_type)
         num[i] = expanded_string_integer(sub[i], FALSE);
         if (expand_string_message != NULL) return NULL;
         }
-      }
     }
 
   /* Result not required */
@@ -2680,7 +2696,7 @@ switch(cond_type)
       uschar digest[16];
 
       md5_start(&base);
-      md5_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
+      md5_end(&base, sub[0], Ustrlen(sub[0]), digest);
 
       /* If the length that we are comparing against is 24, the MD5 digest
       is expressed as a base64 string. This is the way LDAP does it. However,
@@ -2689,7 +2705,7 @@ switch(cond_type)
 
       if (sublen == 24)
         {
-        uschar *coded = b64encode((uschar *)digest, 16);
+        uschar *coded = b64encode(digest, 16);
         DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n"
           "  subject=%s\n  crypted=%s\n", coded, sub[1]+5);
         tempcond = (Ustrcmp(coded, sub[1]+5) == 0);
@@ -2715,11 +2731,11 @@ switch(cond_type)
     else if (strncmpic(sub[1], US"{sha1}", 6) == 0)
       {
       int sublen = Ustrlen(sub[1]+6);
-      sha1 base;
+      hctx h;
       uschar digest[20];
 
-      sha1_start(&base);
-      sha1_end(&base, (uschar *)sub[0], Ustrlen(sub[0]), digest);
+      sha1_start(&h);
+      sha1_end(&h, sub[0], Ustrlen(sub[0]), digest);
 
       /* If the length that we are comparing against is 28, assume the SHA1
       digest is expressed as a base64 string. If the length is 40, assume a
@@ -2727,7 +2743,7 @@ switch(cond_type)
 
       if (sublen == 28)
         {
-        uschar *coded = b64encode((uschar *)digest, 20);
+        uschar *coded = b64encode(digest, 20);
         DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n"
           "  subject=%s\n  crypted=%s\n", coded, sub[1]+6);
         tempcond = (Ustrcmp(coded, sub[1]+6) == 0);
@@ -3034,6 +3050,8 @@ switch(cond_type)
        "value \"%s\"", t);
       return NULL;
       }
+    DEBUG(D_expand) debug_printf("%s: condition evaluated to %s\n", ourname,
+        boolvalue? "true":"false");
     if (yield != NULL) *yield = (boolvalue == testfor);
     return s;
     }
@@ -3165,6 +3183,7 @@ process_yesno(BOOL skipping, BOOL yes, uschar *save_lookup, const uschar **sptr,
 int rc = 0;
 const uschar *s = *sptr;    /* Local value */
 uschar *sub1, *sub2;
+const uschar * errwhere;
 
 /* If there are no following strings, we substitute the contents of $value for
 lookups and for extractions in the success case. For the ${if item, the string
@@ -3174,23 +3193,28 @@ items. */
 while (isspace(*s)) s++;
 if (*s == '}')
   {
-  if (type[0] == 'i')
-    {
-    if (yes) *yieldptr = string_catn(*yieldptr, sizeptr, ptrptr, US"true", 4);
-    }
-  else
-    {
-    if (yes && lookup_value)
-      *yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value);
-    lookup_value = save_lookup;
-    }
+  if (!skipping)
+    if (type[0] == 'i')
+      {
+      if (yes) *yieldptr = string_catn(*yieldptr, sizeptr, ptrptr, US"true", 4);
+      }
+    else
+      {
+      if (yes && lookup_value)
+	*yieldptr = string_cat(*yieldptr, sizeptr, ptrptr, lookup_value);
+      lookup_value = save_lookup;
+      }
   s++;
   goto RETURN;
   }
 
 /* The first following string must be braced. */
 
-if (*s++ != '{') goto FAILED_CURLY;
+if (*s++ != '{')
+  {
+  errwhere = US"'yes' part did not start with '{'";
+  goto FAILED_CURLY;
+  }
 
 /* Expand the first substring. Forced failures are noticed only if we actually
 want this string. Set skipping in the call in the fail case (this will always
@@ -3199,7 +3223,11 @@ be the case if we were already skipping). */
 sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok);
 if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED;
 expand_string_forcedfail = FALSE;
-if (*s++ != '}') goto FAILED_CURLY;
+if (*s++ != '}')
+  {
+  errwhere = US"'yes' part did not end with '}'";
+  goto FAILED_CURLY;
+  }
 
 /* If we want the first string, add it to the output */
 
@@ -3225,7 +3253,11 @@ if (*s == '{')
   sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok);
   if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED;
   expand_string_forcedfail = FALSE;
-  if (*s++ != '}') goto FAILED_CURLY;
+  if (*s++ != '}')
+    {
+    errwhere = US"'no' part did not start with '{'";
+    goto FAILED_CURLY;
+    }
 
   /* If we want the second string, add it to the output */
 
@@ -3248,7 +3280,11 @@ else if (*s != '}')
     if (!yes && !skipping)
       {
       while (isspace(*s)) s++;
-      if (*s++ != '}') goto FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	errwhere = US"did not close with '}' after forcedfail";
+	goto FAILED_CURLY;
+	}
       expand_string_message =
         string_sprintf("\"%s\" failed and \"fail\" requested", type);
       expand_string_forcedfail = TRUE;
@@ -3266,23 +3302,30 @@ else if (*s != '}')
 /* All we have to do now is to check on the final closing brace. */
 
 while (isspace(*s)) s++;
-if (*s++ == '}') goto RETURN;
-
-/* Get here if there is a bracketing failure */
-
-FAILED_CURLY:
-rc++;
-
-/* Get here for other failures */
-
-FAILED:
-rc++;
+if (*s++ != '}')
+  {
+  errwhere = US"did not close with '}'";
+  goto FAILED_CURLY;
+  }
 
-/* Update the input pointer value before returning */
 
 RETURN:
+/* Update the input pointer value before returning */
 *sptr = s;
 return rc;
+
+FAILED_CURLY:
+  /* Get here if there is a bracketing failure */
+  expand_string_message = string_sprintf(
+    "curly-bracket problem in conditional yes/no parsing: %s\n"
+    " remaining string is '%s'", errwhere, --s);
+  rc = 2;
+  goto RETURN;
+
+FAILED:
+  /* Get here for other failures */
+  rc = 1;
+  goto RETURN;
 }
 
 
@@ -3308,7 +3351,7 @@ chash_start(int type, void *base)
 if (type == HMAC_MD5)
   md5_start((md5 *)base);
 else
-  sha1_start((sha1 *)base);
+  sha1_start((hctx *)base);
 }
 
 static void
@@ -3317,7 +3360,7 @@ chash_mid(int type, void *base, uschar *string)
 if (type == HMAC_MD5)
   md5_mid((md5 *)base, string);
 else
-  sha1_mid((sha1 *)base, string);
+  sha1_mid((hctx *)base, string);
 }
 
 static void
@@ -3326,7 +3369,7 @@ chash_end(int type, void *base, uschar *string, int length, uschar *digest)
 if (type == HMAC_MD5)
   md5_end((md5 *)base, string, length, digest);
 else
-  sha1_end((sha1 *)base, string, length, digest);
+  sha1_end((hctx *)base, string, length, digest);
 }
 
 
@@ -3385,8 +3428,7 @@ prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp)
 {
 uschar *hash_source, *p;
 int size = 0,offset = 0,i;
-sha1 sha1_base;
-void *use_base = &sha1_base;
+hctx h;
 uschar innerhash[20];
 uschar finalhash[20];
 uschar innerkey[64];
@@ -3415,13 +3457,13 @@ for (i = 0; i < Ustrlen(key); i++)
   outerkey[i] ^= key[i];
   }
 
-chash_start(HMAC_SHA1, use_base);
-chash_mid(HMAC_SHA1, use_base, innerkey);
-chash_end(HMAC_SHA1, use_base, hash_source, offset, innerhash);
+chash_start(HMAC_SHA1, &h);
+chash_mid(HMAC_SHA1, &h, innerkey);
+chash_end(HMAC_SHA1, &h, hash_source, offset, innerhash);
 
-chash_start(HMAC_SHA1, use_base);
-chash_mid(HMAC_SHA1, use_base, outerkey);
-chash_end(HMAC_SHA1, use_base, innerhash, 20, finalhash);
+chash_start(HMAC_SHA1, &h);
+chash_mid(HMAC_SHA1, &h, outerkey);
+chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash);
 
 p = finalhash_hex;
 for (i = 0; i < 3; i++)
@@ -3458,7 +3500,6 @@ Returns:       new value of string pointer
 static uschar *
 cat_file(FILE *f, uschar *yield, int *sizep, int *ptrp, uschar *eol)
 {
-int eollen = eol ? Ustrlen(eol) : 0;
 uschar buffer[1024];
 
 while (Ufgets(buffer, sizeof(buffer), f))
@@ -3466,8 +3507,8 @@ while (Ufgets(buffer, sizeof(buffer), f))
   int len = Ustrlen(buffer);
   if (eol && buffer[len-1] == '\n') len--;
   yield = string_catn(yield, sizep, ptrp, buffer, len);
-  if (buffer[len] != 0)
-    yield = string_catn(yield, sizep, ptrp, eol, eollen);
+  if (eol && buffer[len])
+    yield = string_cat(yield, sizep, ptrp, eol);
   }
 
 if (yield) yield[*ptrp] = 0;
@@ -3821,13 +3862,16 @@ expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left,
 {
 int ptr = 0;
 int size = Ustrlen(string)+ 64;
-int item_type;
 uschar *yield = store_get(size);
+int item_type;
 const uschar *s = string;
 uschar *save_expand_nstring[EXPAND_MAXN+1];
 int save_expand_nlength[EXPAND_MAXN+1];
 BOOL resetok = TRUE;
 
+DEBUG(D_expand)
+  debug_printf("%s: %s\n", skipping ? "   scanning" : "considering", string);
+
 expand_string_forcedfail = FALSE;
 expand_string_message = US"";
 
@@ -4068,12 +4112,13 @@ while (*s != 0)
         save_expand_strings(save_expand_nstring, save_expand_nlength);
 
       while (isspace(*s)) s++;
-      next_s = eval_condition(s, &resetok, skipping? NULL : &cond);
+      next_s = eval_condition(s, &resetok, skipping ? NULL : &cond);
       if (next_s == NULL) goto EXPAND_FAILED;  /* message already set */
 
       DEBUG(D_expand)
-        debug_printf("condition: %.*s\n   result: %s\n", (int)(next_s - s), s,
-          cond? "true" : "false");
+        debug_printf("  condition: %.*s\n     result: %s\n",
+	  (int)(next_s - s), s,
+          cond ? "true" : "false");
 
       s = next_s;
 
@@ -4131,11 +4176,13 @@ while (*s != 0)
 	goto EXPAND_FAILED;
 	}
 
-      if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
-			  sub_arg[1][0], sub_arg[2], &expand_string_message)))
-	goto EXPAND_FAILED;
       if (!skipping)
+	{
+	if (!(encoded = imap_utf7_encode(sub_arg[0], headers_charset,
+			    sub_arg[1][0], sub_arg[2], &expand_string_message)))
+	  goto EXPAND_FAILED;
 	yield = string_cat(yield, &size, &ptr, encoded);
+	}
       continue;
       }
 #endif
@@ -4171,8 +4218,12 @@ while (*s != 0)
       if (*s == '{')					/*}*/
         {
         key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
-        if (key == NULL) goto EXPAND_FAILED;		/*{*/
-        if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+        if (!key) goto EXPAND_FAILED;			/*{{*/
+        if (*s++ != '}')
+	  {
+	  expand_string_message = US"missing '}' after lookup key";
+	  goto EXPAND_FAILED_CURLY;
+	  }
         while (isspace(*s)) s++;
         }
       else key = NULL;
@@ -4235,10 +4286,18 @@ while (*s != 0)
       queries that also require a file name (e.g. sqlite), the file name comes
       first. */
 
-      if (*s != '{') goto EXPAND_FAILED_CURLY;
+      if (*s != '{')
+        {
+	expand_string_message = US"missing '{' for lookup file-or-query arg";
+	goto EXPAND_FAILED_CURLY;
+	}
       filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
       if (filename == NULL) goto EXPAND_FAILED;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing lookup file-or-query arg";
+	goto EXPAND_FAILED_CURLY;
+	}
       while (isspace(*s)) s++;
 
       /* If this isn't a single-key+file lookup, re-arrange the variables
@@ -4246,15 +4305,13 @@ while (*s != 0)
       there is just a "key", and no file name. For the special query-style +
       file types, the query (i.e. "key") starts with a file name. */
 
-      if (key == NULL)
+      if (!key)
         {
         while (isspace(*filename)) filename++;
         key = filename;
 
         if (mac_islookup(stype, lookup_querystyle))
-          {
           filename = NULL;
-          }
         else
           {
           if (*filename != '/')
@@ -4603,7 +4660,6 @@ while (*s != 0)
         prvscheck_keynum = NULL;
         }
       else
-        {
         /* Does not look like a prvs encoded address, return the empty string.
            We need to make sure all subs are expanded first, so as to skip over
            the entire item. */
@@ -4614,7 +4670,6 @@ while (*s != 0)
           case 2:
           case 3: goto EXPAND_FAILED;
           }
-        }
 
       continue;
       }
@@ -4841,10 +4896,20 @@ while (*s != 0)
         {
         if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL)
           goto EXPAND_FAILED;
-        if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+        if (*s++ != '}')
+	  {
+	  expand_string_message = US"missing '}' closing failstring for readsocket";
+	  goto EXPAND_FAILED_CURLY;
+	  }
         while (isspace(*s)) s++;
         }
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+
+    readsock_done:
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing readsocket";
+	goto EXPAND_FAILED_CURLY;
+	}
       continue;
 
       /* Come here on failure to create socket, connect socket, write to the
@@ -4857,10 +4922,13 @@ while (*s != 0)
       if (!(arg = expand_string_internal(s+1, TRUE, &s, FALSE, TRUE, &resetok)))
         goto EXPAND_FAILED;
       yield = string_cat(yield, &size, &ptr, arg);
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing failstring for readsocket";
+	goto EXPAND_FAILED_CURLY;
+	}
       while (isspace(*s)) s++;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
-      continue;
+      goto readsock_done;
       }
 
     /* Handle "run" to execute a program. */
@@ -4881,16 +4949,25 @@ while (*s != 0)
         }
 
       while (isspace(*s)) s++;
-      if (*s != '{') goto EXPAND_FAILED_CURLY;
+      if (*s != '{')
+        {
+	expand_string_message = US"missing '{' for command arg of run";
+	goto EXPAND_FAILED_CURLY;
+	}
       arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
       if (arg == NULL) goto EXPAND_FAILED;
       while (isspace(*s)) s++;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing command arg of run";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       if (skipping)   /* Just pretend it worked when we're skipping */
-        {
+	{
         runrc = 0;
-        }
+	lookup_value = NULL;
+	}
       else
         {
         if (!transport_set_up_command(&argv,    /* anchor for arg list */
@@ -4932,9 +5009,9 @@ while (*s != 0)
         return code for serious disasters. Simple non-zero returns are passed on.
         */
 
-        if (sigalrm_seen == TRUE || (runrc = child_close(pid, 30)) < 0)
+        if (sigalrm_seen || (runrc = child_close(pid, 30)) < 0)
           {
-          if (sigalrm_seen == TRUE || runrc == -256)
+          if (sigalrm_seen || runrc == -256)
             {
             expand_string_message = string_sprintf("command timed out");
             killpg(pid, SIGKILL);       /* Kill the whole process group */
@@ -5082,7 +5159,7 @@ while (*s != 0)
       {
       uschar *sub[3];
       md5 md5_base;
-      sha1 sha1_base;
+      hctx sha1_ctx;
       void *use_base;
       int type, i;
       int hashlen;      /* Number of octets for the hash algorithm's output */
@@ -5104,79 +5181,81 @@ while (*s != 0)
         case 3: goto EXPAND_FAILED;
         }
 
-      if (Ustrcmp(sub[0], "md5") == 0)
-        {
-        type = HMAC_MD5;
-        use_base = &md5_base;
-        hashlen = 16;
-        hashblocklen = 64;
-        }
-      else if (Ustrcmp(sub[0], "sha1") == 0)
-        {
-        type = HMAC_SHA1;
-        use_base = &sha1_base;
-        hashlen = 20;
-        hashblocklen = 64;
-        }
-      else
-        {
-        expand_string_message =
-          string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
-        goto EXPAND_FAILED;
-        }
+      if (!skipping)
+	{
+	if (Ustrcmp(sub[0], "md5") == 0)
+	  {
+	  type = HMAC_MD5;
+	  use_base = &md5_base;
+	  hashlen = 16;
+	  hashblocklen = 64;
+	  }
+	else if (Ustrcmp(sub[0], "sha1") == 0)
+	  {
+	  type = HMAC_SHA1;
+	  use_base = &sha1_ctx;
+	  hashlen = 20;
+	  hashblocklen = 64;
+	  }
+	else
+	  {
+	  expand_string_message =
+	    string_sprintf("hmac algorithm \"%s\" is not recognised", sub[0]);
+	  goto EXPAND_FAILED;
+	  }
 
-      keyptr = sub[1];
-      keylen = Ustrlen(keyptr);
+	keyptr = sub[1];
+	keylen = Ustrlen(keyptr);
 
-      /* If the key is longer than the hash block length, then hash the key
-      first */
+	/* If the key is longer than the hash block length, then hash the key
+	first */
 
-      if (keylen > hashblocklen)
-        {
-        chash_start(type, use_base);
-        chash_end(type, use_base, keyptr, keylen, keyhash);
-        keyptr = keyhash;
-        keylen = hashlen;
-        }
+	if (keylen > hashblocklen)
+	  {
+	  chash_start(type, use_base);
+	  chash_end(type, use_base, keyptr, keylen, keyhash);
+	  keyptr = keyhash;
+	  keylen = hashlen;
+	  }
 
-      /* Now make the inner and outer key values */
+	/* Now make the inner and outer key values */
 
-      memset(innerkey, 0x36, hashblocklen);
-      memset(outerkey, 0x5c, hashblocklen);
+	memset(innerkey, 0x36, hashblocklen);
+	memset(outerkey, 0x5c, hashblocklen);
 
-      for (i = 0; i < keylen; i++)
-        {
-        innerkey[i] ^= keyptr[i];
-        outerkey[i] ^= keyptr[i];
-        }
+	for (i = 0; i < keylen; i++)
+	  {
+	  innerkey[i] ^= keyptr[i];
+	  outerkey[i] ^= keyptr[i];
+	  }
 
-      /* Now do the hashes */
+	/* Now do the hashes */
 
-      chash_start(type, use_base);
-      chash_mid(type, use_base, innerkey);
-      chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
+	chash_start(type, use_base);
+	chash_mid(type, use_base, innerkey);
+	chash_end(type, use_base, sub[2], Ustrlen(sub[2]), innerhash);
 
-      chash_start(type, use_base);
-      chash_mid(type, use_base, outerkey);
-      chash_end(type, use_base, innerhash, hashlen, finalhash);
+	chash_start(type, use_base);
+	chash_mid(type, use_base, outerkey);
+	chash_end(type, use_base, innerhash, hashlen, finalhash);
 
-      /* Encode the final hash as a hex string */
+	/* Encode the final hash as a hex string */
 
-      p = finalhash_hex;
-      for (i = 0; i < hashlen; i++)
-        {
-        *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
-        *p++ = hex_digits[finalhash[i] & 0x0f];
-        }
+	p = finalhash_hex;
+	for (i = 0; i < hashlen; i++)
+	  {
+	  *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4];
+	  *p++ = hex_digits[finalhash[i] & 0x0f];
+	  }
 
-      DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%.*s)=%.*s\n", sub[0],
-        (int)keylen, keyptr, Ustrlen(sub[2]), sub[2], hashlen*2, finalhash_hex);
+	DEBUG(D_any) debug_printf("HMAC[%s](%.*s,%s)=%.*s\n",
+	  sub[0], (int)keylen, keyptr, sub[2], hashlen*2, finalhash_hex);
 
-      yield = string_catn(yield, &size, &ptr, finalhash_hex, hashlen*2);
+	yield = string_catn(yield, &size, &ptr, finalhash_hex, hashlen*2);
+	}
+      continue;
       }
 
-    continue;
-
     /* Handle global substitution for "sg" - like Perl's s/xxx/yyy/g operator.
     We have to save the numerical variables and restore them afterwards. */
 
@@ -5297,7 +5376,7 @@ while (*s != 0)
     case EITEM_EXTRACT:
       {
       int i;
-      int j = 2;
+      int j;
       int field_number = 1;
       BOOL field_number_set = FALSE;
       uschar *save_lookup_value = lookup_value;
@@ -5307,30 +5386,49 @@ while (*s != 0)
 
       /* While skipping we cannot rely on the data for expansions being
       available (eg. $item) hence cannot decide on numeric vs. keyed.
-      Just read as many arguments as there are. */
+      Read a maximum of 5 arguments (inclding the yes/no) */
 
       if (skipping)
 	{
         while (isspace(*s)) s++;
-        while (*s == '{')
+        for (j = 5; j > 0 && *s == '{'; j--)
 	  {
           if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))
 	    goto EXPAND_FAILED;					/*{*/
-          if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+          if (*s++ != '}')
+	    {
+	    expand_string_message = US"missing '{' for arg of extract";
+	    goto EXPAND_FAILED_CURLY;
+	    }
+	  while (isspace(*s)) s++;
+	  }
+	if (  Ustrncmp(s, "fail", 4) == 0
+	   && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4])
+	   )
+	  {
+	  s += 4;
 	  while (isspace(*s)) s++;
 	  }
 	if (*s != '}')
+	  {
+	  expand_string_message = US"missing '}' closing extract";
 	  goto EXPAND_FAILED_CURLY;
+	  }
 	}
 
-      else for (i = 0; i < j; i++) /* Read the proper number of arguments */
+      else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */
         {
         while (isspace(*s)) s++;
         if (*s == '{') 						/*}*/
           {
           sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
           if (sub[i] == NULL) goto EXPAND_FAILED;		/*{*/
-          if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+          if (*s++ != '}')
+	    {
+	    expand_string_message = string_sprintf(
+	      "missing '}' closing arg %d of extract", i+1);
+	    goto EXPAND_FAILED_CURLY;
+	    }
 
           /* After removal of leading and trailing white space, the first
           argument must not be empty; if it consists entirely of digits
@@ -5371,7 +5469,12 @@ while (*s != 0)
 	      }
             }
           }
-        else goto EXPAND_FAILED_CURLY;
+        else
+	  {
+	  expand_string_message = string_sprintf(
+	    "missing '{' for arg %d of extract", i+1);
+	  goto EXPAND_FAILED_CURLY;
+	  }
         }
 
       /* Extract either the numbered or the keyed substring into $value. If
@@ -5424,11 +5527,20 @@ while (*s != 0)
         {
         while (isspace(*s)) s++;
         if (*s != '{')					/*}*/
+	  {
+	  expand_string_message = string_sprintf(
+	    "missing '{' for arg %d of listextract", i+1);
 	  goto EXPAND_FAILED_CURLY;
+	  }
 
 	sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
 	if (!sub[i])     goto EXPAND_FAILED;		/*{*/
-	if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+	if (*s++ != '}')
+	  {
+	  expand_string_message = string_sprintf(
+	    "missing '}' closing arg %d of listextract", i+1);
+	  goto EXPAND_FAILED_CURLY;
+	  }
 
 	/* After removal of leading and trailing white space, the first
 	argument must be numeric and nonempty. */
@@ -5511,10 +5623,17 @@ while (*s != 0)
       /* Read the field argument */
       while (isspace(*s)) s++;
       if (*s != '{')					/*}*/
+	{
+	expand_string_message = US"missing '{' for field arg of certextract";
 	goto EXPAND_FAILED_CURLY;
+	}
       sub[0] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
       if (!sub[0])     goto EXPAND_FAILED;		/*{*/
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing field arg of certextract";
+	goto EXPAND_FAILED_CURLY;
+	}
       /* strip spaces fore & aft */
       {
       int len;
@@ -5531,7 +5650,10 @@ while (*s != 0)
       /* inspect the cert argument */
       while (isspace(*s)) s++;
       if (*s != '{')					/*}*/
+	{
+	expand_string_message = US"missing '{' for cert variable arg of certextract";
 	goto EXPAND_FAILED_CURLY;
+	}
       if (*++s != '$')
         {
 	expand_string_message = US"second argument of \"certextract\" must "
@@ -5540,7 +5662,11 @@ while (*s != 0)
 	}
       sub[1] = expand_string_internal(s+1, TRUE, &s, skipping, FALSE, &resetok);
       if (!sub[1])     goto EXPAND_FAILED;		/*{*/
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message = US"missing '}' closing cert variable arg of certextract";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       if (skipping)
 	lookup_value = NULL;
@@ -5584,25 +5710,48 @@ while (*s != 0)
       uschar *save_lookup_value = lookup_value;
 
       while (isspace(*s)) s++;
-      if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '{')
+        {
+	expand_string_message =
+	  string_sprintf("missing '{' for first arg of %s", name);
+	goto EXPAND_FAILED_CURLY;
+	}
 
       list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
       if (list == NULL) goto EXPAND_FAILED;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+	expand_string_message =
+	  string_sprintf("missing '}' closing first arg of %s", name);
+	goto EXPAND_FAILED_CURLY;
+	}
 
       if (item_type == EITEM_REDUCE)
         {
 	uschar * t;
         while (isspace(*s)) s++;
-        if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+        if (*s++ != '{')
+	  {
+	  expand_string_message = US"missing '{' for second arg of reduce";
+	  goto EXPAND_FAILED_CURLY;
+	  }
         t = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
         if (!t) goto EXPAND_FAILED;
         lookup_value = t;
-        if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+        if (*s++ != '}')
+	  {
+	  expand_string_message = US"missing '}' closing second arg of reduce";
+	  goto EXPAND_FAILED_CURLY;
+	  }
         }
 
       while (isspace(*s)) s++;
-      if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '{')
+        {
+	expand_string_message =
+	  string_sprintf("missing '{' for last arg of %s", name);
+	goto EXPAND_FAILED_CURLY;
+	}
 
       expr = s;
 
@@ -5757,28 +5906,52 @@ while (*s != 0)
       uschar *save_iterate_item = iterate_item;
 
       while (isspace(*s)) s++;
-      if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '{')
+        {
+        expand_string_message = US"missing '{' for list arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       srclist = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok);
       if (!srclist) goto EXPAND_FAILED;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+        expand_string_message = US"missing '}' closing list arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       while (isspace(*s)) s++;
-      if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '{')
+        {
+        expand_string_message = US"missing '{' for comparator arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       cmp = expand_string_internal(s, TRUE, &s, skipping, FALSE, &resetok);
       if (!cmp) goto EXPAND_FAILED;
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+        expand_string_message = US"missing '}' closing comparator arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       while (isspace(*s)) s++;
-      if (*s++ != '{') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '{')
+        {
+        expand_string_message = US"missing '{' for extractor arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       xtract = s;
       tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok);
       if (!tmp) goto EXPAND_FAILED;
       xtract = string_copyn(xtract, s - xtract);
 
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+        expand_string_message = US"missing '}' closing extractor arg of sort";
+	goto EXPAND_FAILED_CURLY;
+	}
 							/*{*/
       if (*s++ != '}')
         {						/*{*/
@@ -5998,7 +6171,11 @@ while (*s != 0)
 
       key = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok);
       if (!key) goto EXPAND_FAILED;			/*{*/
-      if (*s++ != '}') goto EXPAND_FAILED_CURLY;
+      if (*s++ != '}')
+        {
+        expand_string_message = US"missing '{' for name arg of env";
+	goto EXPAND_FAILED_CURLY;
+	}
 
       lookup_value = US getenv(CS key);
 
@@ -6062,7 +6239,12 @@ while (*s != 0)
 	  sub = expand_string_internal(s+2, TRUE, &s1, skipping,
 		  FALSE, &resetok);
 	  if (!sub)       goto EXPAND_FAILED;		/*{*/
-	  if (*s1 != '}') goto EXPAND_FAILED_CURLY;
+	  if (*s1 != '}')
+	    {
+	    expand_string_message =
+	      string_sprintf("missing '}' closing cert arg of %s", name);
+	    goto EXPAND_FAILED_CURLY;
+	    }
 	  if ((vp = find_var_ent(sub)) && vp->type == vtype_cert)
 	    {
 	    s = s1+1;
@@ -6091,6 +6273,47 @@ while (*s != 0)
 
     switch(c)
       {
+      case EOP_BASE32:
+	{
+        uschar *t;
+        unsigned long int n = Ustrtoul(sub, &t, 10);
+	uschar * s = NULL;
+	int sz = 0, i = 0;
+
+        if (*t != 0)
+          {
+          expand_string_message = string_sprintf("argument for base32 "
+            "operator is \"%s\", which is not a decimal number", sub);
+          goto EXPAND_FAILED;
+          }
+	for ( ; n; n >>= 5)
+	  s = string_catn(s, &sz, &i, &base32_chars[n & 0x1f], 1);
+
+	while (i > 0) yield = string_catn(yield, &size, &ptr, &s[--i], 1);
+	continue;
+	}
+
+      case EOP_BASE32D:
+        {
+        uschar *tt = sub;
+        unsigned long int n = 0;
+	uschar * s;
+        while (*tt)
+          {
+          uschar * t = Ustrchr(base32_chars, *tt++);
+          if (t == NULL)
+            {
+            expand_string_message = string_sprintf("argument for base32d "
+              "operator is \"%s\", which is not a base 32 number", sub);
+            goto EXPAND_FAILED;
+            }
+          n = n * 32 + (t - base32_chars);
+          }
+        s = string_sprintf("%ld", n);
+        yield = string_cat(yield, &size, &ptr, s);
+        continue;
+        }
+
       case EOP_BASE62:
         {
         uschar *t;
@@ -6193,29 +6416,78 @@ while (*s != 0)
 	else
 #endif
 	  {
-	  sha1 base;
+	  hctx h;
 	  uschar digest[20];
 	  int j;
 	  char st[41];
-	  sha1_start(&base);
-	  sha1_end(&base, sub, Ustrlen(sub), digest);
+	  sha1_start(&h);
+	  sha1_end(&h, sub, Ustrlen(sub), digest);
 	  for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]);
-	  yield = string_cat(yield, &size, &ptr, US st);
+	  yield = string_catn(yield, &size, &ptr, US st, 40);
 	  }
         continue;
 
       case EOP_SHA256:
-#ifdef SUPPORT_TLS
+#ifdef EXIM_HAVE_SHA2
 	if (vp && *(void **)vp->value)
 	  {
 	  uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value);
 	  yield = string_cat(yield, &size, &ptr, cp);
 	  }
 	else
+	  {
+	  hctx h;
+	  blob b;
+	  char st[3];
+
+	  exim_sha_init(&h, HASH_SHA256);
+	  exim_sha_update(&h, sub, Ustrlen(sub));
+	  exim_sha_finish(&h, &b);
+	  while (b.len-- > 0)
+	    {
+	    sprintf(st, "%02X", *b.data++);
+	    yield = string_catn(yield, &size, &ptr, US st, 2);
+	    }
+	  }
+#else
+	  expand_string_message = US"sha256 only supported with TLS";
 #endif
-	  expand_string_message = US"sha256 only supported for certificates";
         continue;
 
+      case EOP_SHA3:
+#ifdef EXIM_HAVE_SHA3
+	{
+	hctx h;
+	blob b;
+	char st[3];
+	hashmethod m = !arg ? HASH_SHA3_256
+	  : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224
+	  : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256
+	  : Ustrcmp(arg, "384") == 0 ? HASH_SHA3_384
+	  : Ustrcmp(arg, "512") == 0 ? HASH_SHA3_512
+	  : HASH_BADTYPE;
+
+	if (m == HASH_BADTYPE)
+	  {
+	  expand_string_message = US"unrecognised sha3 variant";
+	  goto EXPAND_FAILED;
+	  }
+
+	exim_sha_init(&h, m);
+	exim_sha_update(&h, sub, Ustrlen(sub));
+	exim_sha_finish(&h, &b);
+	while (b.len-- > 0)
+	  {
+	  sprintf(st, "%02X", *b.data++);
+	  yield = string_catn(yield, &size, &ptr, US st, 2);
+	  }
+	}
+        continue;
+#else
+	expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 +";
+	goto EXPAND_FAILED;
+#endif
+
       /* Convert hex encoding to base64 encoding */
 
       case EOP_HEX2B64:
@@ -6841,11 +7113,23 @@ while (*s != 0)
 
       case EOP_ESCAPE:
         {
-        const uschar *t = string_printing(sub);
+        const uschar * t = string_printing(sub);
         yield = string_cat(yield, &size, &ptr, t);
         continue;
         }
 
+      case EOP_ESCAPE8BIT:
+	{
+	const uschar * s = sub;
+	uschar c;
+
+	for (s = sub; (c = *s); s++)
+	  yield = c < 127 && c != '\\'
+	    ? string_catn(yield, &size, &ptr, s, 1)
+	    : string_catn(yield, &size, &ptr, string_sprintf("\\%03o", c), 4);
+	continue;
+	}
+
       /* Handle numeric expression evaluation */
 
       case EOP_EVAL:
@@ -7143,8 +7427,7 @@ while (*s != 0)
       yield = NULL;
       size = 0;
       }
-    value = find_variable(name, FALSE, skipping, &newsize);
-    if (value == NULL)
+    if (!(value = find_variable(name, FALSE, skipping, &newsize)))
       {
       expand_string_message =
         string_sprintf("unknown variable in \"${%s}\"", name);
@@ -7152,13 +7435,14 @@ while (*s != 0)
       goto EXPAND_FAILED;
       }
     len = Ustrlen(value);
-    if (yield == NULL && newsize != 0)
+    if (!yield && newsize)
       {
       yield = value;
       size = newsize;
       ptr = len;
       }
-    else yield = string_catn(yield, &size, &ptr, value, len);
+    else
+      yield = string_catn(yield, &size, &ptr, value, len);
     continue;
     }
 
@@ -7199,9 +7483,9 @@ else if (resetok_p) *resetok_p = FALSE;
 
 DEBUG(D_expand)
   {
-  debug_printf("expanding: %.*s\n   result: %s\n", (int)(s - string), string,
+  debug_printf("  expanding: %.*s\n     result: %s\n", (int)(s - string), string,
     yield);
-  if (skipping) debug_printf("skipping: result is not used\n");
+  if (skipping) debug_printf("   skipping: result is not used\n");
   }
 return yield;
 
@@ -7210,10 +7494,12 @@ to update the pointer to the terminator, for cases of nested calls with "fail".
 */
 
 EXPAND_FAILED_CURLY:
-expand_string_message = malformed_header?
-  US"missing or misplaced { or } - could be header name not terminated by colon"
-  :
-  US"missing or misplaced { or }";
+if (malformed_header)
+  expand_string_message =
+    US"missing or misplaced { or } - could be header name not terminated by colon";
+
+else if (!expand_string_message || !*expand_string_message)
+  expand_string_message = US"missing or misplaced { or }";
 
 /* At one point, Exim reset the store to yield (if yield was not NULL), but
 that is a bad idea, because expand_string_message is in dynamic store. */
@@ -7464,6 +7750,29 @@ return OK;
 
 
 
+/* Avoid potentially exposing a password in a string about to be logged */
+
+uschar *
+expand_hide_passwords(uschar * s)
+{
+return (  (  Ustrstr(s, "failed to expand") != NULL
+	  || Ustrstr(s, "expansion of ")    != NULL
+	  ) 
+       && (  Ustrstr(s, "mysql")   != NULL
+	  || Ustrstr(s, "pgsql")   != NULL
+	  || Ustrstr(s, "redis")   != NULL
+	  || Ustrstr(s, "sqlite")  != NULL
+	  || Ustrstr(s, "ldap:")   != NULL
+	  || Ustrstr(s, "ldaps:")  != NULL
+	  || Ustrstr(s, "ldapi:")  != NULL
+	  || Ustrstr(s, "ldapdn:") != NULL
+	  || Ustrstr(s, "ldapm:")  != NULL
+       )  ) 
+  ? US"Temporary internal error" : s;
+}
+
+
+
 
 /*************************************************
 **************************************************