X-Git-Url: https://vcs.fsf.org/?p=exim.git;a=blobdiff_plain;f=src%2Fsrc%2Fexpand.c;h=2be7f90941767d2ec88b304e688e8177480922ca;hp=9cbfbe883b1e5d62166a52aba732f5a485256e75;hb=8c513105fde2b8be3397216a0153f9b266fc7dfb;hpb=94e1f16d6033683bdebaf5092f64c58bc044dd2d diff --git a/src/src/expand.c b/src/src/expand.c index 9cbfbe883..2be7f9094 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2016 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ @@ -17,22 +17,22 @@ static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOO static int_eximarith_t expanded_string_integer(const uschar *, BOOL); #ifdef STAND_ALONE -#ifndef SUPPORT_CRYPTEQ -#define SUPPORT_CRYPTEQ -#endif +# ifndef SUPPORT_CRYPTEQ +# define SUPPORT_CRYPTEQ +# endif #endif #ifdef LOOKUP_LDAP -#include "lookups/ldap.h" +# include "lookups/ldap.h" #endif #ifdef SUPPORT_CRYPTEQ -#ifdef CRYPT_H -#include -#endif -#ifndef HAVE_CRYPT16 +# ifdef CRYPT_H +# include +# endif +# ifndef HAVE_CRYPT16 extern char* crypt16(char*, char*); -#endif +# endif #endif /* The handling of crypt16() is a mess. I will record below the analysis of the @@ -103,6 +103,7 @@ alphabetical order. */ static uschar *item_table[] = { US"acl", + US"authresults", US"certextract", US"dlfunc", US"env", @@ -133,6 +134,7 @@ static uschar *item_table[] = { enum { EITEM_ACL, + EITEM_AUTHRESULTS, EITEM_CERTEXTRACT, EITEM_DLFUNC, EITEM_ENV, @@ -459,6 +461,12 @@ static var_entry var_table[] = { { "address_data", vtype_stringptr, &deliver_address_data }, { "address_file", vtype_stringptr, &address_file }, { "address_pipe", vtype_stringptr, &address_pipe }, +#ifdef EXPERIMENTAL_ARC + { "arc_domains", vtype_string_func, &fn_arc_domains }, + { "arc_oldest_pass", vtype_int, &arc_oldest_pass }, + { "arc_state", vtype_stringptr, &arc_state }, + { "arc_state_reason", vtype_stringptr, &arc_state_reason }, +#endif { "authenticated_fail_id",vtype_stringptr, &authenticated_fail_id }, { "authenticated_id", vtype_stringptr, &authenticated_id }, { "authenticated_sender",vtype_stringptr, &authenticated_sender }, @@ -508,11 +516,10 @@ static var_entry var_table[] = { { "dkim_key_testing", vtype_dkim, (void *)DKIM_KEY_TESTING }, { "dkim_selector", vtype_stringptr, &dkim_signing_selector }, { "dkim_signers", vtype_stringptr, &dkim_signers }, - { "dkim_verify_reason", vtype_dkim, (void *)DKIM_VERIFY_REASON }, - { "dkim_verify_status", vtype_dkim, (void *)DKIM_VERIFY_STATUS}, + { "dkim_verify_reason", vtype_stringptr, &dkim_verify_reason }, + { "dkim_verify_status", vtype_stringptr, &dkim_verify_status }, #endif #ifdef EXPERIMENTAL_DMARC - { "dmarc_ar_header", vtype_stringptr, &dmarc_ar_header }, { "dmarc_domain_policy", vtype_stringptr, &dmarc_domain_policy }, { "dmarc_status", vtype_stringptr, &dmarc_status }, { "dmarc_status_text", vtype_stringptr, &dmarc_status_text }, @@ -557,7 +564,9 @@ static var_entry var_table[] = { { "local_part_data", vtype_stringptr, &deliver_localpart_data }, { "local_part_prefix", vtype_stringptr, &deliver_localpart_prefix }, { "local_part_suffix", vtype_stringptr, &deliver_localpart_suffix }, +#ifdef HAVE_LOCAL_SCAN { "local_scan_data", vtype_stringptr, &local_scan_data }, +#endif { "local_user_gid", vtype_gid, &local_user_gid }, { "local_user_uid", vtype_uid, &local_user_uid }, { "localhost_number", vtype_int, &host_number }, @@ -651,6 +660,9 @@ static var_entry var_table[] = { { "regex_match_string", vtype_stringptr, ®ex_match_string }, #endif { "reply_address", vtype_reply, NULL }, +#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS) + { "requiretls", vtype_bool, &tls_requiretls }, +#endif { "return_path", vtype_stringptr, &return_path }, { "return_size_limit", vtype_int, &bounce_return_size_limit }, { "router_name", vtype_stringptr, &router_name }, @@ -700,11 +712,12 @@ static var_entry var_table[] = { { "spam_score", vtype_stringptr, &spam_score }, { "spam_score_int", vtype_stringptr, &spam_score_int }, #endif -#ifdef EXPERIMENTAL_SPF +#ifdef SUPPORT_SPF { "spf_guess", vtype_stringptr, &spf_guess }, { "spf_header_comment", vtype_stringptr, &spf_header_comment }, { "spf_received", vtype_stringptr, &spf_received }, { "spf_result", vtype_stringptr, &spf_result }, + { "spf_result_guessed", vtype_bool, &spf_result_guessed }, { "spf_smtp_comment", vtype_stringptr, &spf_smtp_comment }, #endif { "spool_directory", vtype_stringptr, &spool_directory }, @@ -738,7 +751,7 @@ static var_entry var_table[] = { { "tls_out_bits", vtype_int, &tls_out.bits }, { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified }, { "tls_out_cipher", vtype_stringptr, &tls_out.cipher }, -#ifdef EXPERIMENTAL_DANE +#ifdef SUPPORT_DANE { "tls_out_dane", vtype_bool, &tls_out.dane_verified }, #endif { "tls_out_ocsp", vtype_int, &tls_out.ocsp }, @@ -748,7 +761,7 @@ static var_entry var_table[] = { #if defined(SUPPORT_TLS) { "tls_out_sni", vtype_stringptr, &tls_out.sni }, #endif -#ifdef EXPERIMENTAL_DANE +#ifdef SUPPORT_DANE { "tls_out_tlsa_usage", vtype_int, &tls_out.tlsa_usage }, #endif @@ -807,6 +820,10 @@ static uschar *mtable_setid[] = static uschar *mtable_sticky[] = { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" }; +/* flags for find_header() */ +#define FH_EXISTS_ONLY BIT(0) +#define FH_WANT_RAW BIT(1) +#define FH_WANT_LIST BIT(2) /************************************************* @@ -910,7 +927,7 @@ int rc; uschar *ss = expand_string(condition); if (ss == NULL) { - if (!expand_string_forcedfail && !search_find_defer) + if (!f.expand_string_forcedfail && !f.search_find_defer) log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" " "for %s %s: %s", condition, m1, m2, expand_string_message); return FALSE; @@ -1116,20 +1133,20 @@ Returns: NULL if the subfield was not found, or */ static uschar * -expand_getkeyed(uschar *key, const uschar *s) +expand_getkeyed(uschar * key, const uschar * s) { int length = Ustrlen(key); while (isspace(*s)) s++; /* Loop to search for the key */ -while (*s != 0) +while (*s) { int dkeylength; - uschar *data; - const uschar *dkey = s; + uschar * data; + const uschar * dkey = s; - while (*s != 0 && *s != '=' && !isspace(*s)) s++; + while (*s && *s != '=' && !isspace(*s)) s++; dkeylength = s - dkey; while (isspace(*s)) s++; if (*s == '=') while (isspace((*(++s)))); @@ -1240,17 +1257,17 @@ return fieldtext; static uschar * expand_getlistele(int field, const uschar * list) { -const uschar * tlist= list; -int sep= 0; +const uschar * tlist = list; +int sep = 0; uschar dummy; -if(field<0) +if (field < 0) { - for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; - sep= 0; + for (field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; + sep = 0; } -if(field==0) return NULL; -while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; +if (field == 0) return NULL; +while (--field > 0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; return string_nextinlist(&list, &sep, NULL, 0); } @@ -1484,14 +1501,14 @@ while (*s != 0) /* If value2 is unset, just compute one number */ if (value2 < 0) - s = string_sprintf("%d", total % value1); + s = string_sprintf("%lu", total % value1); /* Otherwise do a div/mod hash */ else { total = total % (value1 * value2); - s = string_sprintf("%d/%d", total/value2, total % value2); + s = string_sprintf("%lu/%lu", total/value2, total % value2); } *len = Ustrlen(s); @@ -1511,22 +1528,25 @@ can also return a concatenation of all the header lines. When concatenating specific headers that contain lists of addresses, a comma is inserted between them. Otherwise we use a straight concatenation. Because some messages can have pathologically large number of lines, there is a limit on the length that is -returned. Also, to avoid massive store use which would result from using -string_cat() as it copies and extends strings, we do a preliminary pass to find -out exactly how much store will be needed. On "normal" messages this will be -pretty trivial. +returned. Arguments: name the name of the header, without the leading $header_ or $h_, or NULL if a concatenation of all headers is required - exists_only TRUE if called from a def: test; don't need to build a string; - just return a string that is not "" and not "0" if the header - exists newsize return the size of memory block that was obtained; may be NULL if exists_only is TRUE - want_raw TRUE if called for $rh_ or $rheader_ variables; no processing, - other than concatenating, will be done on the header. Also used - for $message_headers_raw. + flags FH_EXISTS_ONLY + set if called from a def: test; don't need to build a string; + just return a string that is not "" and not "0" if the header + exists + FH_WANT_RAW + set if called for $rh_ or $rheader_ items; no processing, + other than concatenating, will be done on the header. Also used + for $message_headers_raw. + FH_WANT_LIST + Double colon chars in the content, and replace newline with + colon between each element when concatenating; returning a + colon-sep list (elements might contain newlines) charset name of charset to translate MIME words to; used only if want_raw is false; if NULL, no translation is done (this is used for $bh_ and $bheader_) @@ -1536,124 +1556,141 @@ Returns: NULL if the header does not exist, else a pointer to a new */ static uschar * -find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw, - uschar *charset) +find_header(uschar *name, int *newsize, unsigned flags, uschar *charset) { -BOOL found = name == NULL; -int comma = 0; -int len = found? 0 : Ustrlen(name); -int i; -uschar *yield = NULL; -uschar *ptr = NULL; - -/* Loop for two passes - saves code repetition */ - -for (i = 0; i < 2; i++) - { - int size = 0; - header_line *h; - - for (h = header_list; size < header_insert_maxlen && h; h = h->next) - if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ - if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) - { - int ilen; - uschar *t; - - if (exists_only) return US"1"; /* don't need actual string */ - found = TRUE; - t = h->text + len; /* text to insert */ - if (!want_raw) /* unless wanted raw, */ - while (isspace(*t)) t++; /* remove leading white space */ - ilen = h->slen - (t - h->text); /* length to insert */ - - /* Unless wanted raw, remove trailing whitespace, including the - newline. */ - - if (!want_raw) - while (ilen > 0 && isspace(t[ilen-1])) ilen--; +BOOL found = !name; +int len = name ? Ustrlen(name) : 0; +BOOL comma = FALSE; +header_line * h; +gstring * g = NULL; - /* Set comma = 1 if handling a single header and it's one of those - that contains an address list, except when asked for raw headers. Only - need to do this once. */ +for (h = header_list; h; h = h->next) + if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ + if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) + { + uschar * s, * t; + size_t inc; - if (!want_raw && name && comma == 0 && - Ustrchr("BCFRST", h->type) != NULL) - comma = 1; + if (flags & FH_EXISTS_ONLY) + return US"1"; /* don't need actual string */ - /* First pass - compute total store needed; second pass - compute - total store used, including this header. */ + found = TRUE; + s = h->text + len; /* text to insert */ + if (!(flags & FH_WANT_RAW)) /* unless wanted raw, */ + while (isspace(*s)) s++; /* remove leading white space */ + t = h->text + h->slen; /* end-point */ - size += ilen + comma + 1; /* +1 for the newline */ + /* Unless wanted raw, remove trailing whitespace, including the + newline. */ - /* Second pass - concatenate the data, up to a maximum. Note that - the loop stops when size hits the limit. */ + if (flags & FH_WANT_LIST) + while (t > s && t[-1] == '\n') t--; + else if (!(flags & FH_WANT_RAW)) + { + while (t > s && isspace(t[-1])) t--; - if (i != 0) - { - if (size > header_insert_maxlen) - { - ilen -= size - header_insert_maxlen - 1; - comma = 0; - } - Ustrncpy(ptr, t, ilen); - ptr += ilen; + /* Set comma if handling a single header and it's one of those + that contains an address list, except when asked for raw headers. Only + need to do this once. */ - /* For a non-raw header, put in the comma if needed, then add - back the newline we removed above, provided there was some text in - the header. */ + if (name && !comma && Ustrchr("BCFRST", h->type)) comma = TRUE; + } - if (!want_raw && ilen > 0) - { - if (comma != 0) *ptr++ = ','; - *ptr++ = '\n'; - } - } - } + /* Trim the header roughly if we're approaching limits */ + inc = t - s; + if ((g ? g->ptr : 0) + inc > header_insert_maxlen) + inc = header_insert_maxlen - (g ? g->ptr : 0); + + /* For raw just copy the data; for a list, add the data as a colon-sep + list-element; for comma-list add as an unchecked comma,newline sep + list-elemment; for other nonraw add as an unchecked newline-sep list (we + stripped trailing WS above including the newline). We ignore the potential + expansion due to colon-doubling, just leaving the loop if the limit is met + or exceeded. */ + + if (flags & FH_WANT_LIST) + g = string_append_listele_n(g, ':', s, (unsigned)inc); + else if (flags & FH_WANT_RAW) + { + g = string_catn(g, s, (unsigned)inc); + (void) string_from_gstring(g); + } + else if (inc > 0) + if (comma) + g = string_append2_listele_n(g, US",\n", s, (unsigned)inc); + else + g = string_append2_listele_n(g, US"\n", s, (unsigned)inc); - /* At end of first pass, return NULL if no header found. Then truncate size - if necessary, and get the buffer to hold the data, returning the buffer size. - */ + if (g && g->ptr >= header_insert_maxlen) break; + } - if (i == 0) - { - if (!found) return NULL; - if (size > header_insert_maxlen) size = header_insert_maxlen; - *newsize = size + 1; - ptr = yield = store_get(*newsize); - } - } +if (!found) return NULL; /* No header found */ +if (!g) return US""; /* That's all we do for raw header expansion. */ -if (want_raw) - *ptr = 0; +*newsize = g->size; +if (flags & FH_WANT_RAW) + return g->s; -/* Otherwise, remove a final newline and a redundant added comma. Then we do -RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2() -function can return an error with decoded data if the charset translation -fails. If decoding fails, it returns NULL. */ +/* Otherwise do RFC 2047 decoding, translating the charset if requested. +The rfc2047_decode2() function can return an error with decoded data if the +charset translation fails. If decoding fails, it returns NULL. */ else { uschar *decoded, *error; - if (ptr > yield && ptr[-1] == '\n') ptr--; - if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--; - *ptr = 0; - decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL, + + decoded = rfc2047_decode2(g->s, check_rfc2047_length, charset, '?', NULL, newsize, &error); - if (error != NULL) + if (error) { DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n" - " input was: %s\n", error, yield); + " input was: %s\n", error, g->s); } - if (decoded != NULL) yield = decoded; + return decoded ? decoded : g->s; } +} -return yield; + + + +/* Append a "local" element to an Authentication-Results: header +if this was a non-smtp message. +*/ + +static gstring * +authres_local(gstring * g, const uschar * sysname) +{ +if (!f.authentication_local) + return g; +g = string_append(g, 3, US";\n\tlocal=pass (non-smtp, ", sysname, US")"); +if (authenticated_id) g = string_append(g, 2, " u=", authenticated_id); +return g; } +/* Append an "iprev" element to an Authentication-Results: header +if we have attempted to get the calling host's name. +*/ + +static gstring * +authres_iprev(gstring * g) +{ +if (sender_host_name) + g = string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")"); +else if (host_lookup_deferred) + g = string_catn(g, US";\n\tiprev=temperror", 19); +else if (host_lookup_failed) + g = string_catn(g, US";\n\tiprev=fail", 13); +else + return g; + +if (sender_host_address) + g = string_append(g, 2, US" smtp.remote-ip=", sender_host_address); +return g; +} + /************************************************* @@ -1666,18 +1703,18 @@ generated from a system filter, but not elsewhere. */ static uschar * fn_recipients(void) { +uschar * s; gstring * g = NULL; int i; -if (!enable_dollar_recipients) return NULL; +if (!f.enable_dollar_recipients) return NULL; for (i = 0; i < recipients_count; i++) { - /*XXX variant of list_appendele? */ - if (i != 0) g = string_catn(g, US", ", 2); - g = string_cat(g, recipients_list[i].address); + s = recipients_list[i].address; + g = string_append2_listele_n(g, US", ", s, Ustrlen(s)); } -return string_from_gstring(g); +return g ? g->s : NULL; } @@ -1762,7 +1799,7 @@ val = vp->value; switch (vp->type) { case vtype_filter_int: - if (!filter_running) return NULL; + if (!f.filter_running) return NULL; /* Fall through */ /* VVVVVVVVVVVV */ case vtype_int: @@ -1797,10 +1834,10 @@ switch (vp->type) return var_buffer; case vtype_host_lookup: /* Lookup if not done so */ - if (sender_host_name == NULL && sender_host_address != NULL && - !host_lookup_failed && host_name_lookup() == OK) + if ( !sender_host_name && sender_host_address + && !host_lookup_failed && host_name_lookup() == OK) host_build_sender_fullhost(); - return (sender_host_name == NULL)? US"" : sender_host_name; + return sender_host_name ? sender_host_name : US""; case vtype_localpart: /* Get local part from address */ s = *((uschar **)(val)); @@ -1821,10 +1858,11 @@ switch (vp->type) return (domain == NULL)? US"" : domain + 1; case vtype_msgheaders: - return find_header(NULL, exists_only, newsize, FALSE, NULL); + return find_header(NULL, newsize, exists_only ? FH_EXISTS_ONLY : 0, NULL); case vtype_msgheaders_raw: - return find_header(NULL, exists_only, newsize, TRUE, NULL); + return find_header(NULL, newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, NULL); case vtype_msgbody: /* Pointer to msgbody string */ case vtype_msgbody_end: /* Ditto, the end of the msg */ @@ -1889,15 +1927,18 @@ switch (vp->type) return tod_stamp(tod_log_datestamp_daily); case vtype_reply: /* Get reply address */ - s = find_header(US"reply-to:", exists_only, newsize, TRUE, - headers_charset); - if (s != NULL) while (isspace(*s)) s++; - if (s == NULL || *s == 0) + s = find_header(US"reply-to:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); + if (s) while (isspace(*s)) s++; + if (!s || !*s) { *newsize = 0; /* For the *s==0 case */ - s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset); + s = find_header(US"from:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); } - if (s != NULL) + if (s) { uschar *t; while (isspace(*s)) s++; @@ -1905,7 +1946,7 @@ switch (vp->type) while (t > s && isspace(t[-1])) t--; *t = 0; } - return (s == NULL)? US"" : s; + return s ? s : US""; case vtype_string_func: { @@ -1916,7 +1957,7 @@ switch (vp->type) case vtype_pspace: { int inodes; - sprintf(CS var_buffer, "%d", + sprintf(CS var_buffer, PR_EXIM_ARITH, receive_statvfs(val == (void *)TRUE, &inodes)); } return var_buffer; @@ -2184,56 +2225,58 @@ switch(cond_type) yield == NULL we are in a skipping state, and don't care about the answer. */ case ECOND_DEF: - if (*s != ':') { - expand_string_message = US"\":\" expected after \"def\""; - return NULL; - } + uschar * t; - s = read_name(name, 256, s+1, US"_"); + if (*s != ':') + { + expand_string_message = US"\":\" expected after \"def\""; + return NULL; + } - /* Test for a header's existence. If the name contains a closing brace - character, this may be a user error where the terminating colon has been - omitted. Set a flag to adjust a subsequent error message in this case. */ + s = read_name(name, 256, s+1, US"_"); - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) - { - s = read_header_name(name, 256, s); - /* {-for-text-editors */ - if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; - if (yield != NULL) *yield = - (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor; - } + /* Test for a header's existence. If the name contains a closing brace + character, this may be a user error where the terminating colon has been + omitted. Set a flag to adjust a subsequent error message in this case. */ - /* Test for a variable's having a non-empty value. A non-existent variable - causes an expansion failure. */ + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) + { + s = read_header_name(name, 256, s); + /* {-for-text-editors */ + if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; + if (yield) *yield = + (find_header(name, NULL, FH_EXISTS_ONLY, NULL) != NULL) == testfor; + } - else - { - uschar *value = find_variable(name, TRUE, yield == NULL, NULL); - if (value == NULL) + /* Test for a variable's having a non-empty value. A non-existent variable + causes an expansion failure. */ + + else { - expand_string_message = (name[0] == 0)? - string_sprintf("variable name omitted after \"def:\"") : - string_sprintf("unknown variable \"%s\" after \"def:\"", name); - check_variable_error_message(name); - return NULL; + if (!(t = find_variable(name, TRUE, yield == NULL, NULL))) + { + expand_string_message = (name[0] == 0)? + string_sprintf("variable name omitted after \"def:\"") : + string_sprintf("unknown variable \"%s\" after \"def:\"", name); + check_variable_error_message(name); + return NULL; + } + if (yield) *yield = (t[0] != 0) == testfor; } - if (yield != NULL) *yield = (value[0] != 0) == testfor; - } - return s; + return s; + } /* first_delivery tests for first delivery attempt */ case ECOND_FIRST_DELIVERY: - if (yield != NULL) *yield = deliver_firsttime == testfor; + if (yield != NULL) *yield = f.deliver_firsttime == testfor; return s; @@ -2388,7 +2431,7 @@ switch(cond_type) break; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); @@ -2498,9 +2541,12 @@ switch(cond_type) "after \"%s\"", name); return NULL; } - sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL, - honour_dollar, resetok); - if (sub[i] == NULL) return NULL; + if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL, + honour_dollar, resetok))) + return NULL; + DEBUG(D_expand) if (i == 1 && !sub2_honour_dollar && Ustrchr(sub[1], '$')) + debug_printf_indent("WARNING: the second arg is NOT expanded," + " for security reasons\n"); if (*s++ != '}') goto COND_FAILED_CURLY_END; /* Convert to numerical if required; we know that the names of all the @@ -2821,18 +2867,21 @@ switch(cond_type) uschar *save_iterate_item = iterate_item; int (*compare)(const uschar *, const uschar *); - DEBUG(D_expand) debug_printf_indent("condition: %s\n", name); + DEBUG(D_expand) debug_printf_indent("condition: %s item: %s\n", name, sub[0]); tempcond = FALSE; compare = cond_type == ECOND_INLISTI ? strcmpic : (int (*)(const uschar *, const uschar *)) strcmp; while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0))) + { + DEBUG(D_expand) debug_printf_indent(" compare %s\n", iterate_item); if (compare(sub[0], iterate_item) == 0) { tempcond = TRUE; break; } + } iterate_item = save_iterate_item; } @@ -3209,8 +3258,8 @@ want this string. Set skipping in the call in the fail case (this will always be the case if we were already skipping). */ sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok); -if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED; -expand_string_forcedfail = FALSE; +if (sub1 == NULL && (yes || !f.expand_string_forcedfail)) goto FAILED; +f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'yes' part did not end with '}'"; @@ -3239,8 +3288,8 @@ while (isspace(*s)) s++; if (*s == '{') { sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok); - if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED; - expand_string_forcedfail = FALSE; + if (sub2 == NULL && (!yes || !f.expand_string_forcedfail)) goto FAILED; + f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'no' part did not start with '{'"; @@ -3275,7 +3324,7 @@ else if (*s != '}') } expand_string_message = string_sprintf("\"%s\" failed and \"fail\" requested", type); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; goto FAILED; } } @@ -3504,6 +3553,26 @@ return yield; } +#ifdef SUPPORT_TLS +static gstring * +cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol) +{ +int rc; +uschar * s; +uschar buffer[1024]; + +while ((rc = tls_read(tls_ctx, buffer, sizeof(buffer))) > 0) + for (s = buffer; rc--; s++) + yield = eol && *s == '\n' + ? string_cat(yield, eol) : string_catn(yield, s, 1); + +/* We assume that all errors, and any returns of zero bytes, +are actually EOF. */ + +(void) string_from_gstring(yield); +return yield; +} +#endif /************************************************* @@ -3780,6 +3849,87 @@ return x; +/* Return pointer to dewrapped string, with enclosing specified chars removed. +The given string is modified on return. Leading whitespace is skipped while +looking for the opening wrap character, then the rest is scanned for the trailing +(non-escaped) wrap character. A backslash in the string will act as an escape. + +A nul is written over the trailing wrap, and a pointer to the char after the +leading wrap is returned. + +Arguments: + s String for de-wrapping + wrap Two-char string, the first being the opener, second the closer wrapping + character +Return: + Pointer to de-wrapped string, or NULL on error (with expand_string_message set). +*/ + +static uschar * +dewrap(uschar * s, const uschar * wrap) +{ +uschar * p = s; +unsigned depth = 0; +BOOL quotesmode = wrap[0] == wrap[1]; + +while (isspace(*p)) p++; + +if (*p == *wrap) + { + s = ++p; + wrap++; + while (*p) + { + if (*p == '\\') p++; + else if (!quotesmode && *p == wrap[-1]) depth++; + else if (*p == *wrap) + if (depth == 0) + { + *p = '\0'; + return s; + } + else + depth--; + p++; + } + } +expand_string_message = string_sprintf("missing '%c'", *wrap); +return NULL; +} + + +/* Pull off the leading array or object element, returning +a copy in an allocated string. Update the list pointer. + +The element may itself be an abject or array. +*/ + +uschar * +json_nextinlist(const uschar ** list) +{ +unsigned array_depth = 0, object_depth = 0; +const uschar * s = *list, * item; + +while (isspace(*s)) s++; + +for (item = s; + *s && (*s != ',' || array_depth != 0 || object_depth != 0); + s++) + switch (*s) + { + case '[': array_depth++; break; + case ']': array_depth--; break; + case '{': object_depth++; break; + case '}': object_depth--; break; + } +*list = *s ? s+1 : s; +item = string_copyn(item, s - item); +DEBUG(D_expand) debug_printf_indent(" json ele: '%s'\n", item); +return US item; +} + + + /************************************************* * Expand string * *************************************************/ @@ -3856,13 +4006,17 @@ BOOL resetok = TRUE; expand_level++; DEBUG(D_expand) - debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", - skipping - ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" - : "considering", - string); + DEBUG(D_noutf8) + debug_printf_indent("/%s: %s\n", + skipping ? "---scanning" : "considering", string); + else + debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", + skipping + ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" + : "considering", + string); -expand_string_forcedfail = FALSE; +f.expand_string_forcedfail = FALSE; expand_string_message = US""; while (*s != 0) @@ -3927,33 +4081,38 @@ while (*s != 0) { int len; int newsize = 0; - gstring * g; + gstring * g = NULL; + uschar * t; s = read_name(name, sizeof(name), s, US"_"); /* If this is the first thing to be expanded, release the pre-allocated buffer. */ - if (yield && yield->ptr == 0) + if (!yield) + g = store_get(sizeof(gstring)); + else if (yield->ptr == 0) { if (resetok) store_reset(yield); yield = NULL; - g = store_get(sizeof(gstring)); + g = store_get(sizeof(gstring)); /* alloc _before_ calling find_variable() */ } /* Header */ - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) { - BOOL want_raw = (name[0] == 'r')? TRUE : FALSE; - uschar *charset = (name[0] == 'b')? NULL : headers_charset; + unsigned flags = *name == 'r' ? FH_WANT_RAW + : *name == 'l' ? FH_WANT_RAW|FH_WANT_LIST + : 0; + uschar * charset = *name == 'b' ? NULL : headers_charset; + s = read_header_name(name, sizeof(name), s); - value = find_header(name, FALSE, &newsize, want_raw, charset); + value = find_header(name, &newsize, flags, charset); /* If we didn't find the header, and the header contains a closing brace character, this may be a user error where the terminating colon @@ -3993,7 +4152,8 @@ while (*s != 0) yield->ptr = len; yield->s = value; } - else yield = string_catn(yield, value, len); + else + yield = string_catn(yield, value, len); continue; } @@ -4083,7 +4243,7 @@ while (*s != 0) continue; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); @@ -4091,6 +4251,41 @@ while (*s != 0) } } + case EITEM_AUTHRESULTS: + /* ${authresults {mysystemname}} */ + { + uschar *sub_arg[1]; + + switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name, + &resetok)) + { + case 1: goto EXPAND_FAILED_CURLY; + case 2: + case 3: goto EXPAND_FAILED; + } + + yield = string_append(yield, 3, + US"Authentication-Results: ", sub_arg[0], US"; none"); + yield->ptr -= 6; + + yield = authres_local(yield, sub_arg[0]); + yield = authres_iprev(yield); + yield = authres_smtpauth(yield); +#ifdef SUPPORT_SPF + yield = authres_spf(yield); +#endif +#ifndef DISABLE_DKIM + yield = authres_dkim(yield); +#endif +#ifdef EXPERIMENTAL_DMARC + yield = authres_dmarc(yield); +#endif +#ifdef EXPERIMENTAL_ARC + yield = authres_arc(yield); +#endif + continue; + } + /* Handle conditionals - preserve the values of the numerical expansion variables in case they get changed by a regular expression match in the condition. If not, they retain their external settings. At the end @@ -4108,15 +4303,21 @@ while (*s != 0) if (next_s == NULL) goto EXPAND_FAILED; /* message already set */ DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "condition: %.*s\n", - (int)(next_s - s), s); - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - cond ? "true" : "false"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|--condition: %.*s\n", (int)(next_s - s), s); + debug_printf_indent("|-----result: %s\n", cond ? "true" : "false"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "condition: %.*s\n", + (int)(next_s - s), s); + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + cond ? "true" : "false"); + } s = next_s; @@ -4346,7 +4547,7 @@ while (*s != 0) } lookup_value = search_find(handle, filename, key, partial, affix, affixlen, starflags, &expand_setup); - if (search_find_defer) + if (f.search_find_defer) { expand_string_message = string_sprintf("lookup of \"%s\" gave DEFER: %s", @@ -4455,7 +4656,7 @@ while (*s != 0) expand_string_message = string_sprintf("Perl subroutine \"%s\" returned undef to force " "failure", sub_arg[0]); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; } goto EXPAND_FAILED; } @@ -4463,7 +4664,7 @@ while (*s != 0) /* Yield succeeded. Ensure forcedfail is unset, just in case it got set during a callback from Perl. */ - expand_string_forcedfail = FALSE; + f.expand_string_forcedfail = FALSE; yield = new_yield; continue; } @@ -4488,25 +4689,25 @@ while (*s != 0) if (skipping) continue; /* sub_arg[0] is the address */ - domain = Ustrrchr(sub_arg[0],'@'); - if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) ) + if ( !(domain = Ustrrchr(sub_arg[0],'@')) + || domain == sub_arg[0] || Ustrlen(domain) == 1) { expand_string_message = US"prvs first argument must be a qualified email address"; goto EXPAND_FAILED; } - /* Calculate the hash. The second argument must be a single-digit + /* Calculate the hash. The third argument must be a single-digit key number, or unset. */ - if (sub_arg[2] != NULL && - (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0)) + if ( sub_arg[2] + && (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0)) { - expand_string_message = US"prvs second argument must be a single digit"; + expand_string_message = US"prvs third argument must be a single digit"; goto EXPAND_FAILED; } - p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7)); - if (p == NULL) + p = prvs_hmac_sha1(sub_arg[0], sub_arg[1], sub_arg[2], prvs_daystamp(7)); + if (!p) { expand_string_message = US"prvs hmac-sha1 conversion failed"; goto EXPAND_FAILED; @@ -4622,7 +4823,7 @@ while (*s != 0) prvscheck_result = US"1"; DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n"); } - else + else { prvscheck_result = NULL; DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n"); @@ -4713,10 +4914,14 @@ while (*s != 0) int fd; int timeout = 5; int save_ptr = yield->ptr; - FILE *f; - uschar *arg; - uschar *sub_arg[4]; + FILE * fp; + uschar * arg; + uschar * sub_arg[4]; + uschar * server_name = NULL; + host_item host; BOOL do_shutdown = TRUE; + BOOL do_tls = FALSE; /* Only set under SUPPORT_TLS */ + void * tls_ctx = NULL; /* ditto */ blob reqstr; if (expand_forbid & RDO_READSOCK) @@ -4759,10 +4964,14 @@ while (*s != 0) while ((item = string_nextinlist(&list, &sep, NULL, 0))) if (Ustrncmp(item, US"shutdown=", 9) == 0) - if (Ustrcmp(item + 9, US"no") == 0) - do_shutdown = FALSE; + { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; } +#ifdef SUPPORT_TLS + else if (Ustrncmp(item, US"tls=", 4) == 0) + { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; } +#endif } - else sub_arg[3] = NULL; /* No eol if no timeout */ + else + sub_arg[3] = NULL; /* No eol if no timeout */ /* If skipping, we don't actually do anything. Otherwise, arrange to connect to either an IP or a Unix socket. */ @@ -4774,8 +4983,10 @@ while (*s != 0) if (Ustrncmp(sub_arg[0], "inet:", 5) == 0) { int port; - uschar * server_name = sub_arg[0] + 5; - uschar * port_name = Ustrrchr(server_name, ':'); + uschar * port_name; + + server_name = sub_arg[0] + 5; + port_name = Ustrrchr(server_name, ':'); /* Sort out the port */ @@ -4810,12 +5021,15 @@ while (*s != 0) port = ntohs(service_info->s_port); } + /*XXX we trust that the request is idempotent. Hmm. */ fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port, - timeout, NULL, &expand_string_message, &reqstr); + timeout, &host, &expand_string_message, + do_tls ? NULL : &reqstr); callout_address = NULL; if (fd < 0) - goto SOCK_FAIL; - reqstr.len = 0; + goto SOCK_FAIL; + if (!do_tls) + reqstr.len = 0; } /* Handle a Unix domain socket */ @@ -4835,11 +5049,12 @@ while (*s != 0) sockun.sun_family = AF_UNIX; sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1), sub_arg[0]); + server_name = US sockun.sun_path; sigalrm_seen = FALSE; - alarm(timeout); + ALARM(timeout); rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun)); - alarm(0); + ALARM_CLR(0); if (sigalrm_seen) { expand_string_message = US "socket connect timed out"; @@ -4851,12 +5066,32 @@ while (*s != 0) "%s: %s", sub_arg[0], strerror(errno)); goto SOCK_FAIL; } + host.name = server_name; + host.address = US""; } DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]); +#ifdef SUPPORT_TLS + if (do_tls) + { + tls_support tls_dummy = {.sni=NULL}; + uschar * errstr; + + if (!(tls_ctx = tls_client_start(fd, &host, NULL, NULL, +# ifdef SUPPORT_DANE + NULL, +# endif + &tls_dummy, &errstr))) + { + expand_string_message = string_sprintf("TLS connect failed: %s", errstr); + goto SOCK_FAIL; + } + } +#endif + /* Allow sequencing of test actions */ - if (running_in_test_harness) millisleep(100); + if (f.running_in_test_harness) millisleep(100); /* Write the request string, if not empty or already done */ @@ -4864,7 +5099,11 @@ while (*s != 0) { DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n", reqstr.data); - if (write(fd, reqstr.data, reqstr.len) != reqstr.len) + if ( ( +#ifdef SUPPORT_TLS + tls_ctx ? tls_write(tls_ctx, reqstr.data, reqstr.len, FALSE) : +#endif + write(fd, reqstr.data, reqstr.len)) != reqstr.len) { expand_string_message = string_sprintf("request write to socket " "failed: %s", strerror(errno)); @@ -4877,20 +5116,34 @@ while (*s != 0) system doesn't have this function, make it conditional. */ #ifdef SHUT_WR - if (do_shutdown) shutdown(fd, SHUT_WR); + if (!tls_ctx && do_shutdown) shutdown(fd, SHUT_WR); #endif - if (running_in_test_harness) millisleep(100); + if (f.running_in_test_harness) millisleep(100); /* Now we need to read from the socket, under a timeout. The function that reads a file can be used. */ - f = fdopen(fd, "rb"); + if (!tls_ctx) + fp = fdopen(fd, "rb"); sigalrm_seen = FALSE; - alarm(timeout); - yield = cat_file(f, yield, sub_arg[3]); - alarm(0); - (void)fclose(f); + ALARM(timeout); + yield = +#ifdef SUPPORT_TLS + tls_ctx ? cat_file_tls(tls_ctx, yield, sub_arg[3]) : +#endif + cat_file(fp, yield, sub_arg[3]); + ALARM_CLR(0); + +#ifdef SUPPORT_TLS + if (tls_ctx) + { + tls_close(tls_ctx, TRUE); + close(fd); + } + else +#endif + (void)fclose(fp); /* After a timeout, we restore the pointer in the result, that is, make sure we add nothing from the socket. */ @@ -5013,9 +5266,9 @@ while (*s != 0) resetok = FALSE; f = fdopen(fd_out, "rb"); sigalrm_seen = FALSE; - alarm(60); + ALARM(60); lookup_value = string_from_gstring(cat_file(f, NULL, NULL)); - alarm(0); + ALARM_CLR(0); (void)fclose(f); /* Wait for the process to finish, applying the timeout, and inspect its @@ -5393,6 +5646,16 @@ while (*s != 0) uschar *sub[3]; int save_expand_nmax = save_expand_strings(save_expand_nstring, save_expand_nlength); + enum {extract_basic, extract_json} fmt = extract_basic; + + while (isspace(*s)) s++; + + /* Check for a format-variant specifier */ + + if (*s != '{') /*}*/ + { + if (Ustrncmp(s, "json", 4) == 0) {fmt = extract_json; s += 4;} + } /* While skipping we cannot rely on the data for expansions being available (eg. $item) hence cannot decide on numeric vs. keyed. @@ -5400,11 +5663,10 @@ while (*s != 0) if (skipping) { - while (isspace(*s)) s++; - for (j = 5; j > 0 && *s == '{'; j--) + for (j = 5; j > 0 && *s == '{'; j--) /*'}'*/ { if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok)) - goto EXPAND_FAILED; /*{*/ + goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = US"missing '{' for arg of extract"; @@ -5412,13 +5674,13 @@ while (*s != 0) } while (isspace(*s)) s++; } - if ( Ustrncmp(s, "fail", 4) == 0 + if ( Ustrncmp(s, "fail", 4) == 0 /*'{'*/ && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4]) ) { s += 4; while (isspace(*s)) s++; - } + } /*'{'*/ if (*s != '}') { expand_string_message = US"missing '}' closing extract"; @@ -5428,11 +5690,11 @@ while (*s != 0) else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */ { - while (isspace(*s)) s++; - if (*s == '{') /*}*/ + while (isspace(*s)) s++; + if (*s == '{') /*'}'*/ { sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); - if (sub[i] == NULL) goto EXPAND_FAILED; /*{*/ + if (sub[i] == NULL) goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = string_sprintf( @@ -5443,7 +5705,7 @@ while (*s != 0) /* After removal of leading and trailing white space, the first argument must not be empty; if it consists entirely of digits (optionally preceded by a minus sign), this is a numerical - extraction, and we expect 3 arguments. */ + extraction, and we expect 3 arguments (normal) or 2 (json). */ if (i == 0) { @@ -5474,7 +5736,7 @@ while (*s != 0) if (*p == 0) { field_number *= x; - j = 3; /* Need 3 args */ + if (fmt != extract_json) j = 3; /* Need 3 args */ field_number_set = TRUE; } } @@ -5490,9 +5752,83 @@ while (*s != 0) /* Extract either the numbered or the keyed substring into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : field_number_set? - expand_gettokened(field_number, sub[1], sub[2]) : - expand_getkeyed(sub[0], sub[1]); + if (skipping) + lookup_value = NULL; + else switch (fmt) + { + case extract_basic: + lookup_value = field_number_set + ? expand_gettokened(field_number, sub[1], sub[2]) + : expand_getkeyed(sub[0], sub[1]); + break; + + case extract_json: + { + uschar * s, * item; + const uschar * list; + + /* Array: Bracket-enclosed and comma-separated. + Object: Brace-enclosed, comma-sep list of name:value pairs */ + + if (!(s = dewrap(sub[1], field_number_set ? US"[]" : US"{}"))) + { + expand_string_message = + string_sprintf("%s wrapping %s for extract json", + expand_string_message, + field_number_set ? "array" : "object"); + goto EXPAND_FAILED_CURLY; + } + + list = s; + if (field_number_set) + { + if (field_number <= 0) + { + expand_string_message = US"first argument of \"extract\" must " + "be greater than zero"; + goto EXPAND_FAILED; + } + while (field_number > 0 && (item = json_nextinlist(&list))) + field_number--; + s = item; + lookup_value = s; + while (*s) s++; + while (--s >= lookup_value && isspace(*s)) *s = '\0'; + } + else + { + lookup_value = NULL; + while ((item = json_nextinlist(&list))) + { + /* Item is: string name-sep value. string is quoted. + Dequote the string and compare with the search key. */ + + if (!(item = dewrap(item, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string key for extract json", + expand_string_message); + goto EXPAND_FAILED_CURLY; + } + if (Ustrcmp(item, sub[0]) == 0) /*XXX should be a UTF8-compare */ + { + s = item + Ustrlen(item) + 1; + while (isspace(*s)) s++; + if (*s != ':') + { + expand_string_message = string_sprintf( + "missing object value-separator for extract json"); + goto EXPAND_FAILED_CURLY; + } + s++; + while (isspace(*s)) s++; + lookup_value = s; + break; + } + } + } + } + } /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -5592,7 +5928,7 @@ while (*s != 0) /* Extract the numbered element into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]); + lookup_value = skipping ? NULL : expand_getlistele(field_number, sub[1]); /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -5784,7 +6120,8 @@ while (*s != 0) if (*s++ != '}') { /*{*/ expand_string_message = string_sprintf("missing } at end of condition " - "or expression inside \"%s\"", name); + "or expression inside \"%s\"; could be an unquoted } in the content", + name); goto EXPAND_FAILED; } @@ -6156,7 +6493,7 @@ while (*s != 0) else { expand_string_message = result == NULL ? US"(no message)" : result; - if(status == FAIL_FORCED) expand_string_forcedfail = TRUE; + if(status == FAIL_FORCED) f.expand_string_forcedfail = TRUE; else if(status != FAIL) log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s", argv[0], argv[1], status, expand_string_message); @@ -6210,7 +6547,9 @@ while (*s != 0) int c; uschar *arg = NULL; uschar *sub; +#ifdef SUPPORT_TLS var_entry *vp = NULL; +#endif /* Owing to an historical mis-design, an underscore may be part of the operator name, or it may introduce arguments. We therefore first scan the @@ -6335,7 +6674,6 @@ while (*s != 0) case EOP_BASE62D: { - uschar buf[16]; uschar *tt = sub; unsigned long int n = 0; while (*tt != 0) @@ -6350,8 +6688,7 @@ while (*s != 0) } n = n * BASE_62 + (t - base62_chars); } - (void)sprintf(CS buf, "%ld", n); - yield = string_cat(yield, buf); + yield = string_fmt_append(yield, "%ld", n); continue; } @@ -6400,11 +6737,10 @@ while (*s != 0) md5 base; uschar digest[16]; int j; - char st[33]; md5_start(&base); md5_end(&base, sub, Ustrlen(sub), digest); - for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]); - yield = string_cat(yield, US st); + for (j = 0; j < 16; j++) + yield = string_fmt_append(yield, "%02x", digest[j]); } continue; @@ -6421,11 +6757,10 @@ while (*s != 0) hctx h; uschar digest[20]; int j; - char st[41]; sha1_start(&h); sha1_end(&h, sub, Ustrlen(sub), digest); - for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]); - yield = string_catn(yield, US st, 40); + for (j = 0; j < 20; j++) + yield = string_fmt_append(yield, "%02X", digest[j]); } continue; @@ -6440,7 +6775,6 @@ while (*s != 0) { hctx h; blob b; - char st[3]; if (!exim_sha_init(&h, HASH_SHA2_256)) { @@ -6450,10 +6784,7 @@ while (*s != 0) exim_sha_update(&h, sub, Ustrlen(sub)); exim_sha_finish(&h, &b); while (b.len-- > 0) - { - sprintf(st, "%02X", *b.data++); - yield = string_catn(yield, US st, 2); - } + yield = string_fmt_append(yield, "%02X", *b.data++); } #else expand_string_message = US"sha256 only supported with TLS"; @@ -6465,7 +6796,6 @@ while (*s != 0) { hctx h; blob b; - char st[3]; hashmethod m = !arg ? HASH_SHA3_256 : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224 : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256 @@ -6482,14 +6812,11 @@ while (*s != 0) exim_sha_update(&h, sub, Ustrlen(sub)); exim_sha_finish(&h, &b); while (b.len-- > 0) - { - sprintf(st, "%02X", *b.data++); - yield = string_catn(yield, US st, 2); - } + yield = string_fmt_append(yield, "%02X", *b.data++); } continue; #else - expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 +"; + expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 + or OpenSSL 1.1.1 +"; goto EXPAND_FAILED; #endif @@ -6549,7 +6876,7 @@ while (*s != 0) while (*(++t) != 0) { if (*t < 0x21 || 0x7E < *t) - yield = string_catn(yield, string_sprintf("\\x%02x", *t), 4); + yield = string_fmt_append(yield, "\\x%02x", *t); else yield = string_catn(yield, t, 1); } @@ -6562,12 +6889,10 @@ while (*s != 0) { int cnt = 0; int sep = 0; - uschar * cp; uschar buffer[256]; while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++; - cp = string_sprintf("%d", cnt); - yield = string_cat(yield, cp); + yield = string_fmt_append(yield, "%d", cnt); continue; } @@ -6766,8 +7091,15 @@ while (*s != 0) int start, end, domain; /* Not really used */ while (isspace(*sub)) sub++; - if (*sub == '>') { *outsep = *++sub; ++sub; } - parse_allow_group = TRUE; + if (*sub == '>') + if (*outsep = *++sub) ++sub; + else + { + expand_string_message = string_sprintf("output separator " + "missing in expanding ${addresses:%s}", --sub); + goto EXPAND_FAILED; + } + f.parse_allow_group = TRUE; for (;;) { @@ -6816,7 +7148,7 @@ while (*s != 0) separator. */ if (yield->ptr != save_ptr) yield->ptr--; - parse_allow_group = FALSE; + f.parse_allow_group = FALSE; continue; } @@ -6926,9 +7258,9 @@ while (*s != 0) case EOP_RFC2047: { uschar buffer[2048]; - const uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset, - buffer, sizeof(buffer), FALSE); - yield = string_cat(yield, string); + yield = string_cat(yield, + parse_quote_2047(sub, Ustrlen(sub), headers_charset, + buffer, sizeof(buffer), FALSE)); continue; } @@ -6974,12 +7306,13 @@ while (*s != 0) { int seq_len = 0, index = 0; int bytes_left = 0; - long codepoint = -1; + long codepoint = -1; + int complete; uschar seq_buff[4]; /* accumulate utf-8 here */ while (*sub != 0) { - int complete = 0; + complete = 0; uschar c = *sub++; if (bytes_left) @@ -7044,6 +7377,13 @@ while (*s != 0) /* ASCII character follows incomplete sequence */ yield = string_catn(yield, &c, 1); } + /* If given a sequence truncated mid-character, we also want to report ? + * Eg, ${length_1:フィル} is one byte, not one character, so we expect + * ${utf8clean:${length_1:フィル}} to yield '?' */ + if (bytes_left != 0) + { + yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1); + } continue; } @@ -7127,7 +7467,7 @@ while (*s != 0) for (s = sub; (c = *s); s++) yield = c < 127 && c != '\\' ? string_catn(yield, s, 1) - : string_catn(yield, string_sprintf("\\%03o", c), 4); + : string_fmt_append(yield, "\\%03o", c); continue; } @@ -7139,15 +7479,14 @@ while (*s != 0) uschar *save_sub = sub; uschar *error = NULL; int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE); - if (error != NULL) + if (error) { expand_string_message = string_sprintf("error in expression " - "evaluation: %s (after processing \"%.*s\")", error, sub-save_sub, - save_sub); + "evaluation: %s (after processing \"%.*s\")", error, + (int)(sub-save_sub), save_sub); goto EXPAND_FAILED; } - sprintf(CS var_buffer, PR_EXIM_ARITH, n); - yield = string_cat(yield, var_buffer); + yield = string_fmt_append(yield, PR_EXIM_ARITH, n); continue; } @@ -7162,8 +7501,7 @@ while (*s != 0) "Exim time interval in \"%s\" operator", sub, name); goto EXPAND_FAILED; } - sprintf(CS var_buffer, "%d", n); - yield = string_cat(yield, var_buffer); + yield = string_fmt_append(yield, "%d", n); continue; } @@ -7215,12 +7553,8 @@ while (*s != 0) /* strlen returns the length of the string */ case EOP_STRLEN: - { - uschar buff[24]; - (void)sprintf(CS buff, "%d", Ustrlen(sub)); - yield = string_cat(yield, buff); + yield = string_fmt_append(yield, "%d", Ustrlen(sub)); continue; - } /* length_n or l_n takes just the first n characters or the whole string, whichever is the shorter; @@ -7253,7 +7587,7 @@ while (*s != 0) int len; uschar *ret; - if (arg == NULL) + if (!arg) { expand_string_message = string_sprintf("missing values after %s", name); @@ -7317,14 +7651,13 @@ while (*s != 0) case EOP_STAT: { - uschar *s; uschar smode[12]; uschar **modetable[3]; int i; mode_t mode; struct stat st; - if ((expand_forbid & RDO_EXISTS) != 0) + if (expand_forbid & RDO_EXISTS) { expand_string_message = US"Use of the stat() expansion is not permitted"; goto EXPAND_FAILED; @@ -7358,13 +7691,13 @@ while (*s != 0) } smode[10] = 0; - s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld " + yield = string_fmt_append(yield, + "mode=%04lo smode=%s inode=%ld device=%ld links=%ld " "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld", (long)(st.st_mode & 077777), smode, (long)st.st_ino, (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid, (long)st.st_gid, st.st_size, (long)st.st_atime, (long)st.st_mtime, (long)st.st_ctime); - yield = string_cat(yield, s); continue; } @@ -7372,14 +7705,11 @@ while (*s != 0) case EOP_RANDINT: { - int_eximarith_t max; - uschar *s; + int_eximarith_t max = expanded_string_integer(sub, TRUE); - max = expanded_string_integer(sub, TRUE); - if (expand_string_message != NULL) + if (expand_string_message) goto EXPAND_FAILED; - s = string_sprintf("%d", vaguely_random_number((int)max)); - yield = string_cat(yield, s); + yield = string_fmt_append(yield, "%d", vaguely_random_number((int)max)); continue; } @@ -7405,9 +7735,9 @@ while (*s != 0) /* Unknown operator */ default: - expand_string_message = - string_sprintf("unknown expansion operator \"%s\"", name); - goto EXPAND_FAILED; + expand_string_message = + string_sprintf("unknown expansion operator \"%s\"", name); + goto EXPAND_FAILED; } } @@ -7422,13 +7752,15 @@ while (*s != 0) { int len; int newsize = 0; - gstring * g; + gstring * g = NULL; - if (yield && yield->ptr == 0) + if (!yield) + g = store_get(sizeof(gstring)); + else if (yield->ptr == 0) { if (resetok) store_reset(yield); yield = NULL; - g = store_get(sizeof(gstring)); + g = store_get(sizeof(gstring)); /* alloc _before_ calling find_variable() */ } if (!(value = find_variable(name, FALSE, skipping, &newsize))) { @@ -7463,10 +7795,9 @@ terminating brace. */ if (ket_ends && *s == 0) { - expand_string_message = malformed_header? - US"missing } at end of string - could be header name not terminated by colon" - : - US"missing } at end of string"; + expand_string_message = malformed_header + ? US"missing } at end of string - could be header name not terminated by colon" + : US"missing } at end of string"; goto EXPAND_FAILED; } @@ -7487,19 +7818,28 @@ if (resetok) store_reset(yield->s + (yield->size = yield->ptr + 1)); else if (resetok_p) *resetok_p = FALSE; DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "expanding: %.*s\n", - (int)(s - string), string); - debug_printf_indent("%s" - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - yield->s); - if (skipping) - debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "skipping: result is not used\n"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string); + debug_printf_indent("%sresult: %s\n", + skipping ? "|-----" : "\\_____", yield->s); + if (skipping) + debug_printf_indent("\\___skipping: result is not used\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "expanding: %.*s\n", + (int)(s - string), string); + debug_printf_indent("%s" + UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + yield->s); + if (skipping) + debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "skipping: result is not used\n"); + } expand_level--; return yield->s; @@ -7521,17 +7861,26 @@ that is a bad idea, because expand_string_message is in dynamic store. */ EXPAND_FAILED: if (left) *left = s; DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", - string); - debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "error message: %s\n", - expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - expand_string_message); - if (expand_string_forcedfail) - debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); - } -if (resetok_p) *resetok_p = resetok; + DEBUG(D_noutf8) + { + debug_printf_indent("|failed to expand: %s\n", string); + debug_printf_indent("%serror message: %s\n", + f.expand_string_forcedfail ? "|---" : "\\___", expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent("\\failure was forced\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", + string); + debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "error message: %s\n", + f.expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); + } +if (resetok_p && !resetok) *resetok_p = FALSE; expand_level--; return NULL; } @@ -7545,28 +7894,35 @@ Returns: the expanded string, or NULL if expansion failed; if failure was due to a lookup deferring, search_find_defer will be TRUE */ -uschar * -expand_string(uschar *string) +const uschar * +expand_cstring(const uschar * string) { -search_find_defer = FALSE; -malformed_header = FALSE; -return (Ustrpbrk(string, "$\\") == NULL)? string : - expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); +if (Ustrpbrk(string, "$\\") != NULL) + { + int old_pool = store_pool; + uschar * s; + + f.search_find_defer = FALSE; + malformed_header = FALSE; + store_pool = POOL_MAIN; + s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); + store_pool = old_pool; + return s; + } +return string; } - -const uschar * -expand_cstring(const uschar *string) +uschar * +expand_string(uschar * string) { -search_find_defer = FALSE; -malformed_header = FALSE; -return (Ustrpbrk(string, "$\\") == NULL)? string : - expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); +return US expand_cstring(CUS string); } + + /************************************************* * Expand and copy * *************************************************/ @@ -7739,7 +8095,7 @@ if (svalue == NULL) { *rvalue = bvalue; return OK; } expanded = expand_string(svalue); if (expanded == NULL) { - if (expand_string_forcedfail) + if (f.expand_string_forcedfail) { DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname); *rvalue = bvalue; @@ -7777,7 +8133,7 @@ expand_hide_passwords(uschar * s) { return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "expansion of ") != NULL - ) + ) && ( Ustrstr(s, "mysql") != NULL || Ustrstr(s, "pgsql") != NULL || Ustrstr(s, "redis") != NULL @@ -7787,18 +8143,55 @@ return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "ldapi:") != NULL || Ustrstr(s, "ldapdn:") != NULL || Ustrstr(s, "ldapm:") != NULL - ) ) + ) ) ? US"Temporary internal error" : s; } +/* Read given named file into big_buffer. Use for keying material etc. +The content will have an ascii NUL appended. + +Arguments: + filename as it says + +Return: pointer to buffer, or NULL on error. +*/ + +uschar * +expand_file_big_buffer(const uschar * filename) +{ +int fd, off = 0, len; + +if ((fd = open(CS filename, O_RDONLY)) < 0) + { + log_write(0, LOG_MAIN | LOG_PANIC, "unable to open file for reading: %s", + filename); + return NULL; + } + +do + { + if ((len = read(fd, big_buffer + off, big_buffer_size - 2 - off)) < 0) + { + (void) close(fd); + log_write(0, LOG_MAIN|LOG_PANIC, "unable to read file: %s", filename); + return NULL; + } + off += len; + } +while (len > 0); + +(void) close(fd); +big_buffer[off] = '\0'; +return big_buffer; +} + + /************************************************* * Error-checking for testsuite * *************************************************/ typedef struct { - const char * filename; - int linenumber; uschar * region_start; uschar * region_end; const uschar *var_name; @@ -7819,7 +8212,8 @@ if (var_data >= e->region_start && var_data < e->region_end) void assert_no_variables(void * ptr, int len, const char * filename, int linenumber) { -err_ctx e = {filename, linenumber, ptr, US ptr + len, NULL }; +err_ctx e = { .region_start = ptr, .region_end = US ptr + len, + .var_name = NULL, .var_data = NULL }; int i; var_entry * v; @@ -7840,10 +8234,16 @@ for (v = var_table; v < var_table + var_table_size; v++) if (v->type == vtype_stringptr) assert_variable_notin(US v->name, *(USS v->value), &e); +/* check dns and address trees */ +tree_walk(tree_dns_fails, assert_variable_notin, &e); +tree_walk(tree_duplicates, assert_variable_notin, &e); +tree_walk(tree_nonrecipients, assert_variable_notin, &e); +tree_walk(tree_unusable, assert_variable_notin, &e); + if (e.var_name) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'", - e.var_name, e.filename, e.linenumber, e.var_data); + e.var_name, filename, linenumber, e.var_data); } @@ -7952,9 +8352,9 @@ while (fgets(buffer, sizeof(buffer), stdin) != NULL) } else { - if (search_find_defer) printf("search_find deferred\n"); + if (f.search_find_defer) printf("search_find deferred\n"); printf("Failed: %s\n", expand_string_message); - if (expand_string_forcedfail) printf("Forced failure\n"); + if (f.expand_string_forcedfail) printf("Forced failure\n"); printf("\n"); } }