X-Git-Url: https://vcs.fsf.org/?p=exim.git;a=blobdiff_plain;f=src%2Fsrc%2Fverify.c;h=ffa6a14f9b9c70496d1548be78d691881c90eb51;hp=865a01d07f4e836ecedf793d62ae1fc78b1b1fec;hb=72cb765f4ce4f9b503e45060b42e33f1248e8b64;hpb=ff5929e3b91747e2ecb600711d17a7d0e21749ad diff --git a/src/src/verify.c b/src/src/verify.c index 865a01d07..ffa6a14f9 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2016 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ /* Functions concerned with verifying things. The original code for callout @@ -39,7 +39,8 @@ static tree_node *dnsbl_cache = NULL; #define MT_NOT 1 #define MT_ALL 2 -static uschar cutthrough_response(char, uschar **); +static uschar cutthrough_response(int, char, uschar **, int); + /************************************************* @@ -67,9 +68,7 @@ int length, expire; time_t now; dbdata_callout_cache *cache_record; -cache_record = dbfn_read_with_length(dbm_file, key, &length); - -if (cache_record == NULL) +if (!(cache_record = dbfn_read_with_length(dbm_file, key, &length))) { HDEBUG(D_verify) debug_printf("callout cache: no %s record found for %s\n", type, key); return NULL; @@ -118,98 +117,21 @@ return cache_record; -/************************************************* -* Do callout verification for an address * -*************************************************/ - -/* This function is called from verify_address() when the address has routed to -a host list, and a callout has been requested. Callouts are expensive; that is -why a cache is used to improve the efficiency. - -Arguments: - addr the address that's been routed - host_list the list of hosts to try - tf the transport feedback block - - ifstring "interface" option from transport, or NULL - portstring "port" option from transport, or NULL - protocolstring "protocol" option from transport, or NULL - callout the per-command callout timeout - callout_overall the overall callout timeout (if < 0 use 4*callout) - callout_connect the callout connection timeout (if < 0 use callout) - options the verification options - these bits are used: - vopt_is_recipient => this is a recipient address - vopt_callout_no_cache => don't use callout cache - vopt_callout_fullpm => if postmaster check, do full one - vopt_callout_random => do the "random" thing - vopt_callout_recipsender => use real sender for recipient - vopt_callout_recippmaster => use postmaster for recipient - se_mailfrom MAIL FROM address for sender verify; NULL => "" - pm_mailfrom if non-NULL, do the postmaster check with this sender +/* Check the callout cache. +Options * pm_mailfrom may be modified by cache partial results. -Returns: OK/FAIL/DEFER +Return: TRUE if result found */ -static int -do_callout(address_item *addr, host_item *host_list, transport_feedback *tf, - int callout, int callout_overall, int callout_connect, int options, - uschar *se_mailfrom, uschar *pm_mailfrom) +static BOOL +cached_callout_lookup(address_item * addr, uschar * address_key, + uschar * from_address, int * opt_ptr, uschar ** pm_ptr, + int * yield, uschar ** failure_ptr, + dbdata_callout_cache * new_domain_record, int * old_domain_res) { -int yield = OK; -int old_domain_cache_result = ccache_accept; -BOOL done = FALSE; -uschar *address_key; -uschar *from_address; -uschar *random_local_part = NULL; -const uschar *save_deliver_domain = deliver_domain; -uschar **failure_ptr = options & vopt_is_recipient - ? &recipient_verify_failure : &sender_verify_failure; +int options = *opt_ptr; open_db dbblock; open_db *dbm_file = NULL; -dbdata_callout_cache new_domain_record; -dbdata_callout_cache_address new_address_record; -host_item *host; -time_t callout_start_time; -uschar peer_offered = 0; - -new_domain_record.result = ccache_unknown; -new_domain_record.postmaster_result = ccache_unknown; -new_domain_record.random_result = ccache_unknown; - -memset(&new_address_record, 0, sizeof(new_address_record)); - -/* For a recipient callout, the key used for the address cache record must -include the sender address if we are using the real sender in the callout, -because that may influence the result of the callout. */ - -address_key = addr->address; -from_address = US""; - -if (options & vopt_is_recipient) - { - if (options & vopt_callout_recipsender) - { - address_key = string_sprintf("%s/<%s>", addr->address, sender_address); - from_address = sender_address; - if (cutthrough.delivery) options |= vopt_callout_no_cache; - } - else if (options & vopt_callout_recippmaster) - { - address_key = string_sprintf("%s/", addr->address, - qualify_domain_sender); - from_address = string_sprintf("postmaster@%s", qualify_domain_sender); - } - } - -/* For a sender callout, we must adjust the key if the mailfrom address is not -empty. */ - -else - { - from_address = (se_mailfrom == NULL)? US"" : se_mailfrom; - if (from_address[0] != 0) - address_key = string_sprintf("%s/<%s>", addr->address, from_address); - } /* Open the callout cache database, it it exists, for reading only at this stage, unless caching has been disabled. */ @@ -218,21 +140,19 @@ if (options & vopt_callout_no_cache) { HDEBUG(D_verify) debug_printf("callout cache: disabled by no_cache\n"); } -else if ((dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE)) == NULL) +else if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE))) { HDEBUG(D_verify) debug_printf("callout cache: not available\n"); } - -/* If a cache database is available see if we can avoid the need to do an -actual callout by making use of previously-obtained data. */ - -if (dbm_file) +else { - dbdata_callout_cache_address *cache_address_record; - dbdata_callout_cache *cache_record = get_callout_cache_record(dbm_file, - addr->domain, US"domain", - callout_cache_domain_positive_expire, - callout_cache_domain_negative_expire); + /* If a cache database is available see if we can avoid the need to do an + actual callout by making use of previously-obtained data. */ + + dbdata_callout_cache_address * cache_address_record; + dbdata_callout_cache * cache_record = get_callout_cache_record(dbm_file, + addr->domain, US"domain", + callout_cache_domain_positive_expire, callout_cache_domain_negative_expire); /* If an unexpired cache record was found for this domain, see if the callout process can be short-circuited. */ @@ -247,20 +167,21 @@ if (dbm_file) not to disturb the cached domain value if this whole verification succeeds (we don't want it turning into "accept"). */ - old_domain_cache_result = cache_record->result; + *old_domain_res = cache_record->result; - if (cache_record->result == ccache_reject || - (*from_address == 0 && cache_record->result == ccache_reject_mfnull)) + if ( cache_record->result == ccache_reject + || *from_address == 0 && cache_record->result == ccache_reject_mfnull) { setflag(addr, af_verify_nsfail); HDEBUG(D_verify) - debug_printf("callout cache: domain gave initial rejection, or " - "does not accept HELO or MAIL FROM:<>\n"); + debug_printf("callout cache: domain gave initial rejection, or " + "does not accept HELO or MAIL FROM:<>\n"); setflag(addr, af_verify_nsfail); addr->user_message = US"(result of an earlier callout reused)."; - yield = FAIL; + *yield = FAIL; *failure_ptr = US"mail"; - goto END_CALLOUT; + dbfn_close(dbm_file); + return TRUE; } /* If a previous check on a "random" local part was accepted, we assume @@ -275,21 +196,23 @@ if (dbm_file) case ccache_accept: HDEBUG(D_verify) debug_printf("callout cache: domain accepts random addresses\n"); - goto END_CALLOUT; /* Default yield is OK */ + dbfn_close(dbm_file); + return TRUE; /* Default yield is OK */ case ccache_reject: HDEBUG(D_verify) debug_printf("callout cache: domain rejects random addresses\n"); - options &= ~vopt_callout_random; - new_domain_record.random_result = ccache_reject; - new_domain_record.random_stamp = cache_record->random_stamp; + *opt_ptr = options & ~vopt_callout_random; + new_domain_record->random_result = ccache_reject; + new_domain_record->random_stamp = cache_record->random_stamp; break; default: HDEBUG(D_verify) debug_printf("callout cache: need to check random address handling " "(not cached or cache expired)\n"); - goto END_CACHE; + dbfn_close(dbm_file); + return FALSE; } /* If a postmaster check is requested, but there was a previous failure, @@ -297,27 +220,29 @@ if (dbm_file) but has not been done before, we are going to have to do a callout, so skip remaining cache processing. */ - if (pm_mailfrom) + if (*pm_ptr) { if (cache_record->postmaster_result == ccache_reject) - { - setflag(addr, af_verify_pmfail); - HDEBUG(D_verify) - debug_printf("callout cache: domain does not accept " - "RCPT TO:\n"); - yield = FAIL; - *failure_ptr = US"postmaster"; - setflag(addr, af_verify_pmfail); - addr->user_message = US"(result of earlier verification reused)."; - goto END_CALLOUT; - } + { + setflag(addr, af_verify_pmfail); + HDEBUG(D_verify) + debug_printf("callout cache: domain does not accept " + "RCPT TO:\n"); + *yield = FAIL; + *failure_ptr = US"postmaster"; + setflag(addr, af_verify_pmfail); + addr->user_message = US"(result of earlier verification reused)."; + dbfn_close(dbm_file); + return TRUE; + } if (cache_record->postmaster_result == ccache_unknown) - { - HDEBUG(D_verify) - debug_printf("callout cache: need to check RCPT " - "TO: (not cached or cache expired)\n"); - goto END_CACHE; - } + { + HDEBUG(D_verify) + debug_printf("callout cache: need to check RCPT " + "TO: (not cached or cache expired)\n"); + dbfn_close(dbm_file); + return FALSE; + } /* If cache says OK, set pm_mailfrom NULL to prevent a redundant postmaster check if the address itself has to be checked. Also ensure @@ -325,10 +250,10 @@ if (dbm_file) */ HDEBUG(D_verify) debug_printf("callout cache: domain accepts RCPT " - "TO:\n"); - pm_mailfrom = NULL; - new_domain_record.postmaster_result = ccache_accept; - new_domain_record.postmaster_stamp = cache_record->postmaster_stamp; + "TO:\n"); + *pm_ptr = NULL; + new_domain_record->postmaster_result = ccache_accept; + new_domain_record->postmaster_stamp = cache_record->postmaster_stamp; } } @@ -337,681 +262,488 @@ if (dbm_file) sender address if we are doing a recipient callout with a non-empty sender). */ - cache_address_record = (dbdata_callout_cache_address *) - get_callout_cache_record(dbm_file, - address_key, US"address", - callout_cache_positive_expire, - callout_cache_negative_expire); + if (!(cache_address_record = (dbdata_callout_cache_address *) + get_callout_cache_record(dbm_file, address_key, US"address", + callout_cache_positive_expire, callout_cache_negative_expire))) + { + dbfn_close(dbm_file); + return FALSE; + } - if (cache_address_record) + if (cache_address_record->result == ccache_accept) { - if (cache_address_record->result == ccache_accept) - { - HDEBUG(D_verify) - debug_printf("callout cache: address record is positive\n"); - } - else - { - HDEBUG(D_verify) - debug_printf("callout cache: address record is negative\n"); - addr->user_message = US"Previous (cached) callout verification failure"; - *failure_ptr = US"recipient"; - yield = FAIL; - } - goto END_CALLOUT; + HDEBUG(D_verify) + debug_printf("callout cache: address record is positive\n"); + } + else + { + HDEBUG(D_verify) + debug_printf("callout cache: address record is negative\n"); + addr->user_message = US"Previous (cached) callout verification failure"; + *failure_ptr = US"recipient"; + *yield = FAIL; } /* Close the cache database while we actually do the callout for real. */ - END_CACHE: dbfn_close(dbm_file); - dbm_file = NULL; - } - -if (!addr->transport) - { - HDEBUG(D_verify) debug_printf("cannot callout via null transport\n"); + return TRUE; } -else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0) - log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp", - addr->transport->name, addr->transport->driver_name); -else - { - smtp_transport_options_block *ob = - (smtp_transport_options_block *)addr->transport->options_block; - - /* The information wasn't available in the cache, so we have to do a real - callout and save the result in the cache for next time, unless no_cache is set, - or unless we have a previously cached negative random result. If we are to test - with a random local part, ensure that such a local part is available. If not, - log the fact, but carry on without randomming. */ - - if (options & vopt_callout_random && callout_random_local_part != NULL) - if (!(random_local_part = expand_string(callout_random_local_part))) - log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand " - "callout_random_local_part: %s", expand_string_message); - - /* Default the connect and overall callout timeouts if not set, and record the - time we are starting so that we can enforce it. */ - - if (callout_overall < 0) callout_overall = 4 * callout; - if (callout_connect < 0) callout_connect = callout; - callout_start_time = time(NULL); - - /* Before doing a real callout, if this is an SMTP connection, flush the SMTP - output because a callout might take some time. When PIPELINING is active and - there are many recipients, the total time for doing lots of callouts can add up - and cause the client to time out. So in this case we forgo the PIPELINING - optimization. */ +return FALSE; +} - if (smtp_out && !disable_callout_flush) mac_smtp_fflush(); -/* cutthrough-multi: if a nonfirst rcpt has the same routing as the first, -and we are holding a cutthrough conn open, we can just append the rcpt to -that conn for verification purposes (and later delivery also). Simplest -coding means skipping this whole loop and doing the append separately. - -We will need to remember it has been appended so that rcpt-acl tail code -can do it there for the non-rcpt-verify case. For this we keep an addresscount. +/* Write results to callout cache */ +static void +cache_callout_write(dbdata_callout_cache * dom_rec, const uschar * domain, + int done, dbdata_callout_cache_address * addr_rec, uschar * address_key) +{ +open_db dbblock; +open_db *dbm_file = NULL; - /* Can we re-use an open cutthrough connection? */ - if ( cutthrough.fd >= 0 - && (options & (vopt_callout_recipsender | vopt_callout_recippmaster)) - == vopt_callout_recipsender - && !random_local_part - && !pm_mailfrom - ) - { - if (addr->transport == cutthrough.addr.transport) - for (host = host_list; host; host = host->next) - if (Ustrcmp(host->address, cutthrough.host.address) == 0) - { - int host_af; - uschar *interface = NULL; /* Outgoing interface to use; NULL => any */ - int port = 25; - - deliver_host = host->name; - deliver_host_address = host->address; - deliver_host_port = host->port; - deliver_domain = addr->domain; - transport_name = addr->transport->name; - - host_af = (Ustrchr(host->address, ':') == NULL)? AF_INET:AF_INET6; - - if (!smtp_get_interface(tf->interface, host_af, addr, &interface, - US"callout") || - !smtp_get_port(tf->port, addr, &port, US"callout")) - log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address, - addr->message); - - if ( ( interface == cutthrough.interface - || ( interface - && cutthrough.interface - && Ustrcmp(interface, cutthrough.interface) == 0 - ) ) - && port == cutthrough.host.port - ) - { - uschar * resp = NULL; - - /* Match! Send the RCPT TO, append the addr, set done */ - done = - smtp_write_command(&ctblock, FALSE, "RCPT TO:<%.1000s>\r\n", - transport_rcpt_address(addr, - (addr->transport == NULL)? FALSE : - addr->transport->rcpt_include_affixes)) >= 0 && - cutthrough_response('2', &resp) == '2'; - - /* This would go horribly wrong if a callout fail was ignored by ACL. - We punt by abandoning cutthrough on a reject, like the - first-rcpt does. */ - - if (done) - { - address_item * na = store_get(sizeof(address_item)); - *na = cutthrough.addr; - cutthrough.addr = *addr; - cutthrough.addr.host_used = &cutthrough.host; - cutthrough.addr.next = na; - - cutthrough.nrcpt++; - } - else - { - cancel_cutthrough_connection("recipient rejected"); - if (!resp || errno == ETIMEDOUT) - { - HDEBUG(D_verify) debug_printf("SMTP timeout\n"); - } - else if (errno == 0) - { - if (*resp == 0) - Ustrcpy(resp, US"connection dropped"); - - addr->message = - string_sprintf("response to \"%s\" from %s [%s] was: %s", - big_buffer, host->name, host->address, - string_printing(resp)); - - addr->user_message = - string_sprintf("Callout verification failed:\n%s", resp); +/* If we get here with done == TRUE, a successful callout happened, and yield +will be set OK or FAIL according to the response to the RCPT command. +Otherwise, we looped through the hosts but couldn't complete the business. +However, there may be domain-specific information to cache in both cases. - /* Hard rejection ends the process */ +The value of the result field in the new_domain record is ccache_unknown if +there was an error before or with MAIL FROM:, and errno was not zero, +implying some kind of I/O error. We don't want to write the cache in that case. +Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */ - if (resp[0] == '5') /* Address rejected */ - { - yield = FAIL; - done = TRUE; - } - } - } - } - break; - } - if (!done) - cancel_cutthrough_connection("incompatible connection"); +if (dom_rec->result != ccache_unknown) + if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE))) + { + HDEBUG(D_verify) debug_printf("callout cache: not available\n"); } - - /* Now make connections to the hosts and do real callouts. The list of hosts - is passed in as an argument. */ - - for (host = host_list; host && !done; host = host->next) + else { - smtp_inblock inblock; - smtp_outblock outblock; - int host_af; - int port = 25; - BOOL send_quit = TRUE; - uschar *active_hostname = smtp_active_hostname; - BOOL lmtp; - BOOL smtps; - BOOL esmtp; - BOOL suppress_tls = FALSE; - uschar *interface = NULL; /* Outgoing interface to use; NULL => any */ -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) - BOOL dane = FALSE; - BOOL dane_required; - dns_answer tlsa_dnsa; -#endif - uschar inbuffer[4096]; - uschar outbuffer[1024]; - uschar responsebuffer[4096]; - uschar * size_str; - - clearflag(addr, af_verify_pmfail); /* postmaster callout flag */ - clearflag(addr, af_verify_nsfail); /* null sender callout flag */ - - /* Skip this host if we don't have an IP address for it. */ - - if (!host->address) - { - DEBUG(D_verify) debug_printf("no IP address for host name %s: skipping\n", - host->name); - continue; - } - - /* Check the overall callout timeout */ + (void)dbfn_write(dbm_file, domain, dom_rec, + (int)sizeof(dbdata_callout_cache)); + HDEBUG(D_verify) debug_printf("wrote callout cache domain record for %s:\n" + " result=%d postmaster=%d random=%d\n", + domain, + dom_rec->result, + dom_rec->postmaster_result, + dom_rec->random_result); + } - if (time(NULL) - callout_start_time >= callout_overall) - { - HDEBUG(D_verify) debug_printf("overall timeout for callout exceeded\n"); - break; - } +/* If a definite result was obtained for the callout, cache it unless caching +is disabled. */ - /* Set IPv4 or IPv6 */ +if (done && addr_rec->result != ccache_unknown) + { + if (!dbm_file) + dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE); + if (!dbm_file) + { + HDEBUG(D_verify) debug_printf("no callout cache available\n"); + } + else + { + (void)dbfn_write(dbm_file, address_key, addr_rec, + (int)sizeof(dbdata_callout_cache_address)); + HDEBUG(D_verify) debug_printf("wrote %s callout cache address record for %s\n", + addr_rec->result == ccache_accept ? "positive" : "negative", + address_key); + } + } - host_af = Ustrchr(host->address, ':') == NULL ? AF_INET : AF_INET6; +if (dbm_file) dbfn_close(dbm_file); +} - /* Expand and interpret the interface and port strings. The latter will not - be used if there is a host-specific port (e.g. from a manualroute router). - This has to be delayed till now, because they may expand differently for - different hosts. If there's a failure, log it, but carry on with the - defaults. */ - deliver_host = host->name; - deliver_host_address = host->address; - deliver_host_port = host->port; - deliver_domain = addr->domain; - transport_name = addr->transport->name; +/* Cutthrough-multi. If the existing cached cutthrough connection matches +the one we would make for a subsequent recipient, use it. Send the RCPT TO +and check the result, nonpipelined as it may be wanted immediately for +recipient-verification. - if ( !smtp_get_interface(tf->interface, host_af, addr, &interface, - US"callout") - || !smtp_get_port(tf->port, addr, &port, US"callout") - ) - log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address, - addr->message); +It seems simpler to deal with this case separately from the main callout loop. +We will need to remember it has sent, or not, so that rcpt-acl tail code +can do it there for the non-rcpt-verify case. For this we keep an addresscount. - /* Set HELO string according to the protocol */ - lmtp= Ustrcmp(tf->protocol, "lmtp") == 0; - smtps= Ustrcmp(tf->protocol, "smtps") == 0; +Return: TRUE for a definitive result for the recipient +*/ +static int +cutthrough_multi(address_item * addr, host_item * host_list, + transport_feedback * tf, int * yield) +{ +BOOL done = FALSE; +host_item * host; +if (addr->transport == cutthrough.addr.transport) + for (host = host_list; host; host = host->next) + if (Ustrcmp(host->address, cutthrough.host.address) == 0) + { + int host_af; + uschar *interface = NULL; /* Outgoing interface to use; NULL => any */ + int port = 25; - HDEBUG(D_verify) debug_printf("interface=%s port=%d\n", interface, port); + deliver_host = host->name; + deliver_host_address = host->address; + deliver_host_port = host->port; + deliver_domain = addr->domain; + transport_name = addr->transport->name; - /* Set up the buffer for reading SMTP response packets. */ + host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET; - inblock.buffer = inbuffer; - inblock.buffersize = sizeof(inbuffer); - inblock.ptr = inbuffer; - inblock.ptrend = inbuffer; + if ( !smtp_get_interface(tf->interface, host_af, addr, &interface, + US"callout") + || !smtp_get_port(tf->port, addr, &port, US"callout") + ) + log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address, + addr->message); - /* Set up the buffer for holding SMTP commands while pipelining */ + smtp_port_for_connect(host, port); - outblock.buffer = outbuffer; - outblock.buffersize = sizeof(outbuffer); - outblock.ptr = outbuffer; - outblock.cmd_count = 0; - outblock.authenticating = FALSE; + if ( ( interface == cutthrough.interface + || ( interface + && cutthrough.interface + && Ustrcmp(interface, cutthrough.interface) == 0 + ) ) + && host->port == cutthrough.host.port + ) + { + uschar * resp = NULL; - /* Connect to the host; on failure, just loop for the next one, but we - set the error for the last one. Use the callout_connect timeout. */ + /* Match! Send the RCPT TO, set done from the response */ + done = + smtp_write_command(&ctblock, SCMD_FLUSH, "RCPT TO:<%.1000s>\r\n", + transport_rcpt_address(addr, + addr->transport->rcpt_include_affixes)) >= 0 && + cutthrough_response(cutthrough.fd, '2', &resp, CUTTHROUGH_DATA_TIMEOUT) == '2'; - tls_retry_connection: + /* This would go horribly wrong if a callout fail was ignored by ACL. + We punt by abandoning cutthrough on a reject, like the + first-rcpt does. */ - /* Reset the parameters of a TLS session */ - tls_out.cipher = tls_out.peerdn = tls_out.peercert = NULL; + if (done) + { + address_item * na = store_get(sizeof(address_item)); + *na = cutthrough.addr; + cutthrough.addr = *addr; + cutthrough.addr.host_used = &cutthrough.host; + cutthrough.addr.next = na; - inblock.sock = outblock.sock = - smtp_connect(host, host_af, port, interface, callout_connect, - addr->transport); - if (inblock.sock < 0) - { - addr->message = string_sprintf("could not connect to %s [%s]: %s", - host->name, host->address, strerror(errno)); - transport_name = NULL; - deliver_host = deliver_host_address = NULL; - deliver_domain = save_deliver_domain; - continue; - } + cutthrough.nrcpt++; + } + else + { + cancel_cutthrough_connection(TRUE, US"recipient rejected"); + if (!resp || errno == ETIMEDOUT) + { + HDEBUG(D_verify) debug_printf("SMTP timeout\n"); + } + else if (errno == 0) + { + if (*resp == 0) + Ustrcpy(resp, US"connection dropped"); -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) - { - int rc; + addr->message = + string_sprintf("response to \"%s\" was: %s", + big_buffer, string_printing(resp)); - tls_out.dane_verified = FALSE; - tls_out.tlsa_usage = 0; + addr->user_message = + string_sprintf("Callout verification failed:\n%s", resp); - dane_required = - verify_check_given_host(&ob->hosts_require_dane, host) == OK; + /* Hard rejection ends the process */ - if (host->dnssec == DS_YES) - { - if( dane_required - || verify_check_given_host(&ob->hosts_try_dane, host) == OK - ) - { - if ((rc = tlsa_lookup(host, &tlsa_dnsa, dane_required)) != OK) - return rc; - dane = TRUE; + if (resp[0] == '5') /* Address rejected */ + { + *yield = FAIL; + done = TRUE; + } + } } } - else if (dane_required) - { - log_write(0, LOG_MAIN, "DANE error: %s lookup not DNSSEC", host->name); - return FAIL; - } - - if (dane) - ob->tls_tempfail_tryclear = FALSE; + break; /* host_list */ } -#endif /*DANE*/ - - /* Expand the helo_data string to find the host name to use. */ - - if (tf->helo_data) - { - uschar * s = expand_string(tf->helo_data); - if (!s) - log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: failed to expand transport's " - "helo_data value for callout: %s", addr->address, - expand_string_message); - else active_hostname = s; - } - - /* Wait for initial response, and send HELO. The smtp_write_command() - function leaves its command in big_buffer. This is used in error responses. - Initialize it in case the connection is rejected. */ +if (!done) + cancel_cutthrough_connection(TRUE, US"incompatible connection"); +return done; +} - Ustrcpy(big_buffer, "initial connection"); - /* Unless ssl-on-connect, wait for the initial greeting */ - smtps_redo_greeting: +/************************************************* +* Do callout verification for an address * +*************************************************/ -#ifdef SUPPORT_TLS - if (!smtps || (smtps && tls_out.active >= 0)) -#endif - { - if (!(done= smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), '2', callout))) - goto RESPONSE_FAILED; +/* This function is called from verify_address() when the address has routed to +a host list, and a callout has been requested. Callouts are expensive; that is +why a cache is used to improve the efficiency. -#ifndef DISABLE_EVENT - lookup_dnssec_authenticated = host->dnssec==DS_YES ? US"yes" - : host->dnssec==DS_NO ? US"no" : NULL; - if (event_raise(addr->transport->event_action, - US"smtp:connect", responsebuffer)) - { - lookup_dnssec_authenticated = NULL; - /* Logging? Debug? */ - goto RESPONSE_FAILED; - } - lookup_dnssec_authenticated = NULL; -#endif - } +Arguments: + addr the address that's been routed + host_list the list of hosts to try + tf the transport feedback block - /* Not worth checking greeting line for ESMTP support */ - if (!(esmtp = verify_check_given_host(&ob->hosts_avoid_esmtp, host) != OK)) - DEBUG(D_transport) - debug_printf("not sending EHLO (host matches hosts_avoid_esmtp)\n"); + ifstring "interface" option from transport, or NULL + portstring "port" option from transport, or NULL + protocolstring "protocol" option from transport, or NULL + callout the per-command callout timeout + callout_overall the overall callout timeout (if < 0 use 4*callout) + callout_connect the callout connection timeout (if < 0 use callout) + options the verification options - these bits are used: + vopt_is_recipient => this is a recipient address + vopt_callout_no_cache => don't use callout cache + vopt_callout_fullpm => if postmaster check, do full one + vopt_callout_random => do the "random" thing + vopt_callout_recipsender => use real sender for recipient + vopt_callout_recippmaster => use postmaster for recipient + vopt_callout_hold => lazy close connection + se_mailfrom MAIL FROM address for sender verify; NULL => "" + pm_mailfrom if non-NULL, do the postmaster check with this sender - tls_redo_helo: +Returns: OK/FAIL/DEFER +*/ -#ifdef SUPPORT_TLS - if (smtps && tls_out.active < 0) /* ssl-on-connect, first pass */ - { - peer_offered &= ~PEER_OFFERED_TLS; - ob->tls_tempfail_tryclear = FALSE; - } - else /* all other cases */ -#endif +static int +do_callout(address_item *addr, host_item *host_list, transport_feedback *tf, + int callout, int callout_overall, int callout_connect, int options, + uschar *se_mailfrom, uschar *pm_mailfrom) +{ +int yield = OK; +int old_domain_cache_result = ccache_accept; +BOOL done = FALSE; +uschar *address_key; +uschar *from_address; +uschar *random_local_part = NULL; +const uschar *save_deliver_domain = deliver_domain; +uschar **failure_ptr = options & vopt_is_recipient + ? &recipient_verify_failure : &sender_verify_failure; +dbdata_callout_cache new_domain_record; +dbdata_callout_cache_address new_address_record; +time_t callout_start_time; - { esmtp_retry: +new_domain_record.result = ccache_unknown; +new_domain_record.postmaster_result = ccache_unknown; +new_domain_record.random_result = ccache_unknown; - if (!(done= smtp_write_command(&outblock, FALSE, "%s %s\r\n", - !esmtp? "HELO" : lmtp? "LHLO" : "EHLO", active_hostname) >= 0)) - goto SEND_FAILED; - if (!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), '2', callout)) - { - if (errno != 0 || responsebuffer[0] == 0 || lmtp || !esmtp || tls_out.active >= 0) - { - done= FALSE; - goto RESPONSE_FAILED; - } -#ifdef SUPPORT_TLS - peer_offered &= ~PEER_OFFERED_TLS; -#endif - esmtp = FALSE; - goto esmtp_retry; /* fallback to HELO */ - } +memset(&new_address_record, 0, sizeof(new_address_record)); - /* Set tls_offered if the response to EHLO specifies support for STARTTLS. */ +/* For a recipient callout, the key used for the address cache record must +include the sender address if we are using the real sender in the callout, +because that may influence the result of the callout. */ - peer_offered = esmtp - ? ehlo_response(responsebuffer, sizeof(responsebuffer), - (!suppress_tls && tls_out.active < 0 ? PEER_OFFERED_TLS : 0) - | 0 /* no IGNQ */ - | 0 /* no PRDR */ -#ifdef SUPPORT_I18N - | (addr->prop.utf8_msg && !addr->prop.utf8_downcvt - ? PEER_OFFERED_UTF8 : 0) -#endif - | 0 /* no DSN */ - | 0 /* no PIPE */ - - /* only care about SIZE if we have size from inbound */ - | (message_size > 0 && ob->size_addition >= 0 - ? PEER_OFFERED_SIZE : 0) - ) - : 0; - } +if (options & vopt_is_recipient) + if (options & vopt_callout_recipsender) + { + from_address = sender_address; + address_key = string_sprintf("%s/<%s>", addr->address, sender_address); + if (cutthrough.delivery) options |= vopt_callout_no_cache; + } + else if (options & vopt_callout_recippmaster) + { + from_address = string_sprintf("postmaster@%s", qualify_domain_sender); + address_key = string_sprintf("%s/", addr->address, + qualify_domain_sender); + } + else + { + from_address = US""; + address_key = addr->address; + } - size_str = peer_offered & PEER_OFFERED_SIZE - ? string_sprintf(" SIZE=%d", message_size + ob->size_addition) : US""; +/* For a sender callout, we must adjust the key if the mailfrom address is not +empty. */ -#ifdef SUPPORT_TLS - smtp_peer_options |= peer_offered & PEER_OFFERED_TLS; -#endif +else + { + from_address = se_mailfrom ? se_mailfrom : US""; + address_key = *from_address + ? string_sprintf("%s/<%s>", addr->address, from_address) : addr->address; + } - /* If TLS is available on this connection attempt to - start up a TLS session, unless the host is in hosts_avoid_tls. If successful, - send another EHLO - the server may give a different answer in secure mode. We - use a separate buffer for reading the response to STARTTLS so that if it is - negative, the original EHLO data is available for subsequent analysis, should - the client not be required to use TLS. If the response is bad, copy the buffer - for error analysis. */ +if (cached_callout_lookup(addr, address_key, from_address, + &options, &pm_mailfrom, &yield, failure_ptr, + &new_domain_record, &old_domain_cache_result)) + { + cancel_cutthrough_connection(TRUE, US"cache-hit"); + goto END_CALLOUT; + } -#ifdef SUPPORT_TLS - if ( peer_offered & PEER_OFFERED_TLS - && verify_check_given_host(&ob->hosts_avoid_tls, host) != OK - && verify_check_given_host(&ob->hosts_verify_avoid_tls, host) != OK - ) - { - uschar buffer2[4096]; - if ( !smtps - && !(done= smtp_write_command(&outblock, FALSE, "STARTTLS\r\n") >= 0)) - goto SEND_FAILED; - - /* If there is an I/O error, transmission of this message is deferred. If - there is a temporary rejection of STARRTLS and tls_tempfail_tryclear is - false, we also defer. However, if there is a temporary rejection of STARTTLS - and tls_tempfail_tryclear is true, or if there is an outright rejection of - STARTTLS, we carry on. This means we will try to send the message in clear, - unless the host is in hosts_require_tls (tested below). */ - - if (!smtps && !smtp_read_response(&inblock, buffer2, sizeof(buffer2), '2', - ob->command_timeout)) - { - if ( errno != 0 - || buffer2[0] == 0 - || buffer2[0] == '4' && !ob->tls_tempfail_tryclear - ) - { - Ustrncpy(responsebuffer, buffer2, sizeof(responsebuffer)); - done= FALSE; - goto RESPONSE_FAILED; - } - } +if (!addr->transport) + { + HDEBUG(D_verify) debug_printf("cannot callout via null transport\n"); + } +else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0) + log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp", + addr->transport->name, addr->transport->driver_name); +else + { + smtp_transport_options_block *ob = + (smtp_transport_options_block *)addr->transport->options_block; + host_item * host; - /* STARTTLS accepted or ssl-on-connect: try to negotiate a TLS session. */ - else - { - int oldtimeout = ob->command_timeout; - int rc; - - tls_negotiate: - ob->command_timeout = callout; - rc = tls_client_start(inblock.sock, host, addr, addr->transport -# ifdef EXPERIMENTAL_DANE - , dane ? &tlsa_dnsa : NULL -# endif - ); - ob->command_timeout = oldtimeout; - - /* TLS negotiation failed; give an error. Try in clear on a new - connection, if the options permit it for this host. */ - if (rc != OK) - { - if (rc == DEFER) - { - (void)close(inblock.sock); -# ifndef DISABLE_EVENT - (void) event_raise(addr->transport->event_action, - US"tcp:close", NULL); -# endif - if ( ob->tls_tempfail_tryclear - && !smtps - && verify_check_given_host(&ob->hosts_require_tls, host) != OK - ) - { - log_write(0, LOG_MAIN, "TLS session failure:" - " delivering unencrypted to %s [%s] (not in hosts_require_tls)", - host->name, host->address); - suppress_tls = TRUE; - goto tls_retry_connection; - } - } + /* The information wasn't available in the cache, so we have to do a real + callout and save the result in the cache for next time, unless no_cache is set, + or unless we have a previously cached negative random result. If we are to test + with a random local part, ensure that such a local part is available. If not, + log the fact, but carry on without randomising. */ - /*save_errno = ERRNO_TLSFAILURE;*/ - /*message = US"failure while setting up TLS session";*/ - send_quit = FALSE; - done= FALSE; - goto TLS_FAILED; - } + if (options & vopt_callout_random && callout_random_local_part) + if (!(random_local_part = expand_string(callout_random_local_part))) + log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand " + "callout_random_local_part: %s", expand_string_message); - /* TLS session is set up. Copy info for logging. */ - addr->cipher = tls_out.cipher; - addr->peerdn = tls_out.peerdn; + /* Default the connect and overall callout timeouts if not set, and record the + time we are starting so that we can enforce it. */ - /* For SMTPS we need to wait for the initial OK response, then do HELO. */ - if (smtps) - goto smtps_redo_greeting; + if (callout_overall < 0) callout_overall = 4 * callout; + if (callout_connect < 0) callout_connect = callout; + callout_start_time = time(NULL); - /* For STARTTLS we need to redo EHLO */ - goto tls_redo_helo; - } - } + /* Before doing a real callout, if this is an SMTP connection, flush the SMTP + output because a callout might take some time. When PIPELINING is active and + there are many recipients, the total time for doing lots of callouts can add up + and cause the client to time out. So in this case we forgo the PIPELINING + optimization. */ - /* If the host is required to use a secure channel, ensure that we have one. */ - if (tls_out.active < 0) - if ( -# ifdef EXPERIMENTAL_DANE - dane || -# endif - verify_check_given_host(&ob->hosts_require_tls, host) == OK - ) - { - /*save_errno = ERRNO_TLSREQUIRED;*/ - log_write(0, LOG_MAIN, - "H=%s [%s]: a TLS session is required for this host, but %s", - host->name, host->address, - peer_offered & PEER_OFFERED_TLS - ? "an attempt to start TLS failed" - : "the server did not offer TLS support"); - done= FALSE; - goto TLS_FAILED; - } + if (smtp_out && !disable_callout_flush) mac_smtp_fflush(); -#endif /*SUPPORT_TLS*/ + clearflag(addr, af_verify_pmfail); /* postmaster callout flag */ + clearflag(addr, af_verify_nsfail); /* null sender callout flag */ - done = TRUE; /* so far so good; have response to HELO */ +/* cutthrough-multi: if a nonfirst rcpt has the same routing as the first, +and we are holding a cutthrough conn open, we can just append the rcpt to +that conn for verification purposes (and later delivery also). Simplest +coding means skipping this whole loop and doing the append separately. */ - /* For now, transport_filter by cutthrough-delivery is not supported */ - /* Need proper integration with the proper transport mechanism. */ - if (cutthrough.delivery) - { -#ifndef DISABLE_DKIM - uschar * s; -#endif - if (addr->transport->filter_command) - { - cutthrough.delivery = FALSE; - HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n"); - } -#ifndef DISABLE_DKIM - else if ((s = ob->dkim.dkim_domain) && (s = expand_string(s)) && *s) - { - cutthrough.delivery = FALSE; - HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n"); - } -#endif - } + /* Can we re-use an open cutthrough connection? */ + if ( cutthrough.fd >= 0 + && (options & (vopt_callout_recipsender | vopt_callout_recippmaster)) + == vopt_callout_recipsender + && !random_local_part + && !pm_mailfrom + ) + done = cutthrough_multi(addr, host_list, tf, &yield); - SEND_FAILED: - RESPONSE_FAILED: - TLS_FAILED: - ; - /* Clear down of the TLS, SMTP and TCP layers on error is handled below. */ + /* If we did not use a cached connection, make connections to the hosts + and do real callouts. The list of hosts is passed in as an argument. */ - /* Failure to accept HELO is cached; this blocks the whole domain for all - senders. I/O errors and defer responses are not cached. */ + for (host = host_list; host && !done; host = host->next) + { + int host_af; + int port = 25; + uschar *interface = NULL; /* Outgoing interface to use; NULL => any */ + smtp_context sx; - if (!done) + if (!host->address) { - *failure_ptr = US"mail"; /* At or before MAIL */ - if (errno == 0 && responsebuffer[0] == '5') - { - setflag(addr, af_verify_nsfail); - new_domain_record.result = ccache_reject; - } + DEBUG(D_verify) debug_printf("no IP address for host name %s: skipping\n", + host->name); + continue; } -#ifdef SUPPORT_I18N - else if ( addr->prop.utf8_msg - && !addr->prop.utf8_downcvt - && !(peer_offered & PEER_OFFERED_UTF8) - ) - { - HDEBUG(D_acl|D_v) debug_printf("utf8 required but not offered\n"); - errno = ERRNO_UTF8_FWD; - setflag(addr, af_verify_nsfail); - done = FALSE; - } - else if ( addr->prop.utf8_msg - && (addr->prop.utf8_downcvt || !(peer_offered & PEER_OFFERED_UTF8)) - && (setflag(addr, af_utf8_downcvt), - from_address = string_address_utf8_to_alabel(from_address, - &addr->message), - addr->message - ) ) + /* Check the overall callout timeout */ + + if (time(NULL) - callout_start_time >= callout_overall) { - errno = ERRNO_EXPANDFAIL; - setflag(addr, af_verify_nsfail); - done = FALSE; + HDEBUG(D_verify) debug_printf("overall timeout for callout exceeded\n"); + break; } -#endif - /* If we haven't authenticated, but are required to, give up. */ - /* Try to AUTH */ + /* Set IPv4 or IPv6 */ - else done = smtp_auth(responsebuffer, sizeof(responsebuffer), - addr, host, ob, esmtp, &inblock, &outblock) == OK && + host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET; - /* Copy AUTH info for logging */ - ( (addr->authenticator = client_authenticator), - (addr->auth_id = client_authenticated_id), + /* Expand and interpret the interface and port strings. The latter will not + be used if there is a host-specific port (e.g. from a manualroute router). + This has to be delayed till now, because they may expand differently for + different hosts. If there's a failure, log it, but carry on with the + defaults. */ - /* Build a mail-AUTH string (re-using responsebuffer for convenience */ - !smtp_mail_auth_str(responsebuffer, sizeof(responsebuffer), addr, ob) - ) && + deliver_host = host->name; + deliver_host_address = host->address; + deliver_host_port = host->port; + deliver_domain = addr->domain; + transport_name = addr->transport->name; - ( (addr->auth_sndr = client_authenticated_sender), + if ( !smtp_get_interface(tf->interface, host_af, addr, &interface, + US"callout") + || !smtp_get_port(tf->port, addr, &port, US"callout") + ) + log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address, + addr->message); - /* Send the MAIL command */ - (smtp_write_command(&outblock, FALSE, -#ifdef SUPPORT_I18N - addr->prop.utf8_msg && !addr->prop.utf8_downcvt - ? "MAIL FROM:<%s>%s%s SMTPUTF8\r\n" - : -#endif - "MAIL FROM:<%s>%s%s\r\n", - from_address, responsebuffer, size_str) >= 0) - ) && + sx.addrlist = addr; + sx.host = host; + sx.host_af = host_af, + sx.port = port; + sx.interface = interface; + sx.helo_data = tf->helo_data; + sx.tblock = addr->transport; + sx.verify = TRUE; - smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), - '2', callout); +tls_retry_connection: + /* Set the address state so that errors are recorded in it */ - deliver_host = deliver_host_address = NULL; - deliver_domain = save_deliver_domain; + addr->transport_return = PENDING_DEFER; + ob->connect_timeout = callout_connect; + ob->command_timeout = callout; - /* If the host does not accept MAIL FROM:<>, arrange to cache this - information, but again, don't record anything for an I/O error or a defer. Do - not cache rejections of MAIL when a non-empty sender has been used, because - that blocks the whole domain for all senders. */ + /* Get the channel set up ready for a message (MAIL FROM being the next + SMTP command to send. If we tried TLS but it failed, try again without + if permitted */ - if (!done) + yield = smtp_setup_conn(&sx, FALSE); +#ifdef SUPPORT_TLS + if ( yield == DEFER + && addr->basic_errno == ERRNO_TLSFAILURE + && ob->tls_tempfail_tryclear + && verify_check_given_host(&ob->hosts_require_tls, host) != OK + ) { - *failure_ptr = US"mail"; /* At or before MAIL */ - if (errno == 0 && responsebuffer[0] == '5') - { - setflag(addr, af_verify_nsfail); - if (from_address[0] == 0) - new_domain_record.result = ccache_reject_mfnull; - } + log_write(0, LOG_MAIN, + "%s: callout unencrypted to %s [%s] (not in hosts_require_tls)", + addr->message, host->name, host->address); + addr->transport_return = PENDING_DEFER; + yield = smtp_setup_conn(&sx, TRUE); + } +#endif + if (yield != OK) + { + errno = addr->basic_errno; + transport_name = NULL; + deliver_host = deliver_host_address = NULL; + deliver_domain = save_deliver_domain; + + /* Failure to accept HELO is cached; this blocks the whole domain for all + senders. I/O errors and defer responses are not cached. */ + + if (yield == FAIL && (errno == 0 || errno == ERRNO_SMTPCLOSED)) + { + setflag(addr, af_verify_nsfail); + new_domain_record.result = ccache_reject; + done = TRUE; + } + else + done = FALSE; + goto no_conn; } - /* Otherwise, proceed to check a "random" address (if required), then the - given address, and the postmaster address (if required). Between each check, - issue RSET, because some servers accept only one recipient after MAIL - FROM:<>. + /* If we needed to authenticate, smtp_setup_conn() did that. Copy + the AUTH info for logging */ - Before doing this, set the result in the domain cache record to "accept", - unless its previous value was ccache_reject_mfnull. In that case, the domain - rejects MAIL FROM:<> and we want to continue to remember that. When that is - the case, we have got here only in the case of a recipient verification with - a non-null sender. */ + addr->authenticator = client_authenticator; + addr->auth_id = client_authenticated_id; - else + sx.from_addr = from_address; + sx.first_addr = sx.sync_addr = addr; + sx.ok = FALSE; /*XXX these 3 last might not be needed for verify? */ + sx.send_rset = TRUE; + sx.completed_addr = FALSE; + + new_domain_record.result = old_domain_cache_result == ccache_reject_mfnull + ? ccache_reject_mfnull : ccache_accept; + + /* Do the random local part check first. Temporarily replace the recipient + with the "random" value */ + + if (random_local_part) { + uschar * main_address = addr->address; const uschar * rcpt_domain = addr->domain; #ifdef SUPPORT_I18N @@ -1029,181 +761,200 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. } #endif - new_domain_record.result = - (old_domain_cache_result == ccache_reject_mfnull)? - ccache_reject_mfnull: ccache_accept; - - /* Do the random local part check first */ + /* This would be ok for 1st rcpt of a cutthrough (the case handled here; + subsequents are done in cutthrough_multi()), but no way to + handle a subsequent because of the RSET vaporising the MAIL FROM. + So refuse to support any. Most cutthrough use will not involve + random_local_part, so no loss. */ + cancel_cutthrough_connection(TRUE, US"random-recipient"); - if (random_local_part != NULL) - { - uschar randombuffer[1024]; - BOOL random_ok = - smtp_write_command(&outblock, FALSE, - "RCPT TO:<%.1000s@%.1000s>\r\n", random_local_part, - rcpt_domain) >= 0 && - smtp_read_response(&inblock, randombuffer, - sizeof(randombuffer), '2', callout); - - /* Remember when we last did a random test */ - - new_domain_record.random_stamp = time(NULL); - - /* If accepted, we aren't going to do any further tests below. */ - - if (random_ok) - new_domain_record.random_result = ccache_accept; - - /* Otherwise, cache a real negative response, and get back to the right - state to send RCPT. Unless there's some problem such as a dropped - connection, we expect to succeed, because the commands succeeded above. - However, some servers drop the connection after responding to an - invalid recipient, so on (any) error we drop and remake the connection. - */ + addr->address = string_sprintf("%s@%.1000s", + random_local_part, rcpt_domain); + done = FALSE; - else if (errno == 0) - { - /* This would be ok for 1st rcpt a cutthrough, but no way to - handle a subsequent. So refuse to support any */ - cancel_cutthrough_connection("random-recipient"); + /* If accepted, we aren't going to do any further tests below. + Otherwise, cache a real negative response, and get back to the right + state to send RCPT. Unless there's some problem such as a dropped + connection, we expect to succeed, because the commands succeeded above. + However, some servers drop the connection after responding to an + invalid recipient, so on (any) error we drop and remake the connection. + XXX We don't care about that for postmaster_full. Should we? + + XXX could we add another flag to the context, and have the common + code emit the RSET too? Even pipelined after the RCPT... + Then the main-verify call could use it if there's to be a subsequent + postmaster-verify. + The sync_responses() would need to be taught about it and we'd + need another return code filtering out to here. + + Avoid using a SIZE option on the MAIL for all randon-rcpt checks. + */ - if (randombuffer[0] == '5') - new_domain_record.random_result = ccache_reject; + sx.avoid_option = OPTION_SIZE; - done = - smtp_write_command(&outblock, FALSE, "RSET\r\n") >= 0 && - smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), - '2', callout) && + /* Remember when we last did a random test */ + new_domain_record.random_stamp = time(NULL); - smtp_write_command(&outblock, FALSE, -#ifdef SUPPORT_I18N - addr->prop.utf8_msg && !addr->prop.utf8_downcvt - ? "MAIL FROM:<%s> SMTPUTF8\r\n" - : -#endif - "MAIL FROM:<%s>\r\n", - from_address) >= 0 && - smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), - '2', callout); + if (smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0) + switch(addr->transport_return) + { + case PENDING_OK: /* random was accepted, unfortunately */ + new_domain_record.random_result = ccache_accept; + yield = OK; /* Only usable verify result we can return */ + done = TRUE; + goto no_conn; + case FAIL: /* rejected: the preferred result */ + new_domain_record.random_result = ccache_reject; + sx.avoid_option = 0; + + /* Between each check, issue RSET, because some servers accept only + one recipient after MAIL FROM:<>. + XXX We don't care about that for postmaster_full. Should we? */ + + if ((done = + smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0 && + smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer), + '2', callout))) + break; - if (!done) - { HDEBUG(D_acl|D_v) - debug_printf("problem after random/rset/mfrom; reopen conn\n"); + debug_printf_indent("problem after random/rset/mfrom; reopen conn\n"); random_local_part = NULL; #ifdef SUPPORT_TLS - tls_close(FALSE, TRUE); + tls_close(FALSE, TLS_SHUTDOWN_NOWAIT); #endif - (void)close(inblock.sock); + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); + (void)close(sx.inblock.sock); + sx.inblock.sock = sx.outblock.sock = -1; #ifndef DISABLE_EVENT (void) event_raise(addr->transport->event_action, US"tcp:close", NULL); #endif + addr->address = main_address; + addr->transport_return = PENDING_DEFER; + sx.first_addr = sx.sync_addr = addr; + sx.ok = FALSE; + sx.send_rset = TRUE; + sx.completed_addr = FALSE; goto tls_retry_connection; - } - } - else done = FALSE; /* Some timeout/connection problem */ - } /* Random check */ + case DEFER: /* 4xx response to random */ + break; /* Just to be clear. ccache_unknown, !done. */ + } - /* If the host is accepting all local parts, as determined by the "random" - check, we don't need to waste time doing any further checking. */ + /* Re-setup for main verify, or for the error message when failing */ + addr->address = main_address; + addr->transport_return = PENDING_DEFER; + sx.first_addr = sx.sync_addr = addr; + sx.ok = FALSE; + sx.send_rset = TRUE; + sx.completed_addr = FALSE; + } + else + done = TRUE; - if (new_domain_record.random_result != ccache_accept && done) - { - /* Get the rcpt_include_affixes flag from the transport if there is one, - but assume FALSE if there is not. */ + /* Main verify. For rcpt-verify use SIZE if we know it and we're not cacheing; + for sndr-verify never use it. */ - uschar * rcpt = transport_rcpt_address(addr, - addr->transport ? addr->transport->rcpt_include_affixes : FALSE); + if (done) + { + if (!(options & vopt_is_recipient && options & vopt_callout_no_cache)) + sx.avoid_option = OPTION_SIZE; -#ifdef SUPPORT_I18N - /*XXX should the conversion be moved into transport_rcpt_address() ? */ - uschar * dummy_errstr = NULL; - if ( testflag(addr, af_utf8_downcvt) - && (rcpt = string_address_utf8_to_alabel(rcpt, &dummy_errstr), - dummy_errstr - ) ) + done = FALSE; + switch(smtp_write_mail_and_rcpt_cmds(&sx, &yield)) { - errno = ERRNO_EXPANDFAIL; - *failure_ptr = US"recipient"; - done = FALSE; + case 0: switch(addr->transport_return) /* ok so far */ + { + case PENDING_OK: done = TRUE; + new_address_record.result = ccache_accept; + break; + case FAIL: done = TRUE; + yield = FAIL; + *failure_ptr = US"recipient"; + new_address_record.result = ccache_reject; + break; + default: break; + } + break; + + case -1: /* MAIL response error */ + *failure_ptr = US"mail"; + if (errno == 0 && sx.buffer[0] == '5') + { + setflag(addr, af_verify_nsfail); + if (from_address[0] == 0) + new_domain_record.result = ccache_reject_mfnull; + } + break; + /* non-MAIL read i/o error */ + /* non-MAIL response timeout */ + /* internal error; channel still usable */ + default: break; /* transmit failed */ } - else -#endif + } - done = - smtp_write_command(&outblock, FALSE, "RCPT TO:<%.1000s>\r\n", - rcpt) >= 0 && - smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), - '2', callout); + addr->auth_sndr = client_authenticated_sender; - if (done) - new_address_record.result = ccache_accept; - else if (errno == 0 && responsebuffer[0] == '5') - { - *failure_ptr = US"recipient"; - new_address_record.result = ccache_reject; - } + deliver_host = deliver_host_address = NULL; + deliver_domain = save_deliver_domain; - /* Do postmaster check if requested; if a full check is required, we - check for RCPT TO: (no domain) in accordance with RFC 821. */ + /* Do postmaster check if requested; if a full check is required, we + check for RCPT TO: (no domain) in accordance with RFC 821. */ - if (done && pm_mailfrom != NULL) - { - /* Could possibly shift before main verify, just above, and be ok - for cutthrough. But no way to handle a subsequent rcpt, so just - refuse any */ - cancel_cutthrough_connection("postmaster verify"); - HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of postmaster verify\n"); - - done = - smtp_write_command(&outblock, FALSE, "RSET\r\n") >= 0 && - smtp_read_response(&inblock, responsebuffer, - sizeof(responsebuffer), '2', callout) && - - smtp_write_command(&outblock, FALSE, - "MAIL FROM:<%s>\r\n", pm_mailfrom) >= 0 && - smtp_read_response(&inblock, responsebuffer, - sizeof(responsebuffer), '2', callout) && - - /* First try using the current domain */ - - (( - smtp_write_command(&outblock, FALSE, - "RCPT TO:\r\n", rcpt_domain) >= 0 && - smtp_read_response(&inblock, responsebuffer, - sizeof(responsebuffer), '2', callout) - ) - - || - - /* If that doesn't work, and a full check is requested, - try without the domain. */ - - ( - (options & vopt_callout_fullpm) != 0 && - smtp_write_command(&outblock, FALSE, - "RCPT TO:\r\n") >= 0 && - smtp_read_response(&inblock, responsebuffer, - sizeof(responsebuffer), '2', callout) - )); - - /* Sort out the cache record */ - - new_domain_record.postmaster_stamp = time(NULL); - - if (done) - new_domain_record.postmaster_result = ccache_accept; - else if (errno == 0 && responsebuffer[0] == '5') - { - *failure_ptr = US"postmaster"; - setflag(addr, af_verify_pmfail); - new_domain_record.postmaster_result = ccache_reject; - } - } - } /* Random not accepted */ - } /* MAIL FROM: accepted */ + if (done && pm_mailfrom) + { + /* Could possibly shift before main verify, just above, and be ok + for cutthrough. But no way to handle a subsequent rcpt, so just + refuse any */ + cancel_cutthrough_connection(TRUE, US"postmaster verify"); + HDEBUG(D_acl|D_v) debug_printf_indent("Cutthrough cancelled by presence of postmaster verify\n"); + + done = smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0 + && smtp_read_response(&sx.inblock, sx.buffer, + sizeof(sx.buffer), '2', callout); + + if (done) + { + uschar * main_address = addr->address; + /*XXX oops, affixes */ + addr->address = string_sprintf("postmaster@%.1000s", addr->domain); + addr->transport_return = PENDING_DEFER; + + sx.from_addr = pm_mailfrom; + sx.first_addr = sx.sync_addr = addr; + sx.ok = FALSE; + sx.send_rset = TRUE; + sx.completed_addr = FALSE; + sx.avoid_option = OPTION_SIZE; + + if( smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0 + && addr->transport_return == PENDING_OK + ) + done = TRUE; + else + done = (options & vopt_callout_fullpm) != 0 + && smtp_write_command(&sx.outblock, SCMD_FLUSH, + "RCPT TO:\r\n") >= 0 + && smtp_read_response(&sx.inblock, sx.buffer, + sizeof(sx.buffer), '2', callout); + + /* Sort out the cache record */ + + new_domain_record.postmaster_stamp = time(NULL); + + if (done) + new_domain_record.postmaster_result = ccache_accept; + else if (errno == 0 && sx.buffer[0] == '5') + { + *failure_ptr = US"postmaster"; + setflag(addr, af_verify_pmfail); + new_domain_record.postmaster_result = ccache_reject; + } + + addr->address = main_address; + } + } /* For any failure of the main check, other than a negative response, we just close the connection and carry on. We can identify a negative response by the fact that errno is zero. For I/O errors it will be non-zero @@ -1214,58 +965,91 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. don't give the IP address because this may be an internal host whose identity is not to be widely broadcast. */ - if (!done) +no_conn: + switch(errno) { - if (errno == ETIMEDOUT) - { - HDEBUG(D_verify) debug_printf("SMTP timeout\n"); - send_quit = FALSE; - } + case ETIMEDOUT: + HDEBUG(D_verify) debug_printf("SMTP timeout\n"); + sx.send_quit = FALSE; + break; + #ifdef SUPPORT_I18N - else if (errno == ERRNO_UTF8_FWD) + case ERRNO_UTF8_FWD: { extern int acl_where; /* src/acl.c */ errno = 0; addr->message = string_sprintf( - "response to \"%s\" from %s [%s] did not include SMTPUTF8", - big_buffer, host->name, host->address); - addr->user_message = acl_where == ACL_WHERE_RCPT - ? US"533 mailbox name not allowed" + "response to \"EHLO\" did not include SMTPUTF8"); + addr->user_message = acl_where == ACL_WHERE_RCPT + ? US"533 no support for internationalised mailbox name" : US"550 mailbox unavailable"; yield = FAIL; done = TRUE; } + break; #endif - else if (errno == 0) - { - if (*responsebuffer == 0) Ustrcpy(responsebuffer, US"connection dropped"); + case ECONNREFUSED: + sx.send_quit = FALSE; + break; - addr->message = - string_sprintf("response to \"%s\" from %s [%s] was: %s", - big_buffer, host->name, host->address, - string_printing(responsebuffer)); + case 0: + if (*sx.buffer == 0) Ustrcpy(sx.buffer, US"connection dropped"); - addr->user_message = options & vopt_is_recipient - ? string_sprintf("Callout verification failed:\n%s", responsebuffer) - : string_sprintf("Called: %s\nSent: %s\nResponse: %s", - host->address, big_buffer, responsebuffer); + /*XXX test here is ugly; seem to have a split of responsibility for + building this message. Need to rationalise. Where is it done + before here, and when not? + Not == 5xx resp to MAIL on main-verify + */ + if (!addr->message) addr->message = + string_sprintf("response to \"%s\" was: %s", + big_buffer, string_printing(sx.buffer)); - /* Hard rejection ends the process */ + addr->user_message = options & vopt_is_recipient + ? string_sprintf("Callout verification failed:\n%s", sx.buffer) + : string_sprintf("Called: %s\nSent: %s\nResponse: %s", + host->address, big_buffer, sx.buffer); - if (responsebuffer[0] == '5') /* Address rejected */ - { - yield = FAIL; - done = TRUE; - } - } + /* Hard rejection ends the process */ + + if (sx.buffer[0] == '5') /* Address rejected */ + { + yield = FAIL; + done = TRUE; + } + break; } /* End the SMTP conversation and close the connection. */ - /* Cutthrough - on a successfull connect and recipient-verify with + /* Cutthrough - on a successful connect and recipient-verify with use-sender and we are 1st rcpt and have no cutthrough conn so far - here is where we want to leave the conn open */ - if ( cutthrough.delivery + here is where we want to leave the conn open. Ditto for a lazy-close + verify. */ + + if (cutthrough.delivery) + { + if (addr->transport->filter_command) + { + cutthrough.delivery= FALSE; + HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n"); + } +#ifndef DISABLE_DKIM + if (ob->dkim.dkim_domain) + { + cutthrough.delivery= FALSE; + HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n"); + } +#endif +#ifdef EXPERIMENTAL_ARC + if (ob->arc_sign) + { + cutthrough.delivery= FALSE; + HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of ARC signing\n"); + } +#endif + } + + if ( (cutthrough.delivery || options & vopt_callout_hold) && rcpt_count == 1 && done && yield == OK @@ -1274,17 +1058,32 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. && !random_local_part && !pm_mailfrom && cutthrough.fd < 0 - && !lmtp + && !sx.lmtp ) { - HDEBUG(D_acl|D_v) debug_printf("holding verify callout open for cutthrough delivery\n"); - - cutthrough.fd = outblock.sock; /* We assume no buffer in use in the outblock */ - cutthrough.nrcpt = 1; - cutthrough.interface = interface; - cutthrough.host = *host; - cutthrough.addr = *addr; /* Save the address_item for later logging */ - cutthrough.addr.next = NULL; + HDEBUG(D_acl|D_v) debug_printf_indent("holding verify callout open for %s\n", + cutthrough.delivery + ? "cutthrough delivery" : "potential further verifies and delivery"); + + cutthrough.callout_hold_only = !cutthrough.delivery; + cutthrough.is_tls = tls_out.active >= 0; + cutthrough.fd = sx.outblock.sock; /* We assume no buffer in use in the outblock */ + cutthrough.nrcpt = 1; + cutthrough.transport = addr->transport->name; + cutthrough.interface = interface; + cutthrough.snd_port = sending_port; + cutthrough.peer_options = smtp_peer_options; + cutthrough.host = *host; + { + int oldpool = store_pool; + store_pool = POOL_PERM; + cutthrough.snd_ip = string_copy(sending_ip_address); + cutthrough.host.name = string_copy(host->name); + cutthrough.host.address = string_copy(host->address); + store_pool = oldpool; + } + cutthrough.addr = *addr; /* Save the address_item for later logging */ + cutthrough.addr.next = NULL; cutthrough.addr.host_used = &cutthrough.host; if (addr->parent) *(cutthrough.addr.parent = store_get(sizeof(address_item))) = @@ -1297,92 +1096,59 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. } else { - /* Ensure no cutthrough on multiple address verifies */ + /* Ensure no cutthrough on multiple verifies that were incompatible */ if (options & vopt_callout_recipsender) - cancel_cutthrough_connection("not usable for cutthrough"); - if (send_quit) - (void) smtp_write_command(&outblock, FALSE, "QUIT\r\n"); + cancel_cutthrough_connection(TRUE, US"not usable for cutthrough"); + if (sx.send_quit) + { + (void) smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n"); + + /* Wait a short time for response, and discard it */ + smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer), + '2', 1); + } + if (sx.inblock.sock >= 0) + { #ifdef SUPPORT_TLS - tls_close(FALSE, TRUE); + tls_close(FALSE, TLS_SHUTDOWN_NOWAIT); #endif - (void)close(inblock.sock); + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); + (void)close(sx.inblock.sock); + sx.inblock.sock = sx.outblock.sock = -1; #ifndef DISABLE_EVENT - (void) event_raise(addr->transport->event_action, US"tcp:close", NULL); + (void) event_raise(addr->transport->event_action, US"tcp:close", NULL); #endif + } } + if (!done || yield != OK) + addr->message = string_sprintf("%s [%s] : %s", host->name, host->address, + addr->message); } /* Loop through all hosts, while !done */ } /* If we get here with done == TRUE, a successful callout happened, and yield will be set OK or FAIL according to the response to the RCPT command. Otherwise, we looped through the hosts but couldn't complete the business. -However, there may be domain-specific information to cache in both cases. - -The value of the result field in the new_domain record is ccache_unknown if -there was an error before or with MAIL FROM:, and errno was not zero, -implying some kind of I/O error. We don't want to write the cache in that case. -Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */ - -if ( !(options & vopt_callout_no_cache) - && new_domain_record.result != ccache_unknown) - { - if ((dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE)) - == NULL) - { - HDEBUG(D_verify) debug_printf("callout cache: not available\n"); - } - else - { - (void)dbfn_write(dbm_file, addr->domain, &new_domain_record, - (int)sizeof(dbdata_callout_cache)); - HDEBUG(D_verify) debug_printf("wrote callout cache domain record for %s:\n" - " result=%d postmaster=%d random=%d\n", - addr->domain, - new_domain_record.result, - new_domain_record.postmaster_result, - new_domain_record.random_result); - } - } - -/* If a definite result was obtained for the callout, cache it unless caching -is disabled. */ +However, there may be domain-specific information to cache in both cases. */ -if (done) - { - if ( !(options & vopt_callout_no_cache) - && new_address_record.result != ccache_unknown) - { - if (!dbm_file) - dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE); - if (!dbm_file) - { - HDEBUG(D_verify) debug_printf("no callout cache available\n"); - } - else - { - (void)dbfn_write(dbm_file, address_key, &new_address_record, - (int)sizeof(dbdata_callout_cache_address)); - HDEBUG(D_verify) debug_printf("wrote %s callout cache address record for %s\n", - new_address_record.result == ccache_accept ? "positive" : "negative", - address_key); - } - } - } /* done */ +if (!(options & vopt_callout_no_cache)) + cache_callout_write(&new_domain_record, addr->domain, + done, &new_address_record, address_key); /* Failure to connect to any host, or any response other than 2xx or 5xx is a temporary error. If there was only one host, and a response was received, leave it alone if supplying details. Otherwise, give a generic response. */ -else /* !done */ +if (!done) { uschar * dullmsg = string_sprintf("Could not complete %s verify callout", options & vopt_is_recipient ? "recipient" : "sender"); yield = DEFER; - if (host_list->next || !addr->message) - addr->message = dullmsg; + addr->message = host_list->next || !addr->message + ? dullmsg : string_sprintf("%s: %s", dullmsg, addr->message); addr->user_message = smtp_return_error_details ? string_sprintf("%s for <%s>.\n" @@ -1404,7 +1170,7 @@ else /* !done */ /* Come here from within the cache-reading code on fast-track exit. */ END_CALLOUT: -if (dbm_file) dbfn_close(dbm_file); +tls_modify_variables(&tls_in); return yield; } @@ -1424,7 +1190,7 @@ int rc; get rewritten. */ addr2 = *addr; -HDEBUG(D_acl) debug_printf("----------- %s cutthrough setup ------------\n", +HDEBUG(D_acl) debug_printf_indent("----------- %s cutthrough setup ------------\n", rcpt_count > 1 ? "more" : "start"); rc = verify_address(&addr2, NULL, vopt_is_recipient | vopt_callout_recipsender | vopt_callout_no_cache, @@ -1432,7 +1198,7 @@ rc = verify_address(&addr2, NULL, NULL, NULL, NULL); addr->message = addr2.message; addr->user_message = addr2.user_message; -HDEBUG(D_acl) debug_printf("----------- end cutthrough setup ------------\n"); +HDEBUG(D_acl) debug_printf_indent("----------- end cutthrough setup ------------\n"); return rc; } @@ -1447,7 +1213,7 @@ if(cutthrough.fd < 0) if( #ifdef SUPPORT_TLS - (tls_out.active == cutthrough.fd) ? tls_write(FALSE, ctblock.buffer, n) : + tls_out.active == cutthrough.fd ? tls_write(FALSE, ctblock.buffer, n, FALSE) : #endif send(cutthrough.fd, ctblock.buffer, n, 0) > 0 ) @@ -1457,7 +1223,7 @@ if( return TRUE; } -HDEBUG(D_transport|D_acl) debug_printf("cutthrough_send failed: %s\n", strerror(errno)); +HDEBUG(D_transport|D_acl) debug_printf_indent("cutthrough_send failed: %s\n", strerror(errno)); return FALSE; } @@ -1478,20 +1244,27 @@ return TRUE; } /* Buffered output of counted data block. Return boolean success */ -BOOL +static BOOL cutthrough_puts(uschar * cp, int n) { if (cutthrough.fd < 0) return TRUE; if (_cutthrough_puts(cp, n)) return TRUE; -cancel_cutthrough_connection("transmit failed"); +cancel_cutthrough_connection(TRUE, US"transmit failed"); return FALSE; } +void +cutthrough_data_puts(uschar * cp, int n) +{ +if (cutthrough.delivery) (void) cutthrough_puts(cp, n); +return; +} + static BOOL _cutthrough_flush_send(void) { -int n= ctblock.ptr-ctblock.buffer; +int n = ctblock.ptr - ctblock.buffer; if(n>0) if(!cutthrough_send(n)) @@ -1505,21 +1278,28 @@ BOOL cutthrough_flush_send(void) { if (_cutthrough_flush_send()) return TRUE; -cancel_cutthrough_connection("transmit failed"); +cancel_cutthrough_connection(TRUE, US"transmit failed"); return FALSE; } -BOOL +static BOOL cutthrough_put_nl(void) { return cutthrough_puts(US"\r\n", 2); } +void +cutthrough_data_put_nl(void) +{ +cutthrough_data_puts(US"\r\n", 2); +} + + /* Get and check response from cutthrough target */ static uschar -cutthrough_response(char expect, uschar ** copy) +cutthrough_response(int fd, char expect, uschar ** copy, int timeout) { smtp_inblock inblock; uschar inbuffer[4096]; @@ -1529,12 +1309,12 @@ inblock.buffer = inbuffer; inblock.buffersize = sizeof(inbuffer); inblock.ptr = inbuffer; inblock.ptrend = inbuffer; -inblock.sock = cutthrough.fd; +inblock.sock = fd; /* this relies on (inblock.sock == tls_out.active) */ -if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, CUTTHROUGH_DATA_TIMEOUT)) - cancel_cutthrough_connection("target timeout on read"); +if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, timeout)) + cancel_cutthrough_connection(TRUE, US"target timeout on read"); -if(copy != NULL) +if(copy) { uschar * cp; *copy = cp = string_copy(responsebuffer); @@ -1552,21 +1332,21 @@ return responsebuffer[0]; BOOL cutthrough_predata(void) { -if(cutthrough.fd < 0) +if(cutthrough.fd < 0 || cutthrough.callout_hold_only) return FALSE; -HDEBUG(D_transport|D_acl|D_v) debug_printf(" SMTP>> DATA\n"); +HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> DATA\n"); cutthrough_puts(US"DATA\r\n", 6); cutthrough_flush_send(); /* Assume nothing buffered. If it was it gets ignored. */ -return cutthrough_response('3', NULL) == '3'; +return cutthrough_response(cutthrough.fd, '3', NULL, CUTTHROUGH_DATA_TIMEOUT) == '3'; } -/* fd and tctx args only to match write_chunk() */ +/* tctx arg only to match write_chunk() */ static BOOL -cutthrough_write_chunk(int fd, transport_ctx * tctx, uschar * s, int len) +cutthrough_write_chunk(transport_ctx * tctx, uschar * s, int len) { uschar * s2; while(s && (s2 = Ustrchr(s, '\n'))) @@ -1587,58 +1367,76 @@ cutthrough_headers_send(void) { transport_ctx tctx; -if(cutthrough.fd < 0) +if(cutthrough.fd < 0 || cutthrough.callout_hold_only) return FALSE; /* We share a routine with the mainline transport to handle header add/remove/rewrites, but having a separate buffered-output function (for now) */ -HDEBUG(D_acl) debug_printf("----------- start cutthrough headers send -----------\n"); +HDEBUG(D_acl) debug_printf_indent("----------- start cutthrough headers send -----------\n"); +tctx.u.fd = cutthrough.fd; tctx.tblock = cutthrough.addr.transport; tctx.addr = &cutthrough.addr; tctx.check_string = US"."; tctx.escape_string = US".."; +/*XXX check under spool_files_wireformat. Might be irrelevant */ tctx.options = topt_use_crlf; -if (!transport_headers_send(cutthrough.fd, &tctx, &cutthrough_write_chunk)) +if (!transport_headers_send(&tctx, &cutthrough_write_chunk)) return FALSE; -HDEBUG(D_acl) debug_printf("----------- done cutthrough headers send ------------\n"); +HDEBUG(D_acl) debug_printf_indent("----------- done cutthrough headers send ------------\n"); return TRUE; } static void -close_cutthrough_connection(const char * why) +close_cutthrough_connection(const uschar * why) { -if(cutthrough.fd >= 0) +int fd = cutthrough.fd; +if(fd >= 0) { /* We could be sending this after a bunch of data, but that is ok as the only way to cancel the transfer in dataphase is to drop the tcp conn before the final dot. */ ctblock.ptr = ctbuffer; - HDEBUG(D_transport|D_acl|D_v) debug_printf(" SMTP>> QUIT\n"); + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n"); _cutthrough_puts(US"QUIT\r\n", 6); /* avoid recursion */ _cutthrough_flush_send(); - /* No wait for response */ - - #ifdef SUPPORT_TLS - tls_close(FALSE, TRUE); - #endif - (void)close(cutthrough.fd); - cutthrough.fd = -1; - HDEBUG(D_acl) debug_printf("----------- cutthrough shutdown (%s) ------------\n", why); + cutthrough.fd = -1; /* avoid recursion via read timeout */ + cutthrough.nrcpt = 0; /* permit re-cutthrough on subsequent message */ + + /* Wait a short time for response, and discard it */ + cutthrough_response(fd, '2', NULL, 1); + +#ifdef SUPPORT_TLS + tls_close(FALSE, TLS_SHUTDOWN_NOWAIT); +#endif + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); + (void)close(fd); + HDEBUG(D_acl) debug_printf_indent("----------- cutthrough shutdown (%s) ------------\n", why); } ctblock.ptr = ctbuffer; } void -cancel_cutthrough_connection(const char * why) +cancel_cutthrough_connection(BOOL close_noncutthrough_verifies, const uschar * why) { -close_cutthrough_connection(why); -cutthrough.delivery = FALSE; +if (cutthrough.delivery || close_noncutthrough_verifies) + close_cutthrough_connection(why); +cutthrough.delivery = cutthrough.callout_hold_only = FALSE; +} + + +void +release_cutthrough_connection(const uschar * why) +{ +if (cutthrough.fd < 0) return; +HDEBUG(D_acl) debug_printf_indent("release cutthrough conn: %s\n", why); +cutthrough.fd = -1; +cutthrough.delivery = cutthrough.callout_hold_only = FALSE; } @@ -1654,7 +1452,7 @@ cutthrough_finaldot(void) { uschar res; address_item * addr; -HDEBUG(D_transport|D_acl|D_v) debug_printf(" SMTP>> .\n"); +HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> .\n"); /* Assume data finshed with new-line */ if( !cutthrough_puts(US".", 1) @@ -1663,7 +1461,7 @@ if( !cutthrough_puts(US".", 1) ) return cutthrough.addr.message; -res = cutthrough_response('2', &cutthrough.addr.message); +res = cutthrough_response(cutthrough.fd, '2', &cutthrough.addr.message, CUTTHROUGH_DATA_TIMEOUT); for (addr = &cutthrough.addr; addr; addr = addr->next) { addr->message = cutthrough.addr.message; @@ -1671,7 +1469,7 @@ for (addr = &cutthrough.addr; addr; addr = addr->next) { case '2': delivery_log(LOG_MAIN, addr, (int)'>', NULL); - close_cutthrough_connection("delivered"); + close_cutthrough_connection(US"delivered"); break; case '4': @@ -1756,7 +1554,7 @@ va_list ap; va_start(ap, format); if (smtp_out && (f == smtp_out)) - smtp_vprintf(format, ap); + smtp_vprintf(format, FALSE, ap); else vfprintf(f, format, ap); va_end(ap); @@ -2018,16 +1816,16 @@ while (addr_new) transport. */ transport_feedback tf = { - NULL, /* interface (=> any) */ - US"smtp", /* port */ - US"smtp", /* protocol */ - NULL, /* hosts */ - US"$smtp_active_hostname", /* helo_data */ - FALSE, /* hosts_override */ - FALSE, /* hosts_randomize */ - FALSE, /* gethostbyname */ - TRUE, /* qualify_single */ - FALSE /* search_parents */ + .interface = NULL, /* interface (=> any) */ + .port = US"smtp", + .protocol = US"smtp", + .hosts = NULL, + .helo_data = US"$smtp_active_hostname", + .hosts_override = FALSE, + .hosts_randomize = FALSE, + .gethostbyname = FALSE, + .qualify_single = TRUE, + .search_parents = FALSE }; /* If verification yielded a remote transport, we want to use that @@ -2074,7 +1872,7 @@ while (addr_new) additional host items being inserted into the chain. Hence we must save the next host first. */ - flags = HOST_FIND_BY_A; + flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA; if (tf.qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE; if (tf.search_parents) flags |= HOST_FIND_SEARCH_PARENTS; @@ -2094,7 +1892,7 @@ while (addr_new) dnssec_domains = &ob->dnssec; } - (void)host_find_bydns(host, NULL, flags, NULL, NULL, NULL, + (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL, dnssec_domains, NULL, NULL); } } @@ -2170,7 +1968,7 @@ while (addr_new) } respond_printf(f, "%s\n", cr); } - cancel_cutthrough_connection("routing hard fail"); + cancel_cutthrough_connection(TRUE, US"routing hard fail"); if (!full_info) { @@ -2209,7 +2007,7 @@ while (addr_new) } respond_printf(f, "%s\n", cr); } - cancel_cutthrough_connection("routing soft fail"); + cancel_cutthrough_connection(TRUE, US"routing soft fail"); if (!full_info) { @@ -2282,7 +2080,7 @@ while (addr_new) /* If stopped because more than one new address, cannot cutthrough */ if (addr_new && addr_new->next) - cancel_cutthrough_connection("multiple addresses from routing"); + cancel_cutthrough_connection(TRUE, US"multiple addresses from routing"); yield = OK; goto out; @@ -2394,7 +2192,7 @@ return yield; *************************************************/ /* This function checks those header lines that contain addresses, and verifies -that all the addresses therein are syntactially correct. +that all the addresses therein are 5322-syntactially correct. Arguments: msgptr where to put an error message @@ -2410,7 +2208,7 @@ header_line *h; uschar *colon, *s; int yield = OK; -for (h = header_list; h != NULL && yield == OK; h = h->next) +for (h = header_list; h && yield == OK; h = h->next) { if (h->type != htype_from && h->type != htype_reply_to && @@ -2429,7 +2227,7 @@ for (h = header_list; h != NULL && yield == OK; h = h->next) parse_allow_group = TRUE; - while (*s != 0) + while (*s) { uschar *ss = parse_find_address_end(s, FALSE); uschar *recipient, *errmess; @@ -2446,7 +2244,7 @@ for (h = header_list; h != NULL && yield == OK; h = h->next) /* Permit an unqualified address only if the message is local, or if the sending host is configured to be permitted to send them. */ - if (recipient != NULL && domain == 0) + if (recipient && !domain) { if (h->type == htype_from || h->type == htype_sender) { @@ -2462,7 +2260,7 @@ for (h = header_list; h != NULL && yield == OK; h = h->next) /* It's an error if no address could be extracted, except for the special case of an empty address. */ - if (recipient == NULL && Ustrcmp(errmess, "empty address") != 0) + if (!recipient && Ustrcmp(errmess, "empty address") != 0) { uschar *verb = US"is"; uschar *t = ss; @@ -2492,7 +2290,7 @@ for (h = header_list; h != NULL && yield == OK; h = h->next) /* deconst cast ok as we're passing a non-const to string_printing() */ *msgptr = US string_printing( string_sprintf("%s: failing address in \"%.*s:\" header %s: %.*s", - errmess, tt - h->text, h->text, verb, len, s)); + errmess, (int)(tt - h->text), h->text, verb, len, s)); yield = FAIL; break; /* Out of address loop */ @@ -2500,7 +2298,7 @@ for (h = header_list; h != NULL && yield == OK; h = h->next) /* Advance to the next address */ - s = ss + (terminator? 1:0); + s = ss + (terminator ? 1 : 0); while (isspace(*s)) s++; } /* Next address */ @@ -2516,7 +2314,7 @@ return yield; * Check header names for 8-bit characters * *************************************************/ -/* This function checks for invalid charcters in header names. See +/* This function checks for invalid characters in header names. See RFC 5322, 2.2. and RFC 6532, 3. Arguments: @@ -2532,18 +2330,16 @@ verify_check_header_names_ascii(uschar **msgptr) header_line *h; uschar *colon, *s; -for (h = header_list; h != NULL; h = h->next) +for (h = header_list; h; h = h->next) { - colon = Ustrchr(h->text, ':'); - for(s = h->text; s < colon; s++) - { - if ((*s < 33) || (*s > 126)) - { - *msgptr = string_sprintf("Invalid character in header \"%.*s\" found", - colon - h->text, h->text); - return FAIL; - } - } + colon = Ustrchr(h->text, ':'); + for(s = h->text; s < colon; s++) + if ((*s < 33) || (*s > 126)) + { + *msgptr = string_sprintf("Invalid character in header \"%.*s\" found", + colon - h->text, h->text); + return FAIL; + } } return OK; } @@ -2795,7 +2591,7 @@ for (i = 0; i < 3 && !done; i++) while (ss > s && isspace(ss[-1])) ss--; *log_msgptr = string_sprintf("syntax error in '%.*s' header when " "scanning for sender: %s in \"%.*s\"", - endname - h->text, h->text, *log_msgptr, ss - s, s); + (int)(endname - h->text), h->text, *log_msgptr, (int)(ss - s), s); yield = FAIL; done = TRUE; break; @@ -2823,11 +2619,9 @@ for (i = 0; i < 3 && !done; i++) { *verrno = vaddr->basic_errno; if (smtp_return_error_details) - { *user_msgptr = string_sprintf("Rejected after DATA: " "could not verify \"%.*s\" header address\n%s: %s", - endname - h->text, h->text, vaddr->address, vaddr->message); - } + (int)(endname - h->text), h->text, vaddr->address, vaddr->message); } /* Success or defer */ @@ -2888,6 +2682,7 @@ verify_get_ident(int port) int sock, host_af, qlen; int received_sender_port, received_interface_port, n; uschar *p; +blob early_data; uschar buffer[2048]; /* Default is no ident. Check whether we want to do an ident check for this @@ -2903,9 +2698,8 @@ DEBUG(D_ident) debug_printf("doing ident callback\n"); to the incoming interface address. If the sender host address is an IPv6 address, the incoming interface address will also be IPv6. */ -host_af = (Ustrchr(sender_host_address, ':') == NULL)? AF_INET : AF_INET6; -sock = ip_socket(SOCK_STREAM, host_af); -if (sock < 0) return; +host_af = Ustrchr(sender_host_address, ':') == NULL ? AF_INET : AF_INET6; +if ((sock = ip_socket(SOCK_STREAM, host_af)) < 0) return; if (ip_bind(sock, host_af, interface_address, 0) < 0) { @@ -2914,29 +2708,22 @@ if (ip_bind(sock, host_af, interface_address, 0) < 0) goto END_OFF; } -if (ip_connect(sock, host_af, sender_host_address, port, rfc1413_query_timeout) - < 0) +/* Construct and send the query. */ + +qlen = snprintf(CS buffer, sizeof(buffer), "%d , %d\r\n", + sender_host_port, interface_port); +early_data.data = buffer; +early_data.len = qlen; + +if (ip_connect(sock, host_af, sender_host_address, port, + rfc1413_query_timeout, &early_data) < 0) { if (errno == ETIMEDOUT && LOGGING(ident_timeout)) - { log_write(0, LOG_MAIN, "ident connection to %s timed out", sender_host_address); - } else - { DEBUG(D_ident) debug_printf("ident connection to %s failed: %s\n", sender_host_address, strerror(errno)); - } - goto END_OFF; - } - -/* Construct and send the query. */ - -sprintf(CS buffer, "%d , %d\r\n", sender_host_port, interface_port); -qlen = Ustrlen(buffer); -if (send(sock, buffer, qlen, 0) < 0) - { - DEBUG(D_ident) debug_printf("ident send failed: %s\n", strerror(errno)); goto END_OFF; } @@ -3102,7 +2889,7 @@ if (*ss == '@') } /* If the pattern is an IP address, optionally followed by a bitmask count, do -a (possibly masked) comparision with the current IP address. */ +a (possibly masked) comparison with the current IP address. */ if (string_is_ip_address(ss, &maskoffset) != 0) return (host_is_in_net(cb->host_address, ss, maskoffset)? OK : FAIL); @@ -3312,7 +3099,7 @@ if (isquery) /* Not a query-style lookup; must ensure the host name is present, and then we do a check on the name and all its aliases. */ -if (sender_host_name == NULL) +if (!sender_host_name) { HDEBUG(D_host_lookup) debug_printf("sender host name required, to match against %s\n", ss); @@ -3327,8 +3114,7 @@ if (sender_host_name == NULL) /* Match on the sender host name, using the general matching function */ -switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, - valueptr)) +switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, valueptr)) { case OK: return OK; case DEFER: return DEFER; @@ -3337,14 +3123,12 @@ switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, /* If there are aliases, try matching on them. */ aliases = sender_host_aliases; -while (*aliases != NULL) - { +while (*aliases) switch(match_check_string(*aliases++, ss, -1, TRUE, TRUE, TRUE, valueptr)) { case OK: return OK; case DEFER: return DEFER; } - } return FAIL; } @@ -3390,18 +3174,16 @@ verify_check_this_host(const uschar **listptr, unsigned int *cache_bits, int rc; unsigned int *local_cache_bits = cache_bits; const uschar *save_host_address = deliver_host_address; -check_host_block cb; -cb.host_name = host_name; -cb.host_address = host_address; +check_host_block cb = { .host_name = host_name, .host_address = host_address }; -if (valueptr != NULL) *valueptr = NULL; +if (valueptr) *valueptr = NULL; /* If the host address starts off ::ffff: it is an IPv6 address in IPv4-compatible mode. Find the IPv4 part for checking against IPv4 addresses. */ -cb.host_ipv4 = (Ustrncmp(host_address, "::ffff:", 7) == 0)? - host_address + 7 : host_address; +cb.host_ipv4 = Ustrncmp(host_address, "::ffff:", 7) == 0 + ? host_address + 7 : host_address; /* During the running of the check, put the IP address into $host_address. In the case of calls from the smtp transport, it will already be there. However, @@ -3617,7 +3399,7 @@ else (void)tree_insertnode(&dnsbl_cache, t); } - /* Do the DNS loopup . */ + /* Do the DNS lookup . */ HDEBUG(D_dnsbl) debug_printf("new DNS lookup for %s\n", query); cb->rc = dns_basic_lookup(&dnsa, query, T_A); @@ -3645,25 +3427,23 @@ else for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS); rr; rr = dns_next_rr(&dnsa, &dnss, RESET_NEXT)) - { if (rr->type == T_A) { dns_address *da = dns_address_from_rr(&dnsa, rr); if (da) { *addrp = da; - while (da->next != NULL) da = da->next; - addrp = &(da->next); + while (da->next) da = da->next; + addrp = &da->next; if (ttl > rr->ttl) ttl = rr->ttl; } } - } /* If we didn't find any A records, change the return code. This can happen when there is a CNAME record but there are no A records for what it points to. */ - if (cb->rhs == NULL) cb->rc = DNS_NODATA; + if (!cb->rhs) cb->rc = DNS_NODATA; } cb->expiry = time(NULL)+ttl; @@ -3685,7 +3465,7 @@ if (cb->rc == DNS_SUCCEED) records. For A6 records (currently not expected to be used) there may be multiple addresses from a single record. */ - for (da = cb->rhs->next; da != NULL; da = da->next) + for (da = cb->rhs->next; da; da = da->next) addlist = string_sprintf("%s, %s", addlist, da->address); HDEBUG(D_dnsbl) debug_printf("DNS lookup for %s succeeded (yielding %s)\n", @@ -3694,9 +3474,9 @@ if (cb->rc == DNS_SUCCEED) /* Address list check; this can be either for equality, or via a bitmask. In the latter case, all the bits must match. */ - if (iplist != NULL) + if (iplist) { - for (da = cb->rhs; da != NULL; da = da->next) + for (da = cb->rhs; da; da = da->next) { int ipsep = ','; uschar ip[46]; @@ -3706,12 +3486,11 @@ if (cb->rc == DNS_SUCCEED) /* Handle exact matching */ if (!bitmask) - { - while ((res = string_nextinlist(&ptr, &ipsep, ip, sizeof(ip))) != NULL) - { - if (Ustrcmp(CS da->address, ip) == 0) break; - } - } + { + while ((res = string_nextinlist(&ptr, &ipsep, ip, sizeof(ip)))) + if (Ustrcmp(CS da->address, ip) == 0) + break; + } /* Handle bitmask matching */ @@ -3731,7 +3510,7 @@ if (cb->rc == DNS_SUCCEED) /* Scan the returned addresses, skipping any that are IPv6 */ - while ((res = string_nextinlist(&ptr, &ipsep, ip, sizeof(ip))) != NULL) + while ((res = string_nextinlist(&ptr, &ipsep, ip, sizeof(ip)))) { if (host_aton(ip, address) != 1) continue; if ((address[0] & mask) == address[0]) break; @@ -3764,17 +3543,13 @@ if (cb->rc == DNS_SUCCEED) switch(match_type) { case 0: - res = US"was no match"; - break; + res = US"was no match"; break; case MT_NOT: - res = US"was an exclude match"; - break; + res = US"was an exclude match"; break; case MT_ALL: - res = US"was an IP address that did not match"; - break; + res = US"was an IP address that did not match"; break; case MT_NOT|MT_ALL: - res = US"were no IP addresses that did not match"; - break; + res = US"were no IP addresses that did not match"; break; } debug_printf("=> but we are not accepting this block class because\n"); debug_printf("=> there %s for %s%c%s\n", @@ -3806,15 +3581,15 @@ if (cb->rc == DNS_SUCCEED) { dns_record *rr; for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS); - rr != NULL; + rr; rr = dns_next_rr(&dnsa, &dnss, RESET_NEXT)) if (rr->type == T_TXT) break; - if (rr != NULL) + if (rr) { int len = (rr->data)[0]; if (len > 511) len = 127; store_pool = POOL_PERM; - cb->text = string_sprintf("%.*s", len, (const uschar *)(rr->data+1)); + cb->text = string_sprintf("%.*s", len, CUS (rr->data+1)); store_pool = old_pool; } }