X-Git-Url: https://vcs.fsf.org/?a=blobdiff_plain;f=src%2Fsrc%2Ftransports%2Fsmtp.c;h=7537e6e4b7f64b84c1b31090fca491e633285406;hb=3c8b357717e895d6dcddc7faa5b7a0eaf4c73417;hp=b932a87c49daddbd93f3a077da9d5f6051696398;hpb=4550006051177e69459d81db58b374e67beaf02b;p=exim.git diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index b932a87c4..7537e6e4b 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -19,6 +19,11 @@ before the lower case letters). Some live in the transport_instance block so as to be publicly visible; these are flagged with opt_public. */ optionlist smtp_transport_options[] = { + { "*expand_multi_domain", opt_stringptr | opt_hidden | opt_public, + (void *)offsetof(transport_instance, expand_multi_domain) }, + { "*expand_retry_include_ip_address", opt_stringptr | opt_hidden, + (void *)(offsetof(smtp_transport_options_block, expand_retry_include_ip_address)) }, + { "address_retry_include_sender", opt_bool, (void *)offsetof(smtp_transport_options_block, address_retry_include_sender) }, { "allow_localhost", opt_bool, @@ -142,18 +147,22 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, lmtp_ignore_quota) }, { "max_rcpt", opt_int | opt_public, (void *)offsetof(transport_instance, max_addresses) }, - { "multi_domain", opt_bool | opt_public, + { "multi_domain", opt_expand_bool | opt_public, (void *)offsetof(transport_instance, multi_domain) }, { "port", opt_stringptr, (void *)offsetof(smtp_transport_options_block, port) }, { "protocol", opt_stringptr, (void *)offsetof(smtp_transport_options_block, protocol) }, - { "retry_include_ip_address", opt_bool, + { "retry_include_ip_address", opt_expand_bool, (void *)offsetof(smtp_transport_options_block, retry_include_ip_address) }, { "serialize_hosts", opt_stringptr, (void *)offsetof(smtp_transport_options_block, serialize_hosts) }, { "size_addition", opt_int, (void *)offsetof(smtp_transport_options_block, size_addition) } +#ifdef EXPERIMENTAL_SOCKS + ,{ "socks_proxy", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, socks_proxy) } +#endif #ifdef SUPPORT_TLS ,{ "tls_certificate", opt_stringptr, (void *)offsetof(smtp_transport_options_block, tls_certificate) }, @@ -171,10 +180,8 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, tls_tempfail_tryclear) }, { "tls_try_verify_hosts", opt_stringptr, (void *)offsetof(smtp_transport_options_block, tls_try_verify_hosts) }, -#ifdef EXPERIMENTAL_CERTNAMES { "tls_verify_cert_hostnames", opt_stringptr, (void *)offsetof(smtp_transport_options_block,tls_verify_cert_hostnames)}, -#endif { "tls_verify_certificates", opt_stringptr, (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) }, { "tls_verify_hosts", opt_stringptr, @@ -209,7 +216,7 @@ smtp_transport_options_block smtp_transport_option_defaults = { NULL, /* hosts_require_dane */ #endif #ifndef DISABLE_PRDR - NULL, /* hosts_try_prdr */ + US"*", /* hosts_try_prdr */ #endif #ifndef DISABLE_OCSP US"*", /* hosts_request_ocsp (except under DANE; tls_client_start()) */ @@ -217,7 +224,7 @@ smtp_transport_options_block smtp_transport_option_defaults = { #endif NULL, /* hosts_require_tls */ NULL, /* hosts_avoid_tls */ - US"*", /* hosts_verify_avoid_tls */ + NULL, /* hosts_verify_avoid_tls */ NULL, /* hosts_avoid_pipelining */ NULL, /* hosts_avoid_esmtp */ NULL, /* hosts_nopass_tls */ @@ -241,7 +248,11 @@ smtp_transport_options_block smtp_transport_option_defaults = { FALSE, /* hosts_randomize */ TRUE, /* keepalive */ FALSE, /* lmtp_ignore_quota */ + NULL, /* expand_retry_include_ip_address */ TRUE /* retry_include_ip_address */ +#ifdef EXPERIMENTAL_SOCKS + ,NULL /* socks_proxy */ +#endif #ifdef SUPPORT_TLS ,NULL, /* tls_certificate */ NULL, /* tls_crl */ @@ -251,15 +262,13 @@ smtp_transport_options_block smtp_transport_option_defaults = { NULL, /* gnutls_require_mac */ NULL, /* gnutls_require_proto */ NULL, /* tls_sni */ - NULL, /* tls_verify_certificates */ + US"system", /* tls_verify_certificates */ EXIM_CLIENT_DH_DEFAULT_MIN_BITS, /* tls_dh_min_bits */ TRUE, /* tls_tempfail_tryclear */ NULL, /* tls_verify_hosts */ - NULL /* tls_try_verify_hosts */ -# ifdef EXPERIMENTAL_CERTNAMES - ,NULL /* tls_verify_cert_hostnames */ -# endif + US"*", /* tls_try_verify_hosts */ + US"*" /* tls_verify_cert_hostnames */ #endif #ifndef DISABLE_DKIM ,NULL, /* dkim_canon */ @@ -271,14 +280,12 @@ smtp_transport_options_block smtp_transport_option_defaults = { #endif }; -#ifdef EXPERIMENTAL_DSN /* some DSN flags for use later */ static int rf_list[] = {rf_notify_never, rf_notify_success, rf_notify_failure, rf_notify_delay }; static uschar *rf_names[] = { US"NEVER", US"SUCCESS", US"FAILURE", US"DELAY" }; -#endif @@ -454,19 +461,19 @@ if (errno_value == ERRNO_CONNECTTIMEOUT) orvalue = RTEF_CTOUT; } for (addr = addrlist; addr != NULL; addr = addr->next) - { - if (addr->transport_return < PENDING) continue; - addr->basic_errno = errno_value; - addr->more_errno |= orvalue; - if (msg != NULL) + if (addr->transport_return >= PENDING) { - addr->message = msg; - if (pass_message) setflag(addr, af_pass_message); + addr->basic_errno = errno_value; + addr->more_errno |= orvalue; + if (msg != NULL) + { + addr->message = msg; + if (pass_message) setflag(addr, af_pass_message); + } + addr->transport_return = rc; + if (host) + addr->host_used = host; } - addr->transport_return = rc; - if (host) - addr->host_used = host; - } } @@ -523,7 +530,7 @@ if (*errno_value == ETIMEDOUT) if (*errno_value == ERRNO_SMTPFORMAT) { - uschar *malfresp = string_printing(buffer); + const uschar *malfresp = string_printing(buffer); while (isspace(*malfresp)) malfresp++; *message = *malfresp == 0 ? string_sprintf("Malformed SMTP reply (an empty line) " @@ -563,11 +570,21 @@ if (*errno_value == ERRNO_WRITEINCOMPLETE) return FALSE; } +#ifdef EXPERIMENTAL_INTERNATIONAL +/* Handle lack of advertised SMTPUTF8, for international message */ +if (*errno_value == ERRNO_UTF8_FWD) + { + *message = US string_sprintf("utf8 support required but not offered for forwarding"); + DEBUG(D_deliver|D_transport) debug_printf("%s\n", *message); + return TRUE; + } +#endif + /* Handle error responses from the remote mailer. */ if (buffer[0] != 0) { - uschar *s = string_printing(buffer); + const uschar *s = string_printing(buffer); *message = US string_sprintf("SMTP error from remote mail server after %s%s: " "%s", pl, smtp_command, s); *pass_message = TRUE; @@ -659,7 +676,7 @@ static void deferred_event_raise(address_item *addr, host_item *host) { uschar * action = addr->transport->event_action; -uschar * save_domain; +const uschar * save_domain; uschar * save_local; if (!action) @@ -693,8 +710,6 @@ router_name = transport_name = NULL; } #endif - - /************************************************* * Synchronize SMTP responses * *************************************************/ @@ -831,10 +846,9 @@ while (count-- > 0) else if (errno == ETIMEDOUT) { - int save_errno = errno; uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>", transport_rcpt_address(addr, include_affixes)); - set_errno(addrlist, save_errno, message, DEFER, FALSE, NULL); + set_errno(addrlist, ETIMEDOUT, message, DEFER, FALSE, NULL); retry_add_item(addr, addr->address_retry_key, 0); update_waiting = FALSE; return -1; @@ -970,8 +984,7 @@ uschar *fail_reason = US"server did not advertise AUTH support"; smtp_authenticated = FALSE; client_authenticator = client_authenticated_id = client_authenticated_sender = NULL; -require_auth = verify_check_this_host(&(ob->hosts_require_auth), NULL, - host->name, host->address, NULL); +require_auth = verify_check_given_host(&ob->hosts_require_auth, host); if (is_esmtp && !regex_AUTH) regex_AUTH = regex_must_compile(US"\\n250[\\s\\-]AUTH\\s+([\\-\\w\\s]+)(?:\\n|$)", @@ -986,8 +999,7 @@ if (is_esmtp && regex_match_and_setup(regex_AUTH, buffer, 0, -1)) regex match above. */ if (require_auth == OK || - verify_check_this_host(&(ob->hosts_try_auth), NULL, host->name, - host->address, NULL) == OK) + verify_check_given_host(&ob->hosts_try_auth, host) == OK) { auth_instance *au; fail_reason = US"no common mechanisms were found"; @@ -1085,7 +1097,8 @@ if (is_esmtp && regex_match_and_setup(regex_AUTH, buffer, 0, -1)) /* Internal problem, message in buffer. */ case ERROR: - set_errno(addrlist, 0, string_copy(buffer), DEFER, FALSE, NULL); + set_errno(addrlist, ERRNO_AUTHPROB, string_copy(buffer), + DEFER, FALSE, NULL); return ERROR; } @@ -1140,7 +1153,7 @@ if (ob->authenticated_sender != NULL) { uschar *message = string_sprintf("failed to expand " "authenticated_sender: %s", expand_string_message); - set_errno(addrlist, 0, message, DEFER, FALSE, NULL); + set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL); return TRUE; } } @@ -1167,12 +1180,12 @@ return FALSE; #ifdef EXPERIMENTAL_DANE int -tlsa_lookup(host_item * host, dns_answer * dnsa, +tlsa_lookup(const host_item * host, dns_answer * dnsa, BOOL dane_required, BOOL * dane) { /* move this out to host.c given the similarity to dns_lookup() ? */ uschar buffer[300]; -uschar * fullname = buffer; +const uschar * fullname = buffer; /* TLSA lookup string */ (void)sprintf(CS buffer, "_%d._tcp.%.256s", host->port, host->name); @@ -1205,6 +1218,80 @@ return OK; #endif + +typedef struct smtp_compare_s +{ + uschar *current_sender_address; + struct transport_instance *tblock; +} smtp_compare_t; + +/* +Create a unique string that identifies this message, it is based on +sender_address, helo_data and tls_certificate if enabled. */ + +static uschar * +smtp_local_identity(uschar * sender, struct transport_instance * tblock) +{ +address_item * addr1; +uschar * if1 = US""; +uschar * helo1 = US""; +#ifdef SUPPORT_TLS +uschar * tlsc1 = US""; +#endif +uschar * save_sender_address = sender_address; +uschar * local_identity = NULL; +smtp_transport_options_block * ob = + (smtp_transport_options_block *)tblock->options_block; + +sender_address = sender; + +addr1 = deliver_make_addr (sender, TRUE); +deliver_set_expansions(addr1); + +if (ob->interface) + if1 = expand_string(ob->interface); + +if (ob->helo_data) + helo1 = expand_string(ob->helo_data); + +#ifdef SUPPORT_TLS +if (ob->tls_certificate) + tlsc1 = expand_string(ob->tls_certificate); +local_identity = string_sprintf ("%s^%s^%s", if1, helo1, tlsc1); +#else +local_identity = string_sprintf ("%s^%s", if1, helo1); +#endif + +deliver_set_expansions(NULL); +sender_address = save_sender_address; + +return local_identity; +} + + + +/* This routine is a callback that is called from transport_check_waiting. +This function will evaluate the incoming message versus the previous +message. If the incoming message is using a different local identity then +we will veto this new message. */ + +static BOOL +smtp_are_same_identities(uschar * message_id, smtp_compare_t * s_compare) +{ +uschar * save_sender_address = sender_address; +uschar * current_local_identity = + smtp_local_identity(s_compare->current_sender_address, s_compare->tblock); +uschar * new_sender_address = deliver_get_sender_address(message_id); +uschar * message_local_identity = + smtp_local_identity(new_sender_address, s_compare->tblock); + +sender_address = save_sender_address; + +return Ustrcmp(current_local_identity, message_local_identity) == 0; +} + + + /************************************************* * Deliver address list to given host * *************************************************/ @@ -1231,8 +1318,6 @@ Arguments: port default TCP/IP port to use, in host byte order interface interface to bind to, or NULL tblock transport instance block - copy_host TRUE if host set in addr->host_used must be copied, because - it is specific to this call of the transport message_defer set TRUE if yield is OK, but all addresses were deferred because of a non-recipient, non-host failure, that is, a 4xx response to MAIL FROM, DATA, or ".". This is a defer @@ -1253,7 +1338,7 @@ Returns: OK - the connection was made and the delivery attempted; static int smtp_deliver(address_item *addrlist, host_item *host, int host_af, int port, - uschar *interface, transport_instance *tblock, BOOL copy_host, + uschar *interface, transport_instance *tblock, BOOL *message_defer, BOOL suppress_tls) { address_item *addr; @@ -1280,24 +1365,30 @@ BOOL pass_message = FALSE; BOOL prdr_offered = FALSE; BOOL prdr_active; #endif -#ifdef EXPERIMENTAL_DSN -BOOL dsn_all_lasthop = TRUE; +#ifdef EXPERIMENTAL_INTERNATIONAL +BOOL utf8_needed = FALSE; +BOOL utf8_offered = FALSE; #endif +BOOL dsn_all_lasthop = TRUE; #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) BOOL dane = FALSE; +BOOL dane_required; dns_answer tlsa_dnsa; #endif smtp_inblock inblock; smtp_outblock outblock; int max_rcpt = tblock->max_addresses; uschar *igquotstr = US""; + uschar *helo_data = NULL; + uschar *message = NULL; uschar new_message_id[MESSAGE_ID_LENGTH + 1]; uschar *p; uschar buffer[4096]; uschar inbuffer[4096]; uschar outbuffer[4096]; +address_item * current_address; suppress_tls = suppress_tls; /* stop compiler warning when no TLS support */ @@ -1341,7 +1432,8 @@ tls_modify_variables(&tls_out); #ifndef SUPPORT_TLS if (smtps) { - set_errno(addrlist, 0, US"TLS support not available", DEFER, FALSE, NULL); + set_errno(addrlist, ERRNO_TLSFAILURE, US"TLS support not available", + DEFER, FALSE, NULL); return ERROR; } #endif @@ -1354,12 +1446,7 @@ if (continue_hostname == NULL) { /* This puts port into host->port */ inblock.sock = outblock.sock = - smtp_connect(host, host_af, port, interface, ob->connect_timeout, - ob->keepalive, ob->dscp -#ifdef EXPERIMENTAL_EVENT - , tblock->event_action -#endif - ); + smtp_connect(host, host_af, port, interface, ob->connect_timeout, tblock); if (inblock.sock < 0) { @@ -1370,22 +1457,19 @@ if (continue_hostname == NULL) #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) { - BOOL dane_required; - tls_out.dane_verified = FALSE; tls_out.tlsa_usage = 0; - dane_required = verify_check_this_host(&ob->hosts_require_dane, NULL, - host->name, host->address, NULL) == OK; + dane_required = verify_check_given_host(&ob->hosts_require_dane, host) == OK; if (host->dnssec == DS_YES) { - if( dane_required - || verify_check_this_host(&ob->hosts_try_dane, NULL, - host->name, host->address, NULL) == OK + if( ( dane_required + || verify_check_given_host(&ob->hosts_try_dane, host) == OK + ) + && (rc = tlsa_lookup(host, &tlsa_dnsa, dane_required, &dane)) != OK ) - if ((rc = tlsa_lookup(host, &tlsa_dnsa, dane_required, &dane)) != OK) - return rc; + return rc; } else if (dane_required) { @@ -1403,6 +1487,19 @@ if (continue_hostname == NULL) delayed till here so that $sending_interface and $sending_port are set. */ helo_data = expand_string(ob->helo_data); +#ifdef EXPERIMENTAL_INTERNATIONAL + if (helo_data) + { + uschar * errstr = NULL; + if ((helo_data = string_domain_utf8_to_alabel(helo_data, &errstr)), errstr) + { + errstr = string_sprintf("failed to expand helo_data: %s", errstr); + set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL); + yield = DEFER; + goto SEND_QUIT; + } + } +#endif /* The first thing is to wait for an initial OK response. The dreaded "goto" is nevertheless a reasonably clean way of programming this kind of logic, @@ -1421,7 +1518,7 @@ if (continue_hostname == NULL) s = event_raise(tblock->event_action, US"smtp:connect", buffer); if (s) { - set_errno(addrlist, 0, + set_errno(addrlist, ERRNO_EXPANDFAIL, string_sprintf("deferred by smtp:connect event expansion: %s", s), DEFER, FALSE, NULL); yield = DEFER; @@ -1437,7 +1534,7 @@ if (continue_hostname == NULL) { uschar *message = string_sprintf("failed to expand helo_data: %s", expand_string_message); - set_errno(addrlist, 0, message, DEFER, FALSE, NULL); + set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL); yield = DEFER; goto SEND_QUIT; } @@ -1478,8 +1575,7 @@ goto SEND_QUIT; mailers use upper case for some reason (the RFC is quite clear about case independence) so, for peace of mind, I gave in. */ - esmtp = verify_check_this_host(&(ob->hosts_avoid_esmtp), NULL, - host->name, host->address, NULL) != OK; + esmtp = verify_check_given_host(&ob->hosts_avoid_esmtp, host) != OK; /* Alas; be careful, since this goto is not an error-out, so conceivably we might set data between here and the target which we assume to exist @@ -1537,15 +1633,28 @@ goto SEND_QUIT; #endif #ifndef DISABLE_PRDR - prdr_offered = esmtp && - (pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(buffer), 0, - PCRE_EOPT, NULL, 0) >= 0) && - (verify_check_this_host(&(ob->hosts_try_prdr), NULL, host->name, - host->address, NULL) == OK); + prdr_offered = esmtp + && pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(buffer), 0, + PCRE_EOPT, NULL, 0) >= 0 + && verify_check_given_host(&ob->hosts_try_prdr, host) == OK; if (prdr_offered) {DEBUG(D_transport) debug_printf("PRDR usable\n");} #endif + +#ifdef EXPERIMENTAL_INTERNATIONAL + if (addrlist->prop.utf8_msg) + { + utf8_needed = !addrlist->prop.utf8_downcvt + && !addrlist->prop.utf8_downcvt_maybe; + DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n", + addrlist->prop.utf8_downcvt ? "mandatory" : "optional"); + + utf8_offered = esmtp + && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; + } +#endif } /* For continuing deliveries down the same channel, the socket is the standard @@ -1571,9 +1680,9 @@ the client not be required to use TLS. If the response is bad, copy the buffer for error analysis. */ #ifdef SUPPORT_TLS -if (tls_offered && !suppress_tls && - verify_check_this_host(&(ob->hosts_avoid_tls), NULL, host->name, - host->address, NULL) != OK) +if ( tls_offered + && !suppress_tls + && verify_check_given_host(&ob->hosts_avoid_tls, host) != OK) { uschar buffer2[4096]; if (smtp_write_command(&outblock, FALSE, "STARTTLS\r\n") < 0) @@ -1614,6 +1723,17 @@ if (tls_offered && !suppress_tls && if (rc != OK) { +# ifdef EXPERIMENTAL_DANE + if (rc == DEFER && dane && !dane_required) + { + log_write(0, LOG_MAIN, "DANE attempt failed;" + " trying CA-root TLS to %s [%s] (not in hosts_require_dane)", + host->name, host->address); + dane = FALSE; + goto TLS_NEGOTIATE; + } +# endif + save_errno = ERRNO_TLSFAILURE; message = US"failure while setting up TLS session"; send_quit = FALSE; @@ -1654,7 +1774,7 @@ if (tls_out.active >= 0) { uschar *message = string_sprintf("failed to expand helo_data: %s", expand_string_message); - set_errno(addrlist, 0, message, DEFER, FALSE, NULL); + set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL); yield = DEFER; goto SEND_QUIT; } @@ -1691,8 +1811,7 @@ else if ( # ifdef EXPERIMENTAL_DANE dane || # endif - verify_check_this_host(&(ob->hosts_require_tls), NULL, host->name, - host->address, NULL) == OK + verify_check_given_host(&ob->hosts_require_tls, host) == OK ) { save_errno = ERRNO_TLSREQUIRED; @@ -1732,32 +1851,36 @@ if (continue_hostname == NULL the current host, esmtp will be false, so PIPELINING can never be used. If the current host matches hosts_avoid_pipelining, don't do it. */ - smtp_use_pipelining = esmtp && - verify_check_this_host(&(ob->hosts_avoid_pipelining), NULL, host->name, - host->address, NULL) != OK && - pcre_exec(regex_PIPELINING, NULL, CS buffer, Ustrlen(CS buffer), 0, - PCRE_EOPT, NULL, 0) >= 0; + smtp_use_pipelining = esmtp + && verify_check_given_host(&ob->hosts_avoid_pipelining, host) != OK + && pcre_exec(regex_PIPELINING, NULL, CS buffer, Ustrlen(CS buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; DEBUG(D_transport) debug_printf("%susing PIPELINING\n", smtp_use_pipelining? "" : "not "); #ifndef DISABLE_PRDR - prdr_offered = esmtp && - pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(CS buffer), 0, - PCRE_EOPT, NULL, 0) >= 0 && - verify_check_this_host(&(ob->hosts_try_prdr), NULL, host->name, - host->address, NULL) == OK; + prdr_offered = esmtp + && pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(CS buffer), 0, + PCRE_EOPT, NULL, 0) >= 0 + && verify_check_given_host(&ob->hosts_try_prdr, host) == OK; if (prdr_offered) {DEBUG(D_transport) debug_printf("PRDR usable\n");} #endif -#ifdef EXPERIMENTAL_DSN +#ifdef EXPERIMENTAL_INTERNATIONAL + if (addrlist->prop.utf8_msg) + utf8_offered = esmtp + && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; +#endif + /* Note if the server supports DSN */ - smtp_use_dsn = esmtp && pcre_exec(regex_DSN, NULL, CS buffer, (int)Ustrlen(CS buffer), 0, - PCRE_EOPT, NULL, 0) >= 0; + smtp_use_dsn = esmtp + && pcre_exec(regex_DSN, NULL, CS buffer, Ustrlen(CS buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; DEBUG(D_transport) debug_printf("use_dsn=%d\n", smtp_use_dsn); -#endif /* Note if the response to EHLO specifies support for the AUTH extension. If it has, check that this host is one we want to authenticate to, and do @@ -1779,6 +1902,15 @@ message-specific. */ setting_up = FALSE; +#ifdef EXPERIMENTAL_INTERNATIONAL +/* If this is an international message we need the host to speak SMTPUTF8 */ +if (utf8_needed && !utf8_offered) + { + errno = ERRNO_UTF8_FWD; + goto RESPONSE_FAILED; + } +#endif + /* If there is a filter command specified for this transport, we can now set it up. This cannot be done until the identify of the host is known. */ @@ -1855,19 +1987,25 @@ if (prdr_offered) } #endif -#ifdef EXPERIMENTAL_DSN +#ifdef EXPERIMENTAL_INTERNATIONAL +if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered) + sprintf(CS p, " SMTPUTF8"), p += 9; +#endif + /* check if all addresses have lasthop flag */ /* do not send RET and ENVID if true */ -dsn_all_lasthop = TRUE; -for (addr = first_addr; +for (dsn_all_lasthop = TRUE, addr = first_addr; address_count < max_rcpt && addr != NULL; addr = addr->next) if ((addr->dsn_flags & rf_dsnlasthop) != 1) + { dsn_all_lasthop = FALSE; + break; + } /* Add any DSN flags to the mail command */ -if ((smtp_use_dsn) && (dsn_all_lasthop == FALSE)) +if (smtp_use_dsn && !dsn_all_lasthop) { if (dsn_ret == dsn_ret_hdrs) { @@ -1885,7 +2023,6 @@ if ((smtp_use_dsn) && (dsn_all_lasthop == FALSE)) while (*p) p++; } } -#endif /* If an authenticated_sender override has been specified for this transport instance, expand it. If the expansion is forced to fail, and there was already @@ -1908,28 +2045,51 @@ buffer. */ pending_MAIL = TRUE; /* The block starts with MAIL */ -rc = smtp_write_command(&outblock, smtp_use_pipelining, - "MAIL FROM:<%s>%s\r\n", return_path, buffer); + { + uschar * s = return_path; +#ifdef EXPERIMENTAL_INTERNATIONAL + uschar * errstr = NULL; + + /* If we must downconvert, do the from-address here. Remember we had to + for the to-addresses (done below), and also (ugly) for re-doing when building + the delivery log line. */ + + if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered)) + { + if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr) + { + set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL); + yield = ERROR; + goto SEND_QUIT; + } + setflag(addrlist, af_utf8_downcvt); + } +#endif + + rc = smtp_write_command(&outblock, smtp_use_pipelining, + "MAIL FROM:<%s>%s\r\n", s, buffer); + } + mail_command = string_copy(big_buffer); /* Save for later error message */ switch(rc) { case -1: /* Transmission error */ - goto SEND_FAILED; + goto SEND_FAILED; case +1: /* Block was sent */ - if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2', + if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2', ob->command_timeout)) - { - if (errno == 0 && buffer[0] == '4') { - errno = ERRNO_MAIL4XX; - addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8; + if (errno == 0 && buffer[0] == '4') + { + errno = ERRNO_MAIL4XX; + addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8; + } + goto RESPONSE_FAILED; } - goto RESPONSE_FAILED; - } - pending_MAIL = FALSE; - break; + pending_MAIL = FALSE; + break; } /* Pass over all the relevant recipient addresses for this host, which are the @@ -1951,17 +2111,15 @@ for (addr = first_addr; { int count; BOOL no_flush; + uschar * rcpt_addr; -#ifdef EXPERIMENTAL_DSN addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no; -#endif if (addr->transport_return != PENDING_DEFER) continue; address_count++; no_flush = smtp_use_pipelining && (!mua_wrapper || addr->next != NULL); -#ifdef EXPERIMENTAL_DSN /* Add any DSN flags to the rcpt command and add to the sent string */ p = buffer; @@ -1992,22 +2150,31 @@ for (addr = first_addr; while (*p) p++; } } -#endif - /* Now send the RCPT command, and process outstanding responses when necessary. After a timeout on RCPT, we just end the function, leaving the yield as OK, because this error can often mean that there is a problem with just one address, so we don't want to delay the host. */ -#ifdef EXPERIMENTAL_DSN - count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n", - transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer); -#else - count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s\r\n", - transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr); + rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes); + +#ifdef EXPERIMENTAL_INTERNATIONAL + { + uschar * dummy_errstr; + if ( testflag(addrlist, af_utf8_downcvt) + && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr), + dummy_errstr + ) ) + { + errno = ERRNO_EXPANDFAIL; + goto SEND_FAILED; + } + } #endif + count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n", + rcpt_addr, igquotstr, buffer); + if (count < 0) goto SEND_FAILED; if (count > 0) { @@ -2039,16 +2206,15 @@ RCPT. */ if (mua_wrapper) { address_item *badaddr; - for (badaddr = first_addr; badaddr != NULL; badaddr = badaddr->next) - { - if (badaddr->transport_return != PENDING_OK) break; - } - if (badaddr != NULL) - { - set_errno(addrlist, 0, badaddr->message, FAIL, - testflag(badaddr, af_pass_message), NULL); - ok = FALSE; - } + for (badaddr = first_addr; badaddr; badaddr = badaddr->next) + if (badaddr->transport_return != PENDING_OK) + { + /*XXX could we find a better errno than 0 here? */ + set_errno(addrlist, 0, badaddr->message, FAIL, + testflag(badaddr, af_pass_message), NULL); + ok = FALSE; + break; + } } /* If ok is TRUE, we know we have got at least one good recipient, and must now @@ -2205,22 +2371,9 @@ if (!ok) ok = TRUE; else int flag = '='; int delivery_time = (int)(time(NULL) - start_delivery_time); int len; - host_item *thost; uschar *conf = NULL; send_rset = FALSE; - /* Make a copy of the host if it is local to this invocation - of the transport. */ - - if (copy_host) - { - thost = store_get(sizeof(host_item)); - *thost = *host; - thost->name = string_copy(host->name); - thost->address = string_copy(host->address); - } - else thost = host; - /* Set up confirmation if needed - applies only to SMTP */ if ( @@ -2230,8 +2383,9 @@ if (!ok) ok = TRUE; else !lmtp ) { - uschar *s = string_printing(buffer); - conf = (s == buffer)? (uschar *)string_copy(s) : s; + const uschar *s = string_printing(buffer); + /* deconst cast ok here as string_printing was checked to have alloc'n'copied */ + conf = (s == buffer)? (uschar *)string_copy(s) : US s; } /* Process all transported addresses - for LMTP or PRDR, read a status for @@ -2281,8 +2435,9 @@ if (!ok) ok = TRUE; else completed_address = TRUE; /* NOW we can set this flag */ if ((log_extra_selector & LX_smtp_confirmation) != 0) { - uschar *s = string_printing(buffer); - conf = (s == buffer)? (uschar *)string_copy(s) : s; + const uschar *s = string_printing(buffer); + /* deconst cast ok here as string_printing was checked to have alloc'n'copied */ + conf = (s == buffer)? (uschar *)string_copy(s) : US s; } } @@ -2291,7 +2446,7 @@ if (!ok) ok = TRUE; else addr->transport_return = OK; addr->more_errno = delivery_time; - addr->host_used = thost; + addr->host_used = host; addr->special_action = flag; addr->message = conf; #ifndef DISABLE_PRDR @@ -2440,24 +2595,29 @@ if (!ok) switch(save_errno) { +#ifdef EXPERIMENTAL_INTERNATIONAL + case ERRNO_UTF8_FWD: + code = '5'; + /*FALLTHROUGH*/ +#endif case 0: case ERRNO_MAIL4XX: case ERRNO_DATA4XX: - message_error = TRUE; - break; + message_error = TRUE; + break; case ETIMEDOUT: - message_error = Ustrncmp(smtp_command,"MAIL",4) == 0 || - Ustrncmp(smtp_command,"end ",4) == 0; - break; + message_error = Ustrncmp(smtp_command,"MAIL",4) == 0 || + Ustrncmp(smtp_command,"end ",4) == 0; + break; case ERRNO_SMTPCLOSED: - message_error = Ustrncmp(smtp_command,"end ",4) == 0; - break; + message_error = Ustrncmp(smtp_command,"end ",4) == 0; + break; default: - message_error = FALSE; - break; + message_error = FALSE; + break; } /* Handle the cases that are treated as message errors. These are: @@ -2465,6 +2625,7 @@ if (!ok) (a) negative response or timeout after MAIL (b) negative response after DATA (c) negative response or timeout or dropped connection after "." + (d) utf8 support required and not offered It won't be a negative response or timeout after RCPT, as that is dealt with separately above. The action in all cases is to set an appropriate @@ -2546,15 +2707,21 @@ DEBUG(D_transport) if (completed_address && ok && send_quit) { BOOL more; - if (first_addr != NULL || continue_more || - ( - (tls_out.active < 0 || - verify_check_this_host(&(ob->hosts_nopass_tls), NULL, host->name, - host->address, NULL) != OK) + smtp_compare_t t_compare; + + t_compare.tblock = tblock; + t_compare.current_sender_address = sender_address; + + if ( first_addr != NULL + || continue_more + || ( ( tls_out.active < 0 + || verify_check_given_host(&ob->hosts_nopass_tls, host) != OK + ) && transport_check_waiting(tblock->name, host->name, - tblock->connection_max_messages, new_message_id, &more) - )) + tblock->connection_max_messages, new_message_id, &more, + (oicf)smtp_are_same_identities, (void*)&t_compare) + ) ) { uschar *msg; BOOL pass_message; @@ -2750,21 +2917,21 @@ prepare_addresses(address_item *addrlist, host_item *host) address_item *first_addr = NULL; address_item *addr; for (addr = addrlist; addr != NULL; addr = addr->next) - { - if (addr->transport_return != DEFER) continue; - if (first_addr == NULL) first_addr = addr; - addr->transport_return = PENDING_DEFER; - addr->basic_errno = 0; - addr->more_errno = (host->mx >= 0)? 'M' : 'A'; - addr->message = NULL; + if (addr->transport_return == DEFER) + { + if (first_addr == NULL) first_addr = addr; + addr->transport_return = PENDING_DEFER; + addr->basic_errno = 0; + addr->more_errno = (host->mx >= 0)? 'M' : 'A'; + addr->message = NULL; #ifdef SUPPORT_TLS - addr->cipher = NULL; - addr->ourcert = NULL; - addr->peercert = NULL; - addr->peerdn = NULL; - addr->ocsp = OCSP_NOT_REQ; + addr->cipher = NULL; + addr->ourcert = NULL; + addr->peercert = NULL; + addr->peerdn = NULL; + addr->ocsp = OCSP_NOT_REQ; #endif - } + } return first_addr; } @@ -2853,8 +3020,7 @@ if (hostlist == NULL || (ob->hosts_override && ob->hosts != NULL)) if (Ustrchr(s, '$') != NULL) { - expanded_hosts = expand_string(s); - if (expanded_hosts == NULL) + if (!(expanded_hosts = expand_string(s))) { addrlist->message = string_sprintf("failed to expand list of hosts " "\"%s\" in %s transport: %s", s, tblock->name, expand_string_message); @@ -2882,13 +3048,14 @@ if (hostlist == NULL || (ob->hosts_override && ob->hosts != NULL)) /* If there was no expansion of hosts, save the host list for next time. */ - if (expanded_hosts == NULL) ob->hostlist = hostlist; + if (!expanded_hosts) ob->hostlist = hostlist; } /* This is not the first time this transport has been run in this delivery; the host list was built previously. */ - else hostlist = ob->hostlist; + else + hostlist = ob->hostlist; } /* The host list was supplied with the address. If hosts_randomize is set, we @@ -2932,12 +3099,10 @@ else if (ob->hosts_randomize && hostlist->mx == MX_NONE && !continuing) hostlist = addrlist->host_list = newlist; } - /* Sort out the default port. */ if (!smtp_get_port(ob->port, addrlist, &port, tid)) return FALSE; - /* For each host-plus-IP-address on the list: . If this is a continued delivery and the host isn't the one with the @@ -3034,7 +3199,6 @@ for (cutoff_retry = 0; expired && { int new_port, flags; host_item *hh; - uschar *canonical_name; if (host->status >= hstatus_unusable) { @@ -3062,11 +3226,11 @@ for (cutoff_retry = 0; expired && if (ob->dns_search_parents) flags |= HOST_FIND_SEARCH_PARENTS; if (ob->gethostbyname || string_is_ip_address(host->name, NULL) != 0) - rc = host_find_byname(host, NULL, flags, &canonical_name, TRUE); + rc = host_find_byname(host, NULL, flags, NULL, TRUE); else rc = host_find_bydns(host, NULL, flags, NULL, NULL, NULL, ob->dnssec_request_domains, ob->dnssec_require_domains, - &canonical_name, NULL); + NULL, NULL); /* Update the host (and any additional blocks, resulting from multihoming) with a host-specific port, if any. */ @@ -3142,7 +3306,8 @@ for (cutoff_retry = 0; expired && doing a two-stage queue run, don't do this if forcing. */ if ((!deliver_force || queue_2stage) && (queue_smtp || - match_isinlist(addrlist->domain, &queue_smtp_domains, 0, + match_isinlist(addrlist->domain, + (const uschar **)&queue_smtp_domains, 0, &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)) { expired = FALSE; @@ -3194,14 +3359,20 @@ for (cutoff_retry = 0; expired && if (cutoff_retry == 0) { + BOOL incl_ip; /* Ensure the status of the address is set by checking retry data if - necessary. There maybe host-specific retry data (applicable to all + necessary. There may be host-specific retry data (applicable to all messages) and also data for retries of a specific message at this host. If either of these retry records are actually read, the keys used are returned to save recomputing them later. */ + if (exp_bool(addrlist, US"transport", tblock->name, D_transport, + US"retry_include_ip_address", ob->retry_include_ip_address, + ob->expand_retry_include_ip_address, &incl_ip) != OK) + continue; /* with next host */ + host_is_expired = retry_check_address(addrlist->domain, host, pistring, - ob->retry_include_ip_address, &retry_host_key, &retry_message_key); + incl_ip, &retry_host_key, &retry_message_key); DEBUG(D_transport) debug_printf("%s [%s]%s status = %s\n", host->name, (host->address == NULL)? US"" : host->address, pistring, @@ -3264,8 +3435,7 @@ for (cutoff_retry = 0; expired && sending the message down a pre-existing connection. */ if (!continuing && - verify_check_this_host(&(ob->serialize_hosts), NULL, host->name, - host->address, NULL) == OK) + verify_check_given_host(&ob->serialize_hosts, host) == OK) { serialize_key = string_sprintf("host-serialize-%s", host->name); if (!enq_start(serialize_key)) @@ -3335,6 +3505,20 @@ for (cutoff_retry = 0; expired && else { + host_item * thost; + /* Make a copy of the host if it is local to this invocation + of the transport. */ + + if (expanded_hosts) + { + thost = store_get(sizeof(host_item)); + *thost = *host; + thost->name = string_copy(host->name); + thost->address = string_copy(host->address); + } + else + thost = host; + if (!host_is_expired && ++unexpired_hosts_tried >= ob->hosts_max_try) { host_item *h; @@ -3354,8 +3538,8 @@ for (cutoff_retry = 0; expired && /* Attempt the delivery. */ total_hosts_tried++; - rc = smtp_deliver(addrlist, host, host_af, port, interface, tblock, - expanded_hosts != NULL, &message_defer, FALSE); + rc = smtp_deliver(addrlist, thost, host_af, port, interface, tblock, + &message_defer, FALSE); /* Yield is one of: OK => connection made, each address contains its result; @@ -3394,15 +3578,14 @@ for (cutoff_retry = 0; expired && if ( rc == DEFER && first_addr->basic_errno == ERRNO_TLSFAILURE && ob->tls_tempfail_tryclear - && verify_check_this_host(&(ob->hosts_require_tls), NULL, host->name, - host->address, NULL) != OK + && verify_check_given_host(&ob->hosts_require_tls, host) != OK ) { log_write(0, LOG_MAIN, "TLS session failure: delivering unencrypted " "to %s [%s] (not in hosts_require_tls)", host->name, host->address); first_addr = prepare_addresses(addrlist, host); - rc = smtp_deliver(addrlist, host, host_af, port, interface, tblock, - expanded_hosts != NULL, &message_defer, TRUE); + rc = smtp_deliver(addrlist, thost, host_af, port, interface, tblock, + &message_defer, TRUE); if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL) write_logs(first_addr, host); # ifdef EXPERIMENTAL_EVENT @@ -3439,7 +3622,13 @@ for (cutoff_retry = 0; expired && int delete_flag = (rc != DEFER)? rf_delete : 0; if (retry_host_key == NULL) { - retry_host_key = ob->retry_include_ip_address? + BOOL incl_ip; + if (exp_bool(addrlist, US"transport", tblock->name, D_transport, + US"retry_include_ip_address", ob->retry_include_ip_address, + ob->expand_retry_include_ip_address, &incl_ip) != OK) + incl_ip = TRUE; /* error; use most-specific retry record */ + + retry_host_key = incl_ip ? string_sprintf("T:%S:%s%s", host->name, host->address, pistring) : string_sprintf("T:%S%s", host->name, pistring); } @@ -3481,7 +3670,13 @@ for (cutoff_retry = 0; expired && int delete_flag = message_defer? 0 : rf_delete; if (retry_message_key == NULL) { - retry_message_key = ob->retry_include_ip_address? + BOOL incl_ip; + if (exp_bool(addrlist, US"transport", tblock->name, D_transport, + US"retry_include_ip_address", ob->retry_include_ip_address, + ob->expand_retry_include_ip_address, &incl_ip) != OK) + incl_ip = TRUE; /* error; use most-specific retry record */ + + retry_message_key = incl_ip ? string_sprintf("T:%S:%s%s:%s", host->name, host->address, pistring, message_id) : string_sprintf("T:%S%s:%s", host->name, pistring, message_id);