X-Git-Url: https://vcs.fsf.org/?p=exim.git;a=blobdiff_plain;f=src%2Fsrc%2Ftransports%2Fsmtp.c;h=1f0256f3d694d5fc883be76b1dc9e7b591e55ad6;hp=965ef79e4c1d8393c87cbe916387acb28bf3e4ab;hb=74f1a42304ce056cf979d22fb970faae161e3ab2;hpb=5903c6ff59527362e869fedb565c56935ce8dd68 diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 965ef79e4..1f0256f3d 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2017 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ #include "../exim.h" @@ -24,6 +24,10 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, address_retry_include_sender) }, { "allow_localhost", opt_bool, (void *)offsetof(smtp_transport_options_block, allow_localhost) }, +#ifdef EXPERIMENTAL_ARC + { "arc_sign", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, arc_sign) }, +#endif { "authenticated_sender", opt_stringptr, (void *)offsetof(smtp_transport_options_block, authenticated_sender) }, { "authenticated_sender_force", opt_bool, @@ -34,6 +38,10 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, connect_timeout) }, { "connection_max_messages", opt_int | opt_public, (void *)offsetof(transport_instance, connection_max_messages) }, +# ifdef SUPPORT_DANE + { "dane_require_tls_ciphers", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, dane_require_tls_ciphers) }, +# endif { "data_timeout", opt_time, (void *)offsetof(smtp_transport_options_block, data_timeout) }, { "delay_after_cutoff", opt_bool, @@ -43,6 +51,10 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, dkim.dkim_canon) }, { "dkim_domain", opt_stringptr, (void *)offsetof(smtp_transport_options_block, dkim.dkim_domain) }, + { "dkim_hash", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, dkim.dkim_hash) }, + { "dkim_identity", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, dkim.dkim_identity) }, { "dkim_private_key", opt_stringptr, (void *)offsetof(smtp_transport_options_block, dkim.dkim_private_key) }, { "dkim_selector", opt_stringptr, @@ -51,6 +63,8 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, dkim.dkim_sign_headers) }, { "dkim_strict", opt_stringptr, (void *)offsetof(smtp_transport_options_block, dkim.dkim_strict) }, + { "dkim_timestamps", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, dkim.dkim_timestamps) }, #endif { "dns_qualify_single", opt_bool, (void *)offsetof(smtp_transport_options_block, dns_qualify_single) }, @@ -101,7 +115,7 @@ optionlist smtp_transport_options[] = { { "hosts_require_auth", opt_stringptr, (void *)offsetof(smtp_transport_options_block, hosts_require_auth) }, #ifdef SUPPORT_TLS -# ifdef EXPERIMENTAL_DANE +# ifdef SUPPORT_DANE { "hosts_require_dane", opt_stringptr, (void *)offsetof(smtp_transport_options_block, hosts_require_dane) }, # endif @@ -116,7 +130,7 @@ optionlist smtp_transport_options[] = { (void *)offsetof(smtp_transport_options_block, hosts_try_auth) }, { "hosts_try_chunking", opt_stringptr, (void *)offsetof(smtp_transport_options_block, hosts_try_chunking) }, -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) +#if defined(SUPPORT_TLS) && defined(SUPPORT_DANE) { "hosts_try_dane", opt_stringptr, (void *)offsetof(smtp_transport_options_block, hosts_try_dane) }, #endif @@ -201,87 +215,98 @@ void smtp_transport_closedown(transport_instance *tblock) {} /* Default private options block for the smtp transport. */ smtp_transport_options_block smtp_transport_option_defaults = { - NULL, /* hosts */ - NULL, /* fallback_hosts */ - NULL, /* hostlist */ - NULL, /* fallback_hostlist */ - NULL, /* authenticated_sender */ - US"$primary_hostname", /* helo_data */ - NULL, /* interface */ - NULL, /* port */ - US"smtp", /* protocol */ - NULL, /* DSCP */ - NULL, /* serialize_hosts */ - NULL, /* hosts_try_auth */ - NULL, /* hosts_require_auth */ - US"*", /* hosts_try_chunking */ -#ifdef EXPERIMENTAL_DANE - NULL, /* hosts_try_dane */ - NULL, /* hosts_require_dane */ + .hosts = NULL, + .fallback_hosts = NULL, + .hostlist = NULL, + .fallback_hostlist = NULL, + .helo_data = US"$primary_hostname", + .interface = NULL, + .port = NULL, + .protocol = US"smtp", + .dscp = NULL, + .serialize_hosts = NULL, + .hosts_try_auth = NULL, + .hosts_require_auth = NULL, + .hosts_try_chunking = US"*", +#ifdef SUPPORT_DANE + .hosts_try_dane = NULL, + .hosts_require_dane = NULL, + .dane_require_tls_ciphers = NULL, #endif - NULL, /* hosts_try_fastopen */ + .hosts_try_fastopen = NULL, #ifndef DISABLE_PRDR - US"*", /* hosts_try_prdr */ + .hosts_try_prdr = US"*", #endif #ifndef DISABLE_OCSP - US"*", /* hosts_request_ocsp (except under DANE; tls_client_start()) */ - NULL, /* hosts_require_ocsp */ + .hosts_request_ocsp = US"*", /* hosts_request_ocsp (except under DANE; tls_client_start()) */ + .hosts_require_ocsp = NULL, #endif - NULL, /* hosts_require_tls */ - NULL, /* hosts_avoid_tls */ - NULL, /* hosts_verify_avoid_tls */ - NULL, /* hosts_avoid_pipelining */ - NULL, /* hosts_avoid_esmtp */ + .hosts_require_tls = NULL, + .hosts_avoid_tls = NULL, + .hosts_verify_avoid_tls = NULL, + .hosts_avoid_pipelining = NULL, + .hosts_avoid_esmtp = NULL, #ifdef SUPPORT_TLS - NULL, /* hosts_nopass_tls */ - US"*", /* hosts_noproxy_tls */ + .hosts_nopass_tls = NULL, + .hosts_noproxy_tls = US"*", #endif - 5*60, /* command_timeout */ - 5*60, /* connect_timeout; shorter system default overrides */ - 5*60, /* data timeout */ - 10*60, /* final timeout */ - 1024, /* size_addition */ - 5, /* hosts_max_try */ - 50, /* hosts_max_try_hardlimit */ - TRUE, /* address_retry_include_sender */ - FALSE, /* allow_localhost */ - FALSE, /* authenticated_sender_force */ - FALSE, /* gethostbyname */ - TRUE, /* dns_qualify_single */ - FALSE, /* dns_search_parents */ - { NULL, NULL }, /* dnssec_domains {request,require} */ - TRUE, /* delay_after_cutoff */ - FALSE, /* hosts_override */ - FALSE, /* hosts_randomize */ - TRUE, /* keepalive */ - FALSE, /* lmtp_ignore_quota */ - NULL, /* expand_retry_include_ip_address */ - TRUE /* retry_include_ip_address */ + .command_timeout = 5*60, + .connect_timeout = 5*60, + .data_timeout = 5*60, + .final_timeout = 10*60, + .size_addition = 1024, + .hosts_max_try = 5, + .hosts_max_try_hardlimit = 50, + .address_retry_include_sender = TRUE, + .allow_localhost = FALSE, + .authenticated_sender_force = FALSE, + .gethostbyname = FALSE, + .dns_qualify_single = TRUE, + .dns_search_parents = FALSE, + .dnssec = { .request=NULL, .require=NULL }, + .delay_after_cutoff = TRUE, + .hosts_override = FALSE, + .hosts_randomize = FALSE, + .keepalive = TRUE, + .lmtp_ignore_quota = FALSE, + .expand_retry_include_ip_address = NULL, + .retry_include_ip_address = TRUE, #ifdef SUPPORT_SOCKS - ,NULL /* socks_proxy */ + .socks_proxy = NULL, #endif #ifdef SUPPORT_TLS - ,NULL, /* tls_certificate */ - NULL, /* tls_crl */ - NULL, /* tls_privatekey */ - NULL, /* tls_require_ciphers */ - NULL, /* tls_sni */ - US"system", /* tls_verify_certificates */ - EXIM_CLIENT_DH_DEFAULT_MIN_BITS, - /* tls_dh_min_bits */ - TRUE, /* tls_tempfail_tryclear */ - NULL, /* tls_verify_hosts */ - US"*", /* tls_try_verify_hosts */ - US"*" /* tls_verify_cert_hostnames */ + .tls_certificate = NULL, + .tls_crl = NULL, + .tls_privatekey = NULL, + .tls_require_ciphers = NULL, + .tls_sni = NULL, + .tls_verify_certificates = US"system", + .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS, + .tls_tempfail_tryclear = TRUE, + .tls_verify_hosts = NULL, + .tls_try_verify_hosts = US"*", + .tls_verify_cert_hostnames = US"*", #endif #ifndef DISABLE_DKIM - , {NULL, /* dkim_canon */ - NULL, /* dkim_domain */ - NULL, /* dkim_private_key */ - NULL, /* dkim_selector */ - NULL, /* dkim_sign_headers */ - NULL, /* dkim_strict */ - FALSE} /* dot_stuffed */ + .dkim = + {.dkim_domain = NULL, + .dkim_identity = NULL, + .dkim_private_key = NULL, + .dkim_selector = NULL, + .dkim_canon = NULL, + .dkim_sign_headers = NULL, + .dkim_strict = NULL, + .dkim_hash = US"sha256", + .dkim_timestamps = NULL, + .dot_stuffed = FALSE, + .force_bodyhash = FALSE, +# ifdef EXPERIMENTAL_ARC + .arc_signspec = NULL, +# endif + }, +# ifdef EXPERIMENTAL_ARC + .arc_sign = NULL, +# endif #endif }; @@ -617,34 +642,34 @@ return FALSE; /* This writes to the main log and to the message log. Arguments: - addr the address item containing error information host the current host + detail the current message (addr_item->message) + basic_errno the errno (addr_item->basic_errno) Returns: nothing */ static void -write_logs(address_item *addr, host_item *host) +write_logs(const host_item *host, const uschar *suffix, int basic_errno) { -uschar * message = LOGGING(outgoing_port) + + +uschar *message = LOGGING(outgoing_port) ? string_sprintf("H=%s [%s]:%d", host->name, host->address, host->port == PORT_NONE ? 25 : host->port) : string_sprintf("H=%s [%s]", host->name, host->address); -if (addr->message) +if (suffix) { - message = string_sprintf("%s: %s", message, addr->message); - if (addr->basic_errno > 0) - message = string_sprintf("%s: %s", message, strerror(addr->basic_errno)); - log_write(0, LOG_MAIN, "%s", message); - deliver_msglog("%s %s\n", tod_stamp(tod_log), message); + message = string_sprintf("%s: %s", message, suffix); + if (basic_errno > 0) + message = string_sprintf("%s: %s", message, strerror(basic_errno)); } else - { - const uschar * s = exim_errstr(addr->basic_errno); - log_write(0, LOG_MAIN, "%s %s", message, s); - deliver_msglog("%s %s %s\n", tod_stamp(tod_log), message, s); - } + message = string_sprintf("%s %s", message, exim_errstr(basic_errno)); + +log_write(0, LOG_MAIN, "%s", message); +deliver_msglog("%s %s\n", tod_stamp(tod_log), message); } static void @@ -1184,7 +1209,7 @@ return FALSE; -#ifdef EXPERIMENTAL_DANE +#ifdef SUPPORT_DANE /* Lookup TLSA record for host/port. Return: OK success with dnssec; DANE mode DEFER Do not use this host now, may retry later @@ -1211,14 +1236,15 @@ DEBUG(D_transport) switch (rc) { - case DNS_SUCCEED: - if (sec) return OK; - - log_write(0, LOG_MAIN, "DANE error: TLSA lookup not DNSSEC"); - /*FALLTHROUGH*/ case DNS_AGAIN: return DEFER; /* just defer this TLS'd conn */ + case DNS_SUCCEED: + if (sec) return OK; + log_write(0, LOG_MAIN, + "DANE error: TLSA lookup for %s not DNSSEC", host->name); + /*FALLTRHOUGH*/ + case DNS_NODATA: /* no TLSA RR for this lookup */ case DNS_NOMATCH: /* no records at all for this lookup */ return dane_required ? FAIL : FAIL_FORCED; @@ -1483,7 +1509,7 @@ Returns: OK - the connection was made and the delivery attempted; int smtp_setup_conn(smtp_context * sx, BOOL suppress_tls) { -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) +#if defined(SUPPORT_TLS) && defined(SUPPORT_DANE) dns_answer tlsa_dnsa; #endif BOOL pass_message = FALSE; @@ -1505,7 +1531,7 @@ sx->esmtp_sent = FALSE; sx->utf8_needed = FALSE; #endif sx->dsn_all_lasthop = TRUE; -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) +#if defined(SUPPORT_TLS) && defined(SUPPORT_DANE) sx->dane = FALSE; sx->dane_required = verify_check_given_host(&sx->ob->hosts_require_dane, sx->host) == OK; #endif @@ -1579,7 +1605,7 @@ if (!continue_hostname) smtp_port_for_connect(sx->host, sx->port); -#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE) +#if defined(SUPPORT_TLS) && defined(SUPPORT_DANE) /* Do TLSA lookup for DANE */ { tls_out.dane_verified = FALSE; @@ -1600,6 +1626,9 @@ if (!continue_hostname) string_sprintf("DANE error: tlsa lookup %s", rc == DEFER ? "DEFER" : "FAIL"), rc, FALSE); + (void) event_raise(sx->tblock->event_action, + US"dane:fail", sx->dane_required + ? US"dane-required" : US"dnssec-invalid"); return rc; } } @@ -1608,6 +1637,8 @@ if (!continue_hostname) set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, string_sprintf("DANE error: %s lookup not DNSSEC", sx->host->name), FAIL, FALSE); + (void) event_raise(sx->tblock->event_action, + US"dane:fail", US"dane-required"); return FAIL; } } @@ -1615,11 +1646,13 @@ if (!continue_hostname) /* Make the TCP connection */ - sx->inblock.sock = sx->outblock.sock = + sx->cctx.sock = smtp_connect(sx->host, sx->host_af, sx->interface, sx->ob->connect_timeout, sx->tblock); + sx->cctx.tls_ctx = NULL; + sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; - if (sx->inblock.sock < 0) + if (sx->cctx.sock < 0) { uschar * msg = NULL; if (sx->verify) @@ -1672,7 +1705,7 @@ if (!continue_hostname) BOOL good_response; #ifdef TCP_QUICKACK - (void) setsockopt(sx->inblock.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); #endif good_response = smtp_read_response(&sx->inblock, sx->buffer, sizeof(sx->buffer), '2', sx->ob->command_timeout); @@ -1814,7 +1847,7 @@ goto SEND_QUIT; errno = ERRNO_SMTPCLOSED; goto EHLOHELO_FAILED; } - Ustrncpy(sx->buffer, rsp, sizeof(sx->buffer)/2); + memmove(sx->buffer, rsp, Ustrlen(rsp)); goto RESPONSE_FAILED; } } @@ -1851,16 +1884,18 @@ separate - we could match up by host ip+port as a bodge. */ else { - if (cutthrough.fd >= 0 && cutthrough.callout_hold_only) + if (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only) { - sx->inblock.sock = sx->outblock.sock = cutthrough.fd; + sx->cctx = cutthrough.cctx; sx->host->port = sx->port = cutthrough.host.port; } else { - sx->inblock.sock = sx->outblock.sock = 0; /* stdin */ + sx->cctx.sock = 0; /* stdin */ + sx->cctx.tls_ctx = NULL; smtp_port_for_connect(sx->host, sx->port); /* Record the port that was used */ } + sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; smtp_command = big_buffer; sx->helo_data = NULL; /* ensure we re-expand ob->helo_data */ @@ -1868,7 +1903,8 @@ else held-open verify connection with TLS, nothing more to do. */ if ( continue_proxy_cipher - || (cutthrough.fd >= 0 && cutthrough.callout_hold_only && cutthrough.is_tls) + || (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only + && cutthrough.is_tls) ) { sx->peer_offered = smtp_peer_options; @@ -1928,22 +1964,28 @@ if ( smtp_peer_options & OPTION_TLS { address_item * addr; uschar * errstr; - int rc = tls_client_start(sx->inblock.sock, sx->host, sx->addrlist, sx->tblock, -# ifdef EXPERIMENTAL_DANE + sx->cctx.tls_ctx = tls_client_start(sx->cctx.sock, sx->host, + sx->addrlist, sx->tblock, +# ifdef SUPPORT_DANE sx->dane ? &tlsa_dnsa : NULL, # endif - &errstr); - - /* TLS negotiation failed; give an error. From outside, this function may - be called again to try in clear on a new connection, if the options permit - it for this host. */ + &tls_out, &errstr); - if (rc != OK) + if (!sx->cctx.tls_ctx) { -# ifdef EXPERIMENTAL_DANE - if (sx->dane) log_write(0, LOG_MAIN, + /* TLS negotiation failed; give an error. From outside, this function may + be called again to try in clear on a new connection, if the options permit + it for this host. */ + +# ifdef SUPPORT_DANE + if (sx->dane) + { + log_write(0, LOG_MAIN, "DANE attempt failed; TLS connection to %s [%s]: %s", sx->host->name, sx->host->address, errstr); + (void) event_raise(sx->tblock->event_action, + US"dane:fail", US"validation-failure"); /* could do with better detail */ + } # endif errno = ERRNO_TLSFAILURE; @@ -1977,7 +2019,7 @@ another process, and so we won't have expanded helo_data above. We have to expand it here. $sending_ip_address and $sending_port are set up right at the start of the Exim process (in exim.c). */ -if (tls_out.active >= 0) +if (tls_out.active.sock >= 0) { char *greeting_cmd; BOOL good_response; @@ -2027,7 +2069,7 @@ if (tls_out.active >= 0) have one. */ else if ( sx->smtps -# ifdef EXPERIMENTAL_DANE +# ifdef SUPPORT_DANE || sx->dane # endif || verify_check_given_host(&sx->ob->hosts_require_tls, sx->host) == OK @@ -2037,6 +2079,13 @@ else if ( sx->smtps message = string_sprintf("a TLS session is required, but %s", smtp_peer_options & OPTION_TLS ? "an attempt to start TLS failed" : "the server did not offer TLS support"); +# ifdef SUPPORT_DANE + if (sx->dane) + (void) event_raise(sx->tblock->event_action, US"dane:fail", + smtp_peer_options & OPTION_TLS + ? US"validation-failure" /* could do with better detail */ + : US"starttls-not-supported"); +# endif goto TLS_FAILED; } #endif /*SUPPORT_TLS*/ @@ -2048,7 +2097,7 @@ we skip this. */ if (continue_hostname == NULL #ifdef SUPPORT_TLS - || tls_out.active >= 0 + || tls_out.active.sock >= 0 #endif ) { @@ -2170,11 +2219,6 @@ return OK; sx->send_quit = FALSE; goto FAILED; - /* This label is jumped to directly when a TLS negotiation has failed, - or was not done for a host for which it is required. Values will be set - in message and errno, and setting_up will always be true. Treat as - a temporary error. */ - EHLOHELO_FAILED: code = '4'; message = string_sprintf("Remote host closed connection in response to %s" @@ -2182,6 +2226,11 @@ return OK; sx->send_quit = FALSE; goto FAILED; + /* This label is jumped to directly when a TLS negotiation has failed, + or was not done for a host for which it is required. Values will be set + in message and errno, and setting_up will always be true. Treat as + a temporary error. */ + #ifdef SUPPORT_TLS TLS_FAILED: code = '4'; @@ -2227,7 +2276,11 @@ if (sx->send_quit) (void)smtp_write_command(&sx->outblock, SCMD_FLUSH, "QUIT\r\n"); #ifdef SUPPORT_TLS -tls_close(FALSE, TRUE); +if (sx->cctx.tls_ctx) + { + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + sx->cctx.tls_ctx = NULL; + } #endif /* Close the socket, and return the appropriate value, first setting @@ -2238,14 +2291,14 @@ remote_max_parallel is forced to 1 when delivering over an existing connection, HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); if (sx->send_quit) { - shutdown(sx->outblock.sock, SHUT_WR); - if (fcntl(sx->inblock.sock, F_SETFL, O_NONBLOCK) == 0) - for (rc = 16; read(sx->inblock.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && rc > 0;) + shutdown(sx->cctx.sock, SHUT_WR); + if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) + for (rc = 16; read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && rc > 0;) rc--; /* drain socket */ sx->send_quit = FALSE; } -(void)close(sx->inblock.sock); -sx->inblock.sock = sx->outblock.sock = -1; +(void)close(sx->cctx.sock); +sx->cctx.sock = -1; #ifndef DISABLE_EVENT (void) event_raise(sx->tblock->event_action, US"tcp:close", NULL); @@ -2504,6 +2557,12 @@ for (addr = sx->first_addr, address_count = 0; BOOL no_flush; uschar * rcpt_addr; + if (tcp_out_fastopen && !tcp_out_fastopen_logged) + { + setflag(addr, af_tcp_fastopen_conn); + if (tcp_out_fastopen > 1) setflag(addr, af_tcp_fastopen); + } + addr->dsn_aware = sx->peer_offered & OPTION_DSN ? dsn_support_yes : dsn_support_no; @@ -2557,6 +2616,7 @@ for (addr = sx->first_addr, address_count = 0; } } /* Loop for next address */ +tcp_out_fastopen_logged = TRUE; sx->next_addr = addr; return 0; } @@ -2567,29 +2627,40 @@ return 0; * Proxy TLS connection for another transport process * ******************************************************/ /* -Use the given buffer as a staging area, and select on both the given fd -and the TLS'd client-fd for data to read (per the coding in ip_recv() and -fd_ready() this is legitimate). Do blocking full-size writes, and reads -under a timeout. +Close the unused end of the pipe, fork once more, then use the given buffer +as a staging area, and select on both the given fd and the TLS'd client-fd for +data to read (per the coding in ip_recv() and fd_ready() this is legitimate). +Do blocking full-size writes, and reads under a timeout. Once both input +channels are closed, exit the process. Arguments: + ct_ctx tls context buf space to use for buffering bufsiz size of buffer - proxy_fd comms to proxied process + pfd pipe filedescriptor array; [0] is comms to proxied process timeout per-read timeout, seconds */ void -smtp_proxy_tls(uschar * buf, size_t bsize, int proxy_fd, int timeout) +smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd, + int timeout) { fd_set rfds, efds; -int max_fd = MAX(proxy_fd, tls_out.active) + 1; +int max_fd = MAX(pfd[0], tls_out.active.sock) + 1; int rc, i, fd_bits, nbytes; +close(pfd[1]); +if ((rc = fork())) + { + DEBUG(D_transport) debug_printf("proxy-proc final-pid %d\n", rc); + _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS); + } + +if (running_in_test_harness) millisleep(100); /* let parent debug out */ set_process_info("proxying TLS connection for continued transport"); FD_ZERO(&rfds); -FD_SET(tls_out.active, &rfds); -FD_SET(proxy_fd, &rfds); +FD_SET(tls_out.active.sock, &rfds); +FD_SET(pfd[0], &rfds); for (fd_bits = 3; fd_bits; ) { @@ -2611,51 +2682,55 @@ for (fd_bits = 3; fd_bits; ) if (rc <= 0) { DEBUG(D_transport) if (rc == 0) debug_printf("%s: timed out\n", __FUNCTION__); - return; + goto done; } - if (FD_ISSET(tls_out.active, &efds) || FD_ISSET(proxy_fd, &efds)) + if (FD_ISSET(tls_out.active.sock, &efds) || FD_ISSET(pfd[0], &efds)) { DEBUG(D_transport) debug_printf("select: exceptional cond on %s fd\n", - FD_ISSET(proxy_fd, &efds) ? "proxy" : "tls"); - return; + FD_ISSET(pfd[0], &efds) ? "proxy" : "tls"); + goto done; } } - while (rc < 0 || !(FD_ISSET(tls_out.active, &rfds) || FD_ISSET(proxy_fd, &rfds))); + while (rc < 0 || !(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))); /* handle inbound data */ - if (FD_ISSET(tls_out.active, &rfds)) - if ((rc = tls_read(FALSE, buf, bsize)) <= 0) + if (FD_ISSET(tls_out.active.sock, &rfds)) + if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) { fd_bits &= ~1; - FD_CLR(tls_out.active, &rfds); - shutdown(proxy_fd, SHUT_WR); + FD_CLR(tls_out.active.sock, &rfds); + shutdown(pfd[0], SHUT_WR); timeout = 5; } else { for (nbytes = 0; rc - nbytes > 0; nbytes += i) - if ((i = write(proxy_fd, buf + nbytes, rc - nbytes)) < 0) return; + if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done; } else if (fd_bits & 1) - FD_SET(tls_out.active, &rfds); + FD_SET(tls_out.active.sock, &rfds); /* handle outbound data */ - if (FD_ISSET(proxy_fd, &rfds)) - if ((rc = read(proxy_fd, buf, bsize)) <= 0) + if (FD_ISSET(pfd[0], &rfds)) + if ((rc = read(pfd[0], buf, bsize)) <= 0) { fd_bits = 0; - tls_close(FALSE, TRUE); + tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); } else { for (nbytes = 0; rc - nbytes > 0; nbytes += i) - if ((i = tls_write(FALSE, buf + nbytes, rc - nbytes, FALSE)) < 0) - return; + if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0) + goto done; } else if (fd_bits & 2) - FD_SET(proxy_fd, &rfds); + FD_SET(pfd[0], &rfds); } + +done: + if (running_in_test_harness) millisleep(100); /* let logging complete */ + exim_exit(0, US"TLS proxy"); } #endif @@ -2889,7 +2964,7 @@ if (!(sx.peer_offered & OPTION_CHUNKING) && !sx.ok) else { transport_ctx tctx = { - {sx.inblock.sock}, + {sx.cctx.sock}, /*XXX will this need TLS info? */ tblock, addrlist, US".", US"..", /* Escaping strings */ @@ -2938,6 +3013,30 @@ else transport_count = 0; #ifndef DISABLE_DKIM + dkim_exim_sign_init(); +# ifdef EXPERIMENTAL_ARC + { + uschar * s = sx.ob->arc_sign; + if (s) + { + if (!(sx.ob->dkim.arc_signspec = s = expand_string(s))) + { + if (!expand_string_forcedfail) + { + message = US"failed to expand arc_sign"; + sx.ok = FALSE; + goto SEND_FAILED; + } + } + else if (*s) + { + /* Ask dkim code to hash the body for ARC */ + (void) arc_ams_setup_sign_bodyhash(); + sx.ob->dkim.force_bodyhash = TRUE; + } + } + } +# endif sx.ok = dkim_transport_write_message(&tctx, &sx.ob->dkim, CUSS &message); #else sx.ok = transport_write_message(&tctx, 0); @@ -3121,9 +3220,9 @@ else addr->special_action = flag; addr->message = conf; #ifndef DISABLE_PRDR - if (sx.prdr_active) addr->flags |= af_prdr_used; + if (sx.prdr_active) setflag(addr, af_prdr_used); #endif - if (sx.peer_offered & OPTION_CHUNKING) addr->flags |= af_chunking_used; + if (sx.peer_offered & OPTION_CHUNKING) setflag(addr, af_chunking_used); flag = '-'; #ifndef DISABLE_PRDR @@ -3151,8 +3250,11 @@ else #ifndef DISABLE_PRDR if (sx.prdr_active) { + const uschar * overall_message; + /* PRDR - get the final, overall response. For any non-success upgrade all the address statuses. */ + sx.ok = smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer), '2', sx.ob->final_timeout); if (!sx.ok) @@ -3168,7 +3270,14 @@ else goto RESPONSE_FAILED; } - /* Update the journal, or setup retry. */ + /* Append the overall response to the individual PRDR response for logging + and update the journal, or setup retry. */ + + overall_message = string_printing(sx.buffer); + for (addr = addrlist; addr != sx.first_addr; addr = addr->next) + if (addr->transport_return == OK) + addr->message = string_sprintf("%s\\n%s", addr->message, overall_message); + for (addr = addrlist; addr != sx.first_addr; addr = addr->next) if (addr->transport_return == OK) { @@ -3294,8 +3403,9 @@ if (!sx.ok) set_rc = DEFER; if (save_errno > 0) message = US string_sprintf("%s: %s", message, strerror(save_errno)); - if (host->next != NULL) log_write(0, LOG_MAIN, "%s", message); - msglog_line(host, message); + + write_logs(host, message, sx.first_addr ? sx.first_addr->basic_errno : 0); + *message_defer = TRUE; } } @@ -3367,7 +3477,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit) || continue_more || ( #ifdef SUPPORT_TLS - ( tls_out.active < 0 && !continue_proxy_cipher + ( tls_out.active.sock < 0 && !continue_proxy_cipher || verify_check_given_host(&sx.ob->hosts_nopass_tls, host) != OK ) && @@ -3405,7 +3515,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit) if (sx.ok) { int pfd[2]; - int socket_fd = sx.inblock.sock; + int socket_fd = sx.cctx.sock; if (sx.first_addr != NULL) /* More addresses still to be sent */ @@ -3420,7 +3530,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit) the connection still open. */ #ifdef SUPPORT_TLS - if (tls_out.active >= 0) + if (tls_out.active.sock >= 0) if ( continue_more || verify_check_given_host(&sx.ob->hosts_noproxy_tls, host) == OK) { @@ -3430,7 +3540,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit) a new EHLO. If we don't get a good response, we don't attempt to pass the socket on. */ - tls_close(FALSE, TRUE); + tls_close(sx.cctx.tls_ctx, TLS_SHUTDOWN_WAIT); smtp_peer_options = smtp_peer_options_wrap; sx.ok = !sx.smtps && smtp_write_command(&sx.outblock, SCMD_FLUSH, @@ -3472,36 +3582,38 @@ propagate it from the initial { sx.send_quit = FALSE; - /* If TLS is still active, we need to proxy it for the transport we + /* We have passed the client socket to a fresh transport process. + If TLS is still active, we need to proxy it for the transport we just passed the baton to. Fork a child to to do it, and return to get logging done asap. Which way to place the work makes assumptions about post-fork prioritisation which may not hold on all platforms. */ #ifdef SUPPORT_TLS - if (tls_out.active >= 0) + if (tls_out.active.sock >= 0) { int pid = fork(); + if (pid == 0) /* child; fork again to disconnect totally */ + { + if (running_in_test_harness) millisleep(100); /* let parent debug out */ + /* does not return */ + smtp_proxy_tls(sx.cctx.tls_ctx, sx.buffer, sizeof(sx.buffer), pfd, + sx.ob->command_timeout); + } + if (pid > 0) /* parent */ { DEBUG(D_transport) debug_printf("proxy-proc inter-pid %d\n", pid); close(pfd[0]); + /* tidy the inter-proc to disconn the proxy proc */ waitpid(pid, NULL, 0); - tls_close(FALSE, FALSE); - (void)close(sx.inblock.sock); + tls_close(sx.cctx.tls_ctx, TLS_NO_SHUTDOWN); + sx.cctx.tls_ctx = NULL; + (void)close(sx.cctx.sock); + sx.cctx.sock = -1; continue_transport = NULL; continue_hostname = NULL; return yield; } - else if (pid == 0) /* child; fork again to disconnect totally */ - { - close(pfd[1]); - if ((pid = fork())) - { - DEBUG(D_transport) debug_printf("proxy-prox final-pid %d\n", pid); - _exit(pid ? EXIT_FAILURE : EXIT_SUCCESS); - } - smtp_proxy_tls(sx.buffer, sizeof(sx.buffer), pfd[0], sx.ob->command_timeout); - exim_exit(0); - } + log_write(0, LOG_PANIC_DIE, "fork failed"); } #endif } @@ -3541,7 +3653,7 @@ if (sx.send_quit) (void)smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n") END_OFF: #ifdef SUPPORT_TLS -tls_close(FALSE, TRUE); +tls_close(sx.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); #endif /* Close the socket, and return the appropriate value, first setting @@ -3557,12 +3669,12 @@ case continue_more won't get set. */ HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); if (sx.send_quit) { - shutdown(sx.outblock.sock, SHUT_WR); - if (fcntl(sx.inblock.sock, F_SETFL, O_NONBLOCK) == 0) - for (rc = 16; read(sx.inblock.sock, sx.inbuffer, sizeof(sx.inbuffer)) > 0 && rc > 0;) + shutdown(sx.cctx.sock, SHUT_WR); + if (fcntl(sx.cctx.sock, F_SETFL, O_NONBLOCK) == 0) + for (rc = 16; read(sx.cctx.sock, sx.inbuffer, sizeof(sx.inbuffer)) > 0 && rc > 0;) rc--; /* drain socket */ } -(void)close(sx.inblock.sock); +(void)close(sx.cctx.sock); #ifndef DISABLE_EVENT (void) event_raise(tblock->event_action, US"tcp:close", NULL); @@ -3598,19 +3710,24 @@ smtp_transport_closedown(transport_instance *tblock) { smtp_transport_options_block *ob = (smtp_transport_options_block *)tblock->options_block; +client_conn_ctx cctx; smtp_inblock inblock; smtp_outblock outblock; uschar buffer[256]; uschar inbuffer[4096]; uschar outbuffer[16]; -inblock.sock = fileno(stdin); +/*XXX really we need an active-smtp-client ctx, rather than assuming stdout */ +cctx.sock = fileno(stdin); +cctx.tls_ctx = cctx.sock == tls_out.active.sock ? tls_out.active.tls_ctx : NULL; + +inblock.cctx = &cctx; inblock.buffer = inbuffer; inblock.buffersize = sizeof(inbuffer); inblock.ptr = inbuffer; inblock.ptrend = inbuffer; -outblock.sock = inblock.sock; +outblock.cctx = &cctx; outblock.buffersize = sizeof(outbuffer); outblock.buffer = outbuffer; outblock.ptr = outbuffer; @@ -3620,7 +3737,7 @@ outblock.authenticating = FALSE; (void)smtp_write_command(&outblock, SCMD_FLUSH, "QUIT\r\n"); (void)smtp_read_response(&inblock, buffer, sizeof(buffer), '2', ob->command_timeout); -(void)close(inblock.sock); +(void)close(cctx.sock); } @@ -3705,7 +3822,7 @@ uschar *tid = string_sprintf("%s transport", tblock->name); smtp_transport_options_block *ob = (smtp_transport_options_block *)(tblock->options_block); host_item *hostlist = addrlist->host_list; -host_item *host = NULL; +host_item *host; DEBUG(D_transport) { @@ -3716,12 +3833,12 @@ DEBUG(D_transport) { debug_printf("hostlist:\n"); for (host = hostlist; host; host = host->next) - debug_printf(" %s:%d\n", host->name, host->port); + debug_printf(" '%s' IP %s port %d\n", host->name, host->address, host->port); } if (continue_hostname) debug_printf("already connected to %s [%s] (on fd %d)\n", continue_hostname, continue_host_address, - cutthrough.fd >= 0 ? cutthrough.fd : 0); + cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0); } /* Set the flag requesting that these hosts be added to the waiting @@ -3897,7 +4014,9 @@ for (cutoff_retry = 0; { host_item *nexthost = NULL; int unexpired_hosts_tried = 0; + BOOL continue_host_tried = FALSE; +retry_non_continued: for (host = hostlist; host && unexpired_hosts_tried < ob->hosts_max_try @@ -3962,7 +4081,7 @@ for (cutoff_retry = 0; /* Find by name if so configured, or if it's an IP address. We don't just copy the IP address, because we need the test-for-local to happen. */ - flags = HOST_FIND_BY_A; + flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA; if (ob->dns_qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE; if (ob->dns_search_parents) flags |= HOST_FIND_SEARCH_PARENTS; @@ -4030,14 +4149,16 @@ for (cutoff_retry = 0; result of the lookup. Set expired FALSE, to save the outer loop executing twice. */ - if ( continue_hostname - && ( Ustrcmp(continue_hostname, host->name) != 0 - || Ustrcmp(continue_host_address, host->address) != 0 - ) ) - { - expired = FALSE; - continue; /* With next host */ - } + if (continue_hostname) + if ( Ustrcmp(continue_hostname, host->name) != 0 + || Ustrcmp(continue_host_address, host->address) != 0 + ) + { + expired = FALSE; + continue; /* With next host */ + } + else + continue_host_tried = TRUE; /* Reset the default next host in case a multihomed host whose addresses are not looked up till just above added to the host list. */ @@ -4303,7 +4424,7 @@ for (cutoff_retry = 0; if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL && first_addr->basic_errno != ERRNO_TLSFAILURE) - write_logs(first_addr, host); + write_logs(host, first_addr->message, first_addr->basic_errno); #ifndef DISABLE_EVENT if (rc == DEFER) @@ -4333,7 +4454,7 @@ for (cutoff_retry = 0; rc = smtp_deliver(addrlist, thost, host_af, defport, interface, tblock, &message_defer, TRUE); if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL) - write_logs(first_addr, host); + write_logs(host, first_addr->message, first_addr->basic_errno); # ifndef DISABLE_EVENT if (rc == DEFER) deferred_event_raise(first_addr, host); @@ -4495,6 +4616,38 @@ for (cutoff_retry = 0; } } /* End of loop for trying multiple hosts. */ + /* If we failed to find a matching host in the list, for an already-open + connection, just close it and start over with the list. This can happen + for routing that changes from run to run, or big multi-IP sites with + round-robin DNS. */ + + if (continue_hostname && !continue_host_tried) + { + int fd = cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0; + + DEBUG(D_transport) debug_printf("no hosts match already-open connection\n"); +#ifdef SUPPORT_TLS + /* A TLS conn could be open for a cutthrough, but not for a plain continued- + transport */ +/*XXX doublecheck that! */ + + if (cutthrough.cctx.sock >= 0 && cutthrough.is_tls) + { + (void) tls_write(cutthrough.cctx.tls_ctx, US"QUIT\r\n", 6, FALSE); + tls_close(cutthrough.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + cutthrough.cctx.tls_ctx = NULL; + cutthrough.is_tls = FALSE; + } + else +#else + (void) write(fd, US"QUIT\r\n", 6); +#endif + (void) close(fd); + cutthrough.cctx.sock = -1; + continue_hostname = NULL; + goto retry_non_continued; + } + /* This is the end of the loop that repeats iff expired is TRUE and ob->delay_after_cutoff is FALSE. The second time round we will try those hosts that haven't been tried since the message arrived. */