From b536a578fbabdc9d39da53d54a8d7700ba537431 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 30 Oct 2018 22:09:15 +0000 Subject: [PATCH] MacOS: TCP Fast Open --- doc/doc-txt/NewStuff | 2 ++ src/OS/os.h-Darwin | 8 ++--- src/src/daemon.c | 16 +++++++-- src/src/expand.c | 1 + src/src/ip.c | 53 +++++++++++++++++++++++----- src/src/malware.c | 1 + src/src/smtp_out.c | 3 +- src/src/transports/smtp_socks.c | 1 + src/src/verify.c | 1 + test/scripts/1990-TCP-Fast-Open/1990 | 22 ++++++++---- test/src/server.c | 17 +++++---- 11 files changed, 96 insertions(+), 29 deletions(-) diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 10461aa05..cc9721ada 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -29,6 +29,8 @@ Version 4.92 8. A "noutf8" debug option, for disabling the UTF-8 characters in debug output. + 9. TCP Fast Open support on MacOS. + Version 4.91 -------------- diff --git a/src/OS/os.h-Darwin b/src/OS/os.h-Darwin index 67aeac913..4667689c9 100644 --- a/src/OS/os.h-Darwin +++ b/src/OS/os.h-Darwin @@ -51,10 +51,8 @@ in "man 2 getgroups". */ #define _DARWIN_UNLIMITED_GETGROUPS #define EXIM_GROUPLIST_SIZE 64 -/* TCP_FASTOPEN support. For the moment, claim there is none -(the probe fails; unsure why). -Sometime in the future need to investigate connectex(). */ - -#define EXIM_TFO_PROBE +/* TCP Fast Open: Darwin uses a connectex() call +rather than a modified sendto() */ +#define EXIM_TFO_CONNECTX /* End */ diff --git a/src/src/daemon.c b/src/src/daemon.c index ee9ddcc4f..1a15d46c0 100644 --- a/src/src/daemon.c +++ b/src/src/daemon.c @@ -1458,7 +1458,7 @@ if (f.daemon_listen && !f.inetd_wait_mode) else debug_printf("listening on %s port %d\n", ipa->address, ipa->port); -#ifdef TCP_FASTOPEN +#if defined(TCP_FASTOPEN) && !defined(__APPLE__) if ( f.tcp_fastopen_ok && setsockopt(listen_sockets[sk], IPPROTO_TCP, TCP_FASTOPEN, &smtp_connect_backlog, sizeof(smtp_connect_backlog))) @@ -1471,7 +1471,19 @@ if (f.daemon_listen && !f.inetd_wait_mode) /* Start listening on the bound socket, establishing the maximum backlog of connections that is allowed. On success, continue to the next address. */ - if (listen(listen_sockets[sk], smtp_connect_backlog) >= 0) continue; + if (listen(listen_sockets[sk], smtp_connect_backlog) >= 0) + { +#if defined(TCP_FASTOPEN) && defined(__APPLE__) + if ( f.tcp_fastopen_ok + && setsockopt(listen_sockets[sk], IPPROTO_TCP, TCP_FASTOPEN, + &on, sizeof(on))) + { + DEBUG(D_any) debug_printf("setsockopt FASTOPEN: %s\n", strerror(errno)); + f.tcp_fastopen_ok = FALSE; + } +#endif + continue; + } /* Listening has failed. In an IPv6 environment, as for bind(), if listen() fails with the error EADDRINUSE and we are doing IPv4 wildcard listening diff --git a/src/src/expand.c b/src/src/expand.c index 5054e151b..49e09ecd8 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -5021,6 +5021,7 @@ while (*s != 0) port = ntohs(service_info->s_port); } + /*XXX we trust that the request is idempotent. Hmm. */ fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port, timeout, &host, &expand_string_message, do_tls ? NULL : &reqstr); diff --git a/src/src/ip.c b/src/src/ip.c index fa688be64..d601b46b3 100644 --- a/src/src/ip.c +++ b/src/src/ip.c @@ -196,7 +196,7 @@ Arguments: port the remote port timeout a timeout (zero for indefinite timeout) fastopen_blob non-null iff TCP_FASTOPEN can be used; may indicate early-data to - be sent in SYN segment + be sent in SYN segment. Any such data must be idempotent. Returns: 0 on success; -1 on failure, with errno set */ @@ -245,19 +245,19 @@ callout_address = string_sprintf("[%s]:%d", address, port); sigalrm_seen = FALSE; if (timeout > 0) ALARM(timeout); -#if defined(TCP_FASTOPEN) && defined(MSG_FASTOPEN) +#ifdef TCP_FASTOPEN /* TCP Fast Open, if the system has a cookie from a previous call to this peer, can send data in the SYN packet. The peer can send data before it gets our ACK of its SYN,ACK - the latter is useful for the SMTP banner. Other (than SMTP) cases of TCP connections can -possibly use the data-on-syn, so support that too. - -This is a Linux implementation. It might be useable on FreeBSD; I have -not checked. I think MacOS has a "connectx" call for this purpose, -rather than using "sendto" ? */ +possibly use the data-on-syn, so support that too. */ if (fastopen_blob && f.tcp_fastopen_ok) { +# ifdef MSG_FASTOPEN + /* This is a Linux implementation. It might be useable on FreeBSD; I have + not checked. */ + if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len, MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0) /* seen for with-data, experimental TFO option, with-cookie case */ @@ -292,9 +292,44 @@ if (fastopen_blob && f.tcp_fastopen_ok) debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n"); goto legacy_connect; } +# endif +# ifdef EXIM_TFO_CONNECTX + /* MacOS */ + sa_endpoints_t ends = { + .sae_srcif = 0, .sae_srcaddr = NULL, .sae_srcaddrlen = 0, + .sae_dstaddr = s_ptr, .sae_dstaddrlen = s_len }; + struct iovec iov = { + .iov_base = fastopen_blob->data, .iov_len = fastopen_blob->len }; + size_t len; + + if ((rc = connectx(sock, &ends, SAE_ASSOCID_ANY, + CONNECT_DATA_IDEMPOTENT, &iov, 1, &len, NULL)) == 0) + { + DEBUG(D_transport|D_v) + debug_printf("TFO mode connection attempt to %s, %lu data\n", + address, (unsigned long)fastopen_blob->len); + tcp_out_fastopen = fastopen_blob->len > 0 ? TFO_USED : TFO_ATTEMPTED; + + if (len != fastopen_blob->len) + DEBUG(D_transport|D_v) + debug_printf(" only queued %lu data!\n", (unsigned long)len); + } + else if (errno == EINPROGRESS) + { + DEBUG(D_transport|D_v) debug_printf("TFO mode sendto, %s data: EINPROGRESS\n", + fastopen_blob->len > 0 ? "with" : "no"); + if (!fastopen_blob->data) + { + tcp_out_fastopen = TFO_ATTEMPTED; /* we tried; unknown if useful yet */ + rc = 0; + } + else /* assume that no data was queued; block in send */ + rc = send(sock, fastopen_blob->data, fastopen_blob->len, 0); + } +# endif } else -#endif +#endif /*TCP_FASTOPEN*/ { legacy_connect: DEBUG(D_transport|D_v) if (fastopen_blob) @@ -350,7 +385,7 @@ Arguments: connhost if not NULL, host_item to be filled in with connection details errstr pointer for allocated string on error fastopen_blob with SOCK_STREAM, if non-null, request TCP Fast Open. - Additionally, optional early-data to send + Additionally, optional idempotent early-data to send Return: socket fd, or -1 on failure (having allocated an error string) diff --git a/src/src/malware.c b/src/src/malware.c index 069a54483..541ff3b3c 100644 --- a/src/src/malware.c +++ b/src/src/malware.c @@ -1573,6 +1573,7 @@ badseek: err = errno; * on both connections (as one host could resolve to multiple ips) */ for (;;) { + /*XXX we trust that the cmd_str is ideempotent */ if ((malware_daemon_ctx.sock = m_tcpsocket(cd->hostspec, cd->tcp_port, &connhost, &errstr, &cmd_str)) >= 0) { diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c index ef2c9fdeb..c5eafbc57 100644 --- a/src/src/smtp_out.c +++ b/src/src/smtp_out.c @@ -194,7 +194,8 @@ if (getsockopt(sock, IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0) /* Arguments as for smtp_connect(), plus - early_data if non-NULL, data to be sent - preferably in the TCP SYN segment + early_data if non-NULL, idenmpotent data to be sent - + preferably in the TCP SYN segment Returns: connected socket number, or -1 with errno set */ diff --git a/src/src/transports/smtp_socks.c b/src/src/transports/smtp_socks.c index c7415c357..7d3a46230 100644 --- a/src/src/transports/smtp_socks.c +++ b/src/src/transports/smtp_socks.c @@ -297,6 +297,7 @@ for(;;) proxy.address = proxy.name = sob->proxy_host; proxy_af = Ustrchr(sob->proxy_host, ':') ? AF_INET6 : AF_INET; + /*XXX we trust that the method-select command is idempotent */ if ((fd = smtp_sock_connect(&proxy, proxy_af, sob->port, interface, tb, sob->timeout, &early_data)) >= 0) { diff --git a/src/src/verify.c b/src/src/verify.c index 9aff78a9a..d14cb685e 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -2745,6 +2745,7 @@ qlen = snprintf(CS buffer, sizeof(buffer), "%d , %d\r\n", early_data.data = buffer; early_data.len = qlen; +/*XXX we trust that the query is idempotent */ if (ip_connect(ident_conn_ctx.sock, host_af, sender_host_address, port, rfc1413_query_timeout, &early_data) < 0) { diff --git a/test/scripts/1990-TCP-Fast-Open/1990 b/test/scripts/1990-TCP-Fast-Open/1990 index 04b41a7af..ec8e32c8f 100644 --- a/test/scripts/1990-TCP-Fast-Open/1990 +++ b/test/scripts/1990-TCP-Fast-Open/1990 @@ -13,6 +13,15 @@ # an unhelpful error from RTNETLINK. # To tidy up: 'sudo tc qdisc delete dev lo root' # +# MacOS: +# The kernel seems to have TFO enabled both ways as default. +# There is a net.inet.tcp.clear_tfocache parameter +## sysctl -w foo-val +# +# For network delays there is something called 'Network Link Conditioner' +# which might do the job. But how to manipulate it? +# +# sudo perl system ("tc qdisc add dev lo root netem delay 50ms"); **** @@ -22,17 +31,14 @@ system ("tc qdisc add dev lo root netem delay 50ms"); # (currently on a separate packet after the server SYN,ACK but before # the client ACK). # -# The client log => lint.ex should have a "TFO" element. -# Assuming this is the first run since boot, the a@test recipient will not. +# The client log => line should have a "TFO" element. +# The server log <= line for a@test.ex should not. # +# First clear any previously-obtained cookie: sudo perl system ("ip tcp_metrics delete 127.0.0.1"); **** # -# The server log <= line for b@test.ex should have a "TFO" element, but -# this will only be obtained when the above delay is inserted into the -# loopback net path. -# # # # FreeBSD: it looks like you have to compile a custom kernel, with @@ -48,6 +54,10 @@ Testing **** sleep 3 # +# The server log <= line for b@test.ex should have a "TFO" element, but +# this will only be obtained when the above delay is inserted into the +# loopback net path. +# exim b@test.ex Testing **** diff --git a/test/src/server.c b/test/src/server.c index fe1c79f02..ba731625b 100644 --- a/test/src/server.c +++ b/test/src/server.c @@ -298,7 +298,7 @@ else printf("IPv6 socket creation failed: %s\n", strerror(errno)); exit(1); } -#ifdef TCP_FASTOPEN +#if defined(TCP_FASTOPEN) && !defined(__APPLE__) if (tfo) { int backlog = 5; @@ -330,7 +330,7 @@ else printf("IPv4 socket creation failed: %s\n", strerror(errno)); exit(1); } -#ifdef TCP_FASTOPEN +#if defined(TCP_FASTOPEN) && !defined(__APPLE__) if (tfo) { int backlog = 5; @@ -438,16 +438,21 @@ error because it means that the IPv6 socket will handle IPv4 connections. Don't output anything, because it will mess up the test output, which will be different for systems that do this and those that don't. */ -for (i = 0; i <= skn; i++) +for (i = 0; i <= skn; i++) if (listen_socket[i] >= 0) { - if (listen_socket[i] >= 0 && listen(listen_socket[i], 5) < 0) - { + if (listen(listen_socket[i], 5) < 0) if (i != v4n || listen_socket[v6n] < 0 || errno != EADDRINUSE) { printf("listen() failed: %s\n", strerror(errno)); exit(1); } - } + +#if defined(TCP_FASTOPEN) && defined(__APPLE__) + if ( tfo + && setsockopt(listen_socket[v4n], IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on)) + && debug) + printf("setsockopt TCP_FASTOPEN: %s\n", strerror(errno)); +#endif } -- 2.25.1