From 73a10da9bbc6aadd03c3aff7a12307252e617a71 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Tue, 3 Dec 2019 22:12:09 +0000 Subject: [PATCH] FreeBSD: better support for TFO --- src/OS/os.c-FreeBSD | 23 +++++++- src/OS/os.h-FreeBSD | 8 +-- src/src/ip.c | 57 ++++++++++--------- src/src/smtp_in.c | 25 ++++++-- src/src/smtp_out.c | 26 ++++++++- src/src/transport.c | 85 +++++++++++++++------------- test/scripts/1990-TCP-Fast-Open/1990 | 10 ++-- 7 files changed, 148 insertions(+), 86 deletions(-) diff --git a/src/OS/os.c-FreeBSD b/src/OS/os.c-FreeBSD index 4cc46c752..c0fd48df8 100644 --- a/src/OS/os.c-FreeBSD +++ b/src/OS/os.c-FreeBSD @@ -10,7 +10,7 @@ src/os.c file. */ /************* -* Sendfile * +Sendfile shim *************/ ssize_t @@ -23,4 +23,25 @@ if (sendfile(in, out, loff, cnt, NULL, &written, 0) < 0) return (ssize_t)-1; return (ssize_t)written; } +/************************************************* +TCP Fast Open: check that the ioctl is accepted +*************************************************/ + +#ifndef COMPILE_UTILITY +void +tfo_probe(void) +{ +# ifdef TCP_FASTOPEN +int sock; + +if ( (sock = socket(AF_INET, SOCK_STREAM, 0)) >= 0 + && setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on) >= 0) + ) + f.tcp_fastopen_ok = TRUE; +close(sock); +# endif +} +#endif + + /* End of os.c-Linux */ diff --git a/src/OS/os.h-FreeBSD b/src/OS/os.h-FreeBSD index 4f1c616f0..a15d47508 100644 --- a/src/OS/os.h-FreeBSD +++ b/src/OS/os.h-FreeBSD @@ -57,15 +57,9 @@ extern ssize_t os_sendfile(int, int, off_t *, size_t); /*******************/ -/* TCP_FASTOPEN support. There does not seems to be a -MSG_FASTOPEN defined yet... */ #define EXIM_TFO_PROBE +#define EXIM_TFO_FREEBSD -#include /* for TCP_FASTOPEN */ -#include /* for MSG_FASTOPEN */ -#if defined(TCP_FASTOPEN) && !defined(MSG_FASTOPEN) -# define MSG_FASTOPEN 0x20000000 -#endif /* for TCP state-variable values, for TFO logging */ #include diff --git a/src/src/ip.c b/src/src/ip.c index 70e3e2064..108c21d92 100644 --- a/src/src/ip.c +++ b/src/src/ip.c @@ -14,6 +14,12 @@ different places in the code where sockets are used. */ #include "exim.h" +#if defined(TCP_FASTOPEN) +# if defined(MSG_FASTOPEN) || defined(EXIM_TFO_CONNECTX) || defined(EXIM_TFO_FREEBSD) +# define EXIM_SUPPORT_TFO +# endif +#endif + /************************************************* * Create a socket * *************************************************/ @@ -160,26 +166,6 @@ return bind(sock, (struct sockaddr *)&sin, s_len); -/************************************************* -*************************************************/ - -#ifdef EXIM_TFO_PROBE -void -tfo_probe(void) -{ -# ifdef TCP_FASTOPEN -int sock, backlog = 5; - -if ( (sock = socket(SOCK_STREAM, AF_INET, 0)) < 0 - && setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &backlog, sizeof(backlog)) - ) - f.tcp_fastopen_ok = TRUE; -close(sock); -# endif -} -#endif - - /************************************************* * Connect socket to remote host * *************************************************/ @@ -245,7 +231,7 @@ callout_address = string_sprintf("[%s]:%d", address, port); sigalrm_seen = FALSE; if (timeout > 0) ALARM(timeout); -#if defined(TCP_FASTOPEN) && (defined(MSG_FASTOPEN) || defined(EXIM_TFO_CONNECTX)) +#ifdef EXIM_SUPPORT_TFO /* TCP Fast Open, if the system has a cookie from a previous call to this peer, can send data in the SYN packet. The peer can send data before it gets our ACK of its SYN,ACK - the latter is useful for @@ -255,8 +241,7 @@ possibly use the data-on-syn, so support that too. */ if (fastopen_blob && f.tcp_fastopen_ok) { # ifdef MSG_FASTOPEN - /* This is a Linux implementation. FreeBSD does not seem to have MSG_FASTOPEN so - how to get TFO is unknown. */ + /* This is a Linux implementation. */ if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len, MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0) @@ -292,8 +277,26 @@ if (fastopen_blob && f.tcp_fastopen_ok) debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n"); goto legacy_connect; } -# endif -# ifdef EXIM_TFO_CONNECTX + +# elif defined(EXIM_TFO_FREEBSD) + /* Re: https://people.freebsd.org/~pkelsey/tfo-tools/tfo-client.c */ + + if (setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on)) < 0) + { + DEBUG(D_transport) + debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n"); + goto legacy_connect; + } + if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len, 0, + s_ptr, s_len)) >= 0) + { + DEBUG(D_transport|D_v) + debug_printf(" TFO mode connection attempt to %s, %lu data\n", + address, (unsigned long)fastopen_blob->len); + tcp_out_fastopen = fastopen_blob->len > 0 ? TFO_ATTEMPTED_DATA : TFO_ATTEMPTED_NODATA; + } + +# elif defined(EXIM_TFO_CONNECTX) /* MacOS */ sa_endpoints_t ends = { .sae_srcif = 0, .sae_srcaddr = NULL, .sae_srcaddrlen = 0, @@ -329,9 +332,9 @@ if (fastopen_blob && f.tcp_fastopen_ok) # endif } else -#endif /*TCP_FASTOPEN*/ +#endif /*EXIM_SUPPORT_TFO*/ { -#if defined(TCP_FASTOPEN) && defined(MSG_FASTOPEN) +#if defined(EXIM_SUPPORT_TFO) && !defined(EXIM_TFO_CONNECTX) legacy_connect: #endif diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index b88fde1b5..c2e234ab5 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -2405,20 +2405,35 @@ struct tcp_info tinfo; socklen_t len = sizeof(tinfo); if (getsockopt(fileno(smtp_out), IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0) -#ifdef TCPI_OPT_SYN_DATA /* FreeBSD 11 does not seem to have this yet */ +# ifdef TCPI_OPT_SYN_DATA /* FreeBSD 11,12 do not seem to have this yet */ if (tinfo.tcpi_options & TCPI_OPT_SYN_DATA) { - DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (ACKd data-on-SYN)\n"); + DEBUG(D_receive) + debug_printf("TCP_FASTOPEN mode connection (ACKd data-on-SYN)\n"); f.tcp_in_fastopen_data = f.tcp_in_fastopen = TRUE; } else -#endif - if (tinfo.tcpi_state == TCP_SYN_RECV) +# endif + if (tinfo.tcpi_state == TCP_SYN_RECV) /* Not seen on FreeBSD 12.1 */ + { + DEBUG(D_receive) + debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n"); + f.tcp_in_fastopen = TRUE; + } +# ifdef __FreeBSD__ + else if (tinfo.tcpi_options & TCPOPT_FAST_OPEN) { - DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n"); + /* This only tells us that some combination of the TCP options was used. It + can be a TFO-R received (as of 12.1). However, pretend it shows real usage + (that an acceptable TFO-C was received and acted on). Ignore the possibility + of data-on-SYN for now. */ + DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (TFO option used)\n"); f.tcp_in_fastopen = TRUE; } +# endif # endif +else DEBUG(D_receive) + debug_printf("TCP_INFO getsockopt: %s\n", strerror(errno)); } #endif diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c index 07cc9b762..3dc3a13fb 100644 --- a/src/src/smtp_out.c +++ b/src/src/smtp_out.c @@ -155,9 +155,28 @@ return TRUE; static void tfo_out_check(int sock) { -# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED) struct tcp_info tinfo; -socklen_t len = sizeof(tinfo); +int val; +socklen_t len = sizeof(val); + +# ifdef __FreeBSD__ +/* The observability as of 12.1 is not useful as a client, only telling us that +a TFO option was used on SYN. It could have been a TFO-R, or ignored by the +server. */ + +/* +if (tcp_out_fastopen == TFO_ATTEMPTED_NODATA || tcp_out_fastopen == TFO_ATTEMPTED_DATA) + if (getsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &val, &len) == 0 && val != 0) {} +*/ +switch (tcp_out_fastopen) + { + case TFO_ATTEMPTED_NODATA: tcp_out_fastopen = TFO_USED_NODATA; break; + case TFO_ATTEMPTED_DATA: tcp_out_fastopen = TFO_USED_DATA; break; + default: break; /* compiler quietening */ + } + +# else /* Linux & Apple */ +# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED) switch (tcp_out_fastopen) { @@ -205,7 +224,8 @@ switch (tcp_out_fastopen) default: break; /* compiler quietening */ } -# endif +# endif +# endif /* Linux & Apple */ } #endif diff --git a/src/src/transport.c b/src/src/transport.c index df7fd1628..14bd91cdb 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -172,6 +172,20 @@ for (transport_instance * t = transports; t; t = t->next) * Write block of data * *************************************************/ +static int +tpt_write(int fd, uschar * block, int len, BOOL more, int options) +{ +return +#ifndef DISABLE_TLS + tls_out.active.sock == fd + ? tls_write(tls_out.active.tls_ctx, block, len, more) : +#endif +#ifdef MSG_MORE + more && !(options & topt_not_socket) ? send(fd, block, len, MSG_MORE) : +#endif + write(fd, block, len); +} + /* Subroutine called by write_chunk() and at the end of the message actually to write a data block. Also called directly by some transports to write additional data to the file descriptor (e.g. prefix, suffix). @@ -215,10 +229,11 @@ Returns: TRUE on success, FALSE on failure (with errno preserved); */ static BOOL -transport_write_block_fd(transport_ctx * tctx, uschar *block, int len, BOOL more) +transport_write_block_fd(transport_ctx * tctx, uschar * block, int len, BOOL more) { int rc, save_errno; int local_timeout = transport_write_timeout; +int connretry = 1; int fd = tctx->u.fd; /* This loop is for handling incomplete writes and other retries. In most @@ -230,48 +245,42 @@ for (int i = 0; i < 100; i++) debug_printf("writing data block fd=%d size=%d timeout=%d%s\n", fd, len, local_timeout, more ? " (more expected)" : ""); - /* This code makes use of alarm() in order to implement the timeout. This - isn't a very tidy way of doing things. Using non-blocking I/O with select() - provides a neater approach. However, I don't know how to do this when TLS is - in use. */ - - if (transport_write_timeout <= 0) /* No timeout wanted */ - { - rc = -#ifndef DISABLE_TLS - tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) : -#endif -#ifdef MSG_MORE - more && !(tctx->options & topt_not_socket) - ? send(fd, block, len, MSG_MORE) : -#endif - write(fd, block, len); - save_errno = errno; - } + /* When doing TCP Fast Open we may get this far before the 3-way handshake + is complete, and write returns ENOTCONN. Detect that, wait for the socket + to become writable, and retry once only. */ - /* Timeout wanted. */ - - else + for(;;) { - ALARM(local_timeout); + fd_set fds; + /* This code makes use of alarm() in order to implement the timeout. This + isn't a very tidy way of doing things. Using non-blocking I/O with select() + provides a neater approach. However, I don't know how to do this when TLS is + in use. */ - rc = -#ifndef DISABLE_TLS - tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) : -#endif -#ifdef MSG_MORE - more && !(tctx->options & topt_not_socket) - ? send(fd, block, len, MSG_MORE) : -#endif - write(fd, block, len); - - save_errno = errno; - local_timeout = ALARM_CLR(0); - if (sigalrm_seen) + if (transport_write_timeout <= 0) /* No timeout wanted */ { - errno = ETIMEDOUT; - return FALSE; + rc = tpt_write(fd, block, len, more, tctx->options); + save_errno = errno; } + else /* Timeout wanted. */ + { + ALARM(local_timeout); + rc = tpt_write(fd, block, len, more, tctx->options); + save_errno = errno; + local_timeout = ALARM_CLR(0); + if (sigalrm_seen) + { + errno = ETIMEDOUT; + return FALSE; + } + } + + if (rc >= 0 || errno != ENOTCONN || connretry <= 0) + break; + + FD_ZERO(&fds); FD_SET(fd, &fds); + select(fd+1, NULL, &fds, NULL, NULL); /* could set timout? */ + connretry--; } /* Hopefully, the most common case is success, so test that first. */ diff --git a/test/scripts/1990-TCP-Fast-Open/1990 b/test/scripts/1990-TCP-Fast-Open/1990 index 1fc4682aa..80059e685 100644 --- a/test/scripts/1990-TCP-Fast-Open/1990 +++ b/test/scripts/1990-TCP-Fast-Open/1990 @@ -22,6 +22,11 @@ # which might do the job. But how to manipulate it? # # +# FreeBSD: it looks like you have to compile a custom kernel, with +# 'options TCP_RFC7413' in the config. Also set +# 'net.inet.tcp.fastopen.server_enable=1' in /etc/sysctl.conf +# Seems to always claim TFO used by transport, if tried. +# sudo perl system ("tc qdisc add dev lo root netem delay 50ms"); **** @@ -50,11 +55,6 @@ system ("ip tcp_metrics delete 127.0.0.1"); # # # -# FreeBSD: it looks like you have to compile a custom kernel, with -# 'options TCP_RFC7413' in the config. Also set -# 'net.inet.tcp.fastopen.enabled=1' in /etc/sysctl.conf -# Untested. -# exim -DSERVER=server -bd -oX PORT_D **** # -- 2.25.1