X-Git-Url: https://vcs.fsf.org/?p=exim.git;a=blobdiff_plain;f=src%2Fsrc%2Ftransport.c;h=86350ba9d8ccee39061081d9ea34d449c95830b5;hp=8b68417835a9951aa64084cec360421f7b7a2536;hb=5fb822fc4839e253d2f839ba7966bc25a5329ac1;hpb=35af9f61534cba784c1718d804567043da64f2df diff --git a/src/src/transport.c b/src/src/transport.c index 8b6841783..86350ba9d 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1,10 +1,8 @@ -/* $Cambridge: exim/src/src/transport.c,v 1.2 2004/10/14 14:52:45 ph10 Exp $ */ - /************************************************* * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2004 */ +/* Copyright (c) University of Cambridge 1995 - 2016 */ /* See the file NOTICE for conditions of use and distribution. */ /* General functions concerned with transportation, and generic options for all @@ -13,6 +11,9 @@ transports. */ #include "exim.h" +#ifdef HAVE_LINUX_SENDFILE +#include +#endif /* Structure for keeping list of addresses that have been added to Envelope-To:, in order to avoid duplication. */ @@ -65,13 +66,17 @@ optionlist optionlist_transports[] = { (void *)offsetof(transport_instance, driver_name) }, { "envelope_to_add", opt_bool|opt_public, (void *)(offsetof(transport_instance, envelope_to_add)) }, +#ifndef DISABLE_EVENT + { "event_action", opt_stringptr | opt_public, + (void *)offsetof(transport_instance, event_action) }, +#endif { "group", opt_expand_gid|opt_public, (void *)offsetof(transport_instance, gid) }, - { "headers_add", opt_stringptr|opt_public, + { "headers_add", opt_stringptr|opt_public|opt_rep_str, (void *)offsetof(transport_instance, add_headers) }, { "headers_only", opt_bool|opt_public, (void *)offsetof(transport_instance, headers_only) }, - { "headers_remove", opt_stringptr|opt_public, + { "headers_remove", opt_stringptr|opt_public|opt_rep_str, (void *)offsetof(transport_instance, remove_headers) }, { "headers_rewrite", opt_rewrite|opt_public, (void *)offsetof(transport_instance, headers_rewrite) }, @@ -79,6 +84,8 @@ optionlist optionlist_transports[] = { (void *)offsetof(transport_instance, home_dir) }, { "initgroups", opt_bool|opt_public, (void *)offsetof(transport_instance, initgroups) }, + { "max_parallel", opt_stringptr|opt_public, + (void *)offsetof(transport_instance, max_parallel) }, { "message_size_limit", opt_stringptr|opt_public, (void *)offsetof(transport_instance, message_size_limit) }, { "rcpt_include_affixes", opt_bool|opt_public, @@ -199,26 +206,42 @@ BOOL transport_write_block(int fd, uschar *block, int len) { int i, rc, save_errno; +int local_timeout = transport_write_timeout; + +/* This loop is for handling incomplete writes and other retries. In most +normal cases, it is only ever executed once. */ for (i = 0; i < 100; i++) { DEBUG(D_transport) debug_printf("writing data block fd=%d size=%d timeout=%d\n", - fd, len, transport_write_timeout); - if (transport_write_timeout > 0) alarm(transport_write_timeout); + fd, len, local_timeout); - #ifdef SUPPORT_TLS - if (tls_active == fd) rc = tls_write(block, len); else - #endif + /* This code makes use of alarm() in order to implement the timeout. This + isn't a very tidy way of doing things. Using non-blocking I/O with select() + provides a neater approach. However, I don't know how to do this when TLS is + in use. */ - rc = write(fd, block, len); - save_errno = errno; + if (transport_write_timeout <= 0) /* No timeout wanted */ + { + #ifdef SUPPORT_TLS + if (tls_out.active == fd) rc = tls_write(FALSE, block, len); else + #endif + rc = write(fd, block, len); + save_errno = errno; + } - /* Cancel the alarm and deal with a timeout */ + /* Timeout wanted. */ - if (transport_write_timeout > 0) + else { - alarm(0); + alarm(local_timeout); + #ifdef SUPPORT_TLS + if (tls_out.active == fd) rc = tls_write(FALSE, block, len); else + #endif + rc = write(fd, block, len); + save_errno = errno; + local_timeout = alarm(0); if (sigalrm_seen) { errno = ETIMEDOUT; @@ -230,7 +253,8 @@ for (i = 0; i < 100; i++) if (rc == len) { transport_count += len; return TRUE; } - /* A non-negative return code is an incomplete write. Try again. */ + /* A non-negative return code is an incomplete write. Try again for the rest + of the block. If we have exactly hit the timeout, give up. */ if (rc >= 0) { @@ -238,7 +262,7 @@ for (i = 0; i < 100; i++) block += rc; transport_count += rc; DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc); - continue; + goto CHECK_TIMEOUT; /* A few lines below */ } /* A negative return code with an EINTR error is another form of @@ -248,7 +272,7 @@ for (i = 0; i < 100; i++) { DEBUG(D_transport) debug_printf("write interrupted before anything written\n"); - continue; + goto CHECK_TIMEOUT; /* A few lines below */ } /* A response of EAGAIN from write() is likely only in the case of writing @@ -259,6 +283,16 @@ for (i = 0; i < 100; i++) DEBUG(D_transport) debug_printf("write temporarily locked out, waiting 1 sec\n"); sleep(1); + + /* Before continuing to try another write, check that we haven't run out of + time. */ + + CHECK_TIMEOUT: + if (transport_write_timeout > 0 && local_timeout <= 0) + { + errno = ETIMEDOUT; + return FALSE; + } continue; } @@ -294,7 +328,7 @@ Returns: the yield of transport_write_block() */ BOOL -transport_write_string(int fd, char *format, ...) +transport_write_string(int fd, const char *format, ...) { va_list ap; va_start(ap, format); @@ -396,6 +430,7 @@ for (ptr = start; ptr < end; ptr++) if (use_crlf) *chunk_ptr++ = '\r'; *chunk_ptr++ = '\n'; + transport_newlines++; /* The check_string test (formerly "from hack") replaces the specific string at the start of a line with an escape string (e.g. "From " becomes @@ -567,6 +602,176 @@ return write_chunk(fd, pp->address, Ustrlen(pp->address), use_crlf); +/* Add/remove/rewwrite headers, and send them plus the empty-line sparator. + +Globals: + header_list + +Arguments: + addr (chain of) addresses (for extra headers), or NULL; + only the first address is used + fd file descriptor to write the message to + sendfn function for output + use_crlf turn NL into CR LF + rewrite_rules chain of header rewriting rules + rewrite_existflags flags for the rewriting rules + +Returns: TRUE on success; FALSE on failure. +*/ +BOOL +transport_headers_send(address_item *addr, int fd, uschar *add_headers, uschar *remove_headers, + BOOL (*sendfn)(int fd, uschar * s, int len, BOOL use_crlf), + BOOL use_crlf, rewrite_rule *rewrite_rules, int rewrite_existflags) +{ +header_line *h; + +/* Then the message's headers. Don't write any that are flagged as "old"; +that means they were rewritten, or are a record of envelope rewriting, or +were removed (e.g. Bcc). If remove_headers is not null, skip any headers that +match any entries therein. It is a colon-sep list; expand the items +separately and squash any empty ones. +Then check addr->prop.remove_headers too, provided that addr is not NULL. */ + +for (h = header_list; h != NULL; h = h->next) if (h->type != htype_old) + { + int i; + const uschar *list = remove_headers; + + BOOL include_header = TRUE; + + for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */ + { + if (list) + { + int sep = ':'; /* This is specified as a colon-separated list */ + uschar *s, *ss; + while ((s = string_nextinlist(&list, &sep, NULL, 0))) + { + int len; + + if (i == 0) + if (!(s = expand_string(s)) && !expand_string_forcedfail) + { + errno = ERRNO_CHHEADER_FAIL; + return FALSE; + } + len = s ? Ustrlen(s) : 0; + if (strncmpic(h->text, s, len) != 0) continue; + ss = h->text + len; + while (*ss == ' ' || *ss == '\t') ss++; + if (*ss == ':') break; + } + if (s != NULL) { include_header = FALSE; break; } + } + if (addr != NULL) list = addr->prop.remove_headers; + } + + /* If this header is to be output, try to rewrite it if there are rewriting + rules. */ + + if (include_header) + { + if (rewrite_rules) + { + void *reset_point = store_get(0); + header_line *hh; + + if ((hh = rewrite_header(h, NULL, NULL, rewrite_rules, rewrite_existflags, FALSE))) + { + if (!sendfn(fd, hh->text, hh->slen, use_crlf)) return FALSE; + store_reset(reset_point); + continue; /* With the next header line */ + } + } + + /* Either no rewriting rules, or it didn't get rewritten */ + + if (!sendfn(fd, h->text, h->slen, use_crlf)) return FALSE; + } + + /* Header removed */ + + else + { + DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text); + } + } + +/* Add on any address-specific headers. If there are multiple addresses, +they will all have the same headers in order to be batched. The headers +are chained in reverse order of adding (so several addresses from the +same alias might share some of them) but we want to output them in the +opposite order. This is a bit tedious, but there shouldn't be very many +of them. We just walk the list twice, reversing the pointers each time, +but on the second time, write out the items. + +Headers added to an address by a router are guaranteed to end with a newline. +*/ + +if (addr) + { + int i; + header_line *hprev = addr->prop.extra_headers; + header_line *hnext; + for (i = 0; i < 2; i++) + { + for (h = hprev, hprev = NULL; h != NULL; h = hnext) + { + hnext = h->next; + h->next = hprev; + hprev = h; + if (i == 1) + { + if (!sendfn(fd, h->text, h->slen, use_crlf)) return FALSE; + DEBUG(D_transport) + debug_printf("added header line(s):\n%s---\n", h->text); + } + } + } + } + +/* If a string containing additional headers exists it is a newline-sep +list. Expand each item and write out the result. This is done last so that +if it (deliberately or accidentally) isn't in header format, it won't mess +up any other headers. An empty string or a forced expansion failure are +noops. An added header string from a transport may not end with a newline; +add one if it does not. */ + +if (add_headers) + { + int sep = '\n'; + uschar * s; + + while ((s = string_nextinlist(CUSS &add_headers, &sep, NULL, 0))) + if (!(s = expand_string(s))) + { + if (!expand_string_forcedfail) + { errno = ERRNO_CHHEADER_FAIL; return FALSE; } + } + else + { + int len = Ustrlen(s); + if (len > 0) + { + if (!sendfn(fd, s, len, use_crlf)) return FALSE; + if (s[len-1] != '\n' && !sendfn(fd, US"\n", 1, use_crlf)) + return FALSE; + DEBUG(D_transport) + { + debug_printf("added header line:\n%s", s); + if (s[len-1] != '\n') debug_printf("\n"); + debug_printf("---\n"); + } + } + } + } + +/* Separate headers from body with a blank line */ + +return sendfn(fd, US"\n", 1, use_crlf); +} + + /************************************************* * Write the message * *************************************************/ @@ -633,7 +838,6 @@ internal_transport_write_message(address_item *addr, int fd, int options, { int written = 0; int len; -header_line *h; BOOL use_crlf = (options & topt_use_crlf) != 0; /* Initialize pointer in output buffer. */ @@ -712,196 +916,268 @@ if ((options & topt_no_headers) == 0) /* Then the message's headers. Don't write any that are flagged as "old"; that means they were rewritten, or are a record of envelope rewriting, or were removed (e.g. Bcc). If remove_headers is not null, skip any headers that - match any entries therein. Then check addr->p.remove_headers too, provided that + match any entries therein. Then check addr->prop.remove_headers too, provided that addr is not NULL. */ + if (!transport_headers_send(addr, fd, add_headers, remove_headers, &write_chunk, + use_crlf, rewrite_rules, rewrite_existflags)) + return FALSE; + } + +/* If the body is required, ensure that the data for check strings (formerly +the "from hack") is enabled by negating the length if necessary. (It will be +negative in cases where it isn't to apply to the headers). Then ensure the body +is positioned at the start of its file (following the message id), then write +it, applying the size limit if required. */ - if (remove_headers != NULL) +if ((options & topt_no_body) == 0) + { + nl_check_length = abs(nl_check_length); + nl_partial_match = 0; + if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0) + return FALSE; + while ((len = read(deliver_datafile, deliver_in_buffer, + DELIVER_IN_BUFFER_SIZE)) > 0) { - uschar *s = expand_string(remove_headers); - if (s == NULL && !expand_string_forcedfail) + if (!write_chunk(fd, deliver_in_buffer, len, use_crlf)) return FALSE; + if (size_limit > 0) { - errno = ERRNO_CHHEADER_FAIL; - return FALSE; + written += len; + if (written > size_limit) + { + len = 0; /* Pretend EOF */ + break; + } } - remove_headers = s; } - for (h = header_list; h != NULL; h = h->next) - { - int i; - uschar *list = NULL; - BOOL include_header; + /* A read error on the body will have left len == -1 and errno set. */ - if (h->type == htype_old) continue; + if (len != 0) return FALSE; + } - include_header = TRUE; - list = remove_headers; +/* Finished with the check string */ - for (i = 0; i < 2; i++) /* For remove_headers && addr->p.remove_headers */ - { - if (list != NULL) - { - int sep = ':'; /* This is specified as a colon-separated list */ - uschar *s, *ss; - uschar buffer[128]; - while ((s = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) - != NULL) - { - int len = Ustrlen(s); - if (strncmpic(h->text, s, len) != 0) continue; - ss = h->text + len; - while (*ss == ' ' || *ss == '\t') ss++; - if (*ss == ':') break; - } - if (s != NULL) { include_header = FALSE; break; } - } - if (addr != NULL) list = addr->p.remove_headers; - } +nl_check_length = nl_escape_length = 0; - /* If this header is to be output, try to rewrite it if there are rewriting - rules. */ +/* If requested, add a terminating "." line (SMTP output). */ - if (include_header) - { - if (rewrite_rules != NULL) - { - void *reset_point = store_get(0); - header_line *hh = - rewrite_header(h, NULL, NULL, rewrite_rules, rewrite_existflags, - FALSE); - if (hh != NULL) - { - if (!write_chunk(fd, hh->text, hh->slen, use_crlf)) return FALSE; - store_reset(reset_point); - continue; /* With the next header line */ - } - } +if ((options & topt_end_dot) != 0 && !write_chunk(fd, US".\n", 2, use_crlf)) + return FALSE; - /* Either no rewriting rules, or it didn't get rewritten */ +/* Write out any remaining data in the buffer before returning. */ - if (!write_chunk(fd, h->text, h->slen, use_crlf)) return FALSE; - } +return (len = chunk_ptr - deliver_out_buffer) <= 0 || + transport_write_block(fd, deliver_out_buffer, len); +} - /* Header removed */ - else - { - DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", - h->text); - } - } +#ifndef DISABLE_DKIM - /* Add on any address-specific headers. If there are multiple addresses, - they will all have the same headers in order to be batched. The headers - are chained in reverse order of adding (so several addresses from the - same alias might share some of them) but we want to output them in the - opposite order. This is a bit tedious, but there shouldn't be very many - of them. We just walk the list twice, reversing the pointers each time, - but on the second time, write out the items. */ +/*************************************************************************************************** +* External interface to write the message, while signing it with DKIM and/or Domainkeys * +***************************************************************************************************/ - if (addr != NULL) - { - int i; - header_line *hprev = addr->p.extra_headers; - header_line *hnext; - for (i = 0; i < 2; i++) - { - for (h = hprev, hprev = NULL; h != NULL; h = hnext) - { - hnext = h->next; - h->next = hprev; - hprev = h; - if (i == 1) - { - if (!write_chunk(fd, h->text, h->slen, use_crlf)) return FALSE; - DEBUG(D_transport) - debug_printf("added header line(s):\n%s---\n", h->text); - } - } - } - } +/* This function is a wrapper around transport_write_message(). + It is only called from the smtp transport if DKIM or Domainkeys support + is compiled in. The function sets up a replacement fd into a -K file, + then calls the normal function. This way, the exact bits that exim would + have put "on the wire" will end up in the file (except for TLS + encapsulation, which is the very very last thing). When we are done + signing the file, send the signed message down the original fd (or TLS fd). + +Arguments: + as for internal_transport_write_message() above, with additional arguments: + uschar *dkim_private_key DKIM: The private key to use (filename or + plain data) + uschar *dkim_domain DKIM: The domain to use + uschar *dkim_selector DKIM: The selector to use. + uschar *dkim_canon DKIM: The canonalization scheme to use, + "simple" or "relaxed" + uschar *dkim_strict DKIM: What to do if signing fails: + 1/true => throw error + 0/false => send anyway + uschar *dkim_sign_headers DKIM: List of headers that should be included + in signature generation - /* If a string containing additional headers exists, expand it and write - out the result. This is done last so that if it (deliberately or accidentally) - isn't in header format, it won't mess up any other headers. An empty string - or a forced expansion failure are noops. */ +Returns: TRUE on success; FALSE (with errno) for any failure +*/ - if (add_headers != NULL) +BOOL +dkim_transport_write_message(address_item *addr, int fd, int options, + int size_limit, uschar *add_headers, uschar *remove_headers, + uschar *check_string, uschar *escape_string, rewrite_rule *rewrite_rules, + int rewrite_existflags, uschar *dkim_private_key, uschar *dkim_domain, + uschar *dkim_selector, uschar *dkim_canon, uschar *dkim_strict, uschar *dkim_sign_headers + ) +{ +int dkim_fd; +int save_errno = 0; +BOOL rc; +uschar dkim_spool_name[256]; +char sbuf[2048]; +int sread = 0; +int wwritten = 0; +uschar *dkim_signature = NULL; + +/* If we can't sign, just call the original function. */ + +if (!(dkim_private_key && dkim_domain && dkim_selector)) + return transport_write_message(addr, fd, options, + size_limit, add_headers, remove_headers, + check_string, escape_string, rewrite_rules, + rewrite_existflags); + +(void)string_format(dkim_spool_name, 256, "%s/input/%s/%s-%d-K", + spool_directory, message_subdir, message_id, (int)getpid()); + +if ((dkim_fd = Uopen(dkim_spool_name, O_RDWR|O_CREAT|O_TRUNC, SPOOL_MODE)) < 0) + { + /* Can't create spool file. Ugh. */ + rc = FALSE; + save_errno = errno; + goto CLEANUP; + } + +/* Call original function to write the -K file */ + +rc = transport_write_message(addr, dkim_fd, options, + size_limit, add_headers, remove_headers, + check_string, escape_string, rewrite_rules, + rewrite_existflags); + +/* Save error state. We must clean up before returning. */ +if (!rc) + { + save_errno = errno; + goto CLEANUP; + } + +if (dkim_private_key && dkim_domain && dkim_selector) + { + /* Rewind file and feed it to the goats^W DKIM lib */ + lseek(dkim_fd, 0, SEEK_SET); + dkim_signature = dkim_exim_sign(dkim_fd, + dkim_private_key, + dkim_domain, + dkim_selector, + dkim_canon, + dkim_sign_headers); + if (!dkim_signature) { - uschar *s = expand_string(add_headers); - if (s == NULL) + if (dkim_strict) { - if (!expand_string_forcedfail) - { - errno = ERRNO_CHHEADER_FAIL; - return FALSE; - } + uschar *dkim_strict_result = expand_string(dkim_strict); + if (dkim_strict_result) + if ( (strcmpic(dkim_strict,US"1") == 0) || + (strcmpic(dkim_strict,US"true") == 0) ) + { + /* Set errno to something halfway meaningful */ + save_errno = EACCES; + log_write(0, LOG_MAIN, "DKIM: message could not be signed," + " and dkim_strict is set. Deferring message delivery."); + rc = FALSE; + goto CLEANUP; + } } - else + } + else + { + int siglen = Ustrlen(dkim_signature); + while(siglen > 0) { - int len = Ustrlen(s); - if (len > 0) +#ifdef SUPPORT_TLS + wwritten = tls_out.active == fd + ? tls_write(FALSE, dkim_signature, siglen) + : write(fd, dkim_signature, siglen); +#else + wwritten = write(fd, dkim_signature, siglen); +#endif + if (wwritten == -1) { - if (!write_chunk(fd, s, len, use_crlf)) return FALSE; - if (s[len-1] != '\n' && !write_chunk(fd, US"\n", 1, use_crlf)) - return FALSE; - DEBUG(D_transport) - debug_printf("added header line(s):\n%s---\n", s); - } + /* error, bail out */ + save_errno = errno; + rc = FALSE; + goto CLEANUP; + } + siglen -= wwritten; + dkim_signature += wwritten; } } + } + +#ifdef HAVE_LINUX_SENDFILE +/* We can use sendfile() to shove the file contents + to the socket. However only if we don't use TLS, + as then there's another layer of indirection + before the data finally hits the socket. */ +if (tls_out.active != fd) + { + off_t size = lseek(dkim_fd, 0, SEEK_END); /* Fetch file size */ + ssize_t copied = 0; + off_t offset = 0; - /* Separate headers from body with a blank line */ + /* Rewind file */ + lseek(dkim_fd, 0, SEEK_SET); - if (!write_chunk(fd, US"\n", 1, use_crlf)) return FALSE; + while(copied >= 0 && offset < size) + copied = sendfile(fd, dkim_fd, &offset, size - offset); + if (copied < 0) + { + save_errno = errno; + rc = FALSE; + } } +else -/* If the body is required, ensure that the data for check strings (formerly -the "from hack") is enabled by negating the length if necessary. (It will be -negative in cases where it isn't to apply to the headers). Then ensure the body -is positioned at the start of its file (following the message id), then write -it, applying the size limit if required. */ +#endif -if ((options & topt_no_body) == 0) { - nl_check_length = abs(nl_check_length); - nl_partial_match = 0; - lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET); - while ((len = read(deliver_datafile, deliver_in_buffer, - DELIVER_IN_BUFFER_SIZE)) > 0) + /* Rewind file */ + lseek(dkim_fd, 0, SEEK_SET); + + /* Send file down the original fd */ + while((sread = read(dkim_fd, sbuf, 2048)) > 0) { - if (!write_chunk(fd, deliver_in_buffer, len, use_crlf)) return FALSE; - if (size_limit > 0) + char *p = sbuf; + /* write the chunk */ + + while (sread) { - written += len; - if (written > size_limit) - { - len = 0; /* Pretend EOF */ - break; - } +#ifdef SUPPORT_TLS + wwritten = tls_out.active == fd + ? tls_write(FALSE, US p, sread) + : write(fd, p, sread); +#else + wwritten = write(fd, p, sread); +#endif + if (wwritten == -1) + { + /* error, bail out */ + save_errno = errno; + rc = FALSE; + goto CLEANUP; + } + p += wwritten; + sread -= wwritten; } } - /* Finished with the check string */ - - nl_check_length = nl_escape_length = 0; - - /* A read error on the body will have left len == -1 and errno set. */ - - if (len != 0) return FALSE; - - /* If requested, add a terminating "." line (SMTP output). */ - - if ((options & topt_end_dot) != 0 && !write_chunk(fd, US".\n", 2, use_crlf)) - return FALSE; + if (sread == -1) + { + save_errno = errno; + rc = FALSE; + } } -/* Write out any remaining data in the buffer before returning. */ - -return (len = chunk_ptr - deliver_out_buffer) <= 0 || - transport_write_block(fd, deliver_out_buffer, len); +CLEANUP: +/* unlink -K file */ +(void)close(dkim_fd); +Uunlink(dkim_spool_name); +errno = save_errno; +return rc; } +#endif @@ -930,13 +1206,18 @@ transport_write_message(address_item *addr, int fd, int options, BOOL use_crlf; BOOL last_filter_was_NL = TRUE; int rc, len, yield, fd_read, fd_write, save_errno; -int pfd[2]; +int pfd[2] = {-1, -1}; pid_t filter_pid, write_pid; +transport_filter_timed_out = FALSE; + /* If there is no filter command set up, call the internal function that does the actual work, passing it the incoming fd, and return its result. */ -if (transport_filter_argv == NULL) +if ( !transport_filter_argv + || !*transport_filter_argv + || !**transport_filter_argv + ) return internal_transport_write_message(addr, fd, options, size_limit, add_headers, remove_headers, check_string, escape_string, rewrite_rules, rewrite_existflags); @@ -969,10 +1250,10 @@ save_errno = 0; yield = FALSE; write_pid = (pid_t)(-1); -fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); -filter_pid = child_open(transport_filter_argv, NULL, 077, &fd_write, &fd_read, - FALSE); -fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) & ~FD_CLOEXEC); +(void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); +filter_pid = child_open(USS transport_filter_argv, NULL, 077, + &fd_write, &fd_read, FALSE); +(void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) & ~FD_CLOEXEC); if (filter_pid < 0) goto TIDY_UP; /* errno set */ DEBUG(D_transport) @@ -987,25 +1268,30 @@ if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */ if ((write_pid = fork()) == 0) { BOOL rc; - close(fd_read); - close(pfd[pipe_read]); + (void)close(fd_read); + (void)close(pfd[pipe_read]); nl_check_length = nl_escape_length = 0; rc = internal_transport_write_message(addr, fd_write, (options & ~(topt_use_crlf | topt_end_dot)), size_limit, add_headers, remove_headers, NULL, NULL, rewrite_rules, rewrite_existflags); save_errno = errno; - write(pfd[pipe_write], (void *)&rc, sizeof(BOOL)); - write(pfd[pipe_write], (void *)&save_errno, sizeof(int)); - write(pfd[pipe_write], (void *)&(addr->more_errno), sizeof(int)); + if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL)) + != sizeof(BOOL) + || write(pfd[pipe_write], (void *)&save_errno, sizeof(int)) + != sizeof(int) + || write(pfd[pipe_write], (void *)&(addr->more_errno), sizeof(int)) + != sizeof(int) + ) + rc = FALSE; /* compiler quietening */ _exit(0); } save_errno = errno; /* Parent process: close our copy of the writing subprocess' pipes. */ -close(pfd[pipe_write]); -close(fd_write); +(void)close(pfd[pipe_write]); +(void)close(fd_write); fd_write = -1; /* Writing process creation failed */ @@ -1045,6 +1331,7 @@ for (;;) if (sigalrm_seen) { errno = ETIMEDOUT; + transport_filter_timed_out = TRUE; goto TIDY_UP; } @@ -1074,8 +1361,8 @@ sure. Also apply a paranoia timeout. */ TIDY_UP: save_errno = errno; -close(fd_read); -if (fd_write > 0) close(fd_write); +(void)close(fd_read); +if (fd_write > 0) (void)close(fd_write); if (!yield) { @@ -1095,7 +1382,7 @@ if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield) } /* Wait for the writing process to complete. If it ends successfully, -read the results from its pipe, provided we haven't already had a filter +read the results from its pipe, provided we haven't already had a filter process failure. */ DEBUG(D_transport) debug_printf("waiting for writing process\n"); @@ -1104,14 +1391,14 @@ if (write_pid > 0) rc = child_close(write_pid, 30); if (yield) { - if (rc == 0) + if (rc == 0) { BOOL ok; - read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)); + int dummy = read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)); if (!ok) { - read(pfd[pipe_read], (void *)&save_errno, sizeof(int)); - read(pfd[pipe_read], (void *)&(addr->more_errno), sizeof(int)); + dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int)); + dummy = read(pfd[pipe_read], (void *)&(addr->more_errno), sizeof(int)); yield = FALSE; } } @@ -1122,9 +1409,9 @@ if (write_pid > 0) addr->more_errno = rc; DEBUG(D_transport) debug_printf("writing process returned %d\n", rc); } - } + } } -close(pfd[pipe_read]); +(void)close(pfd[pipe_read]); /* If there have been no problems we can now add the terminating "." if this is SMTP output, turning off escaping beforehand. If the last character from the @@ -1193,8 +1480,7 @@ better. Old records should eventually get swept up by the exim_tidydb utility. Arguments: - hostlist list of hosts that this message could be sent to; - the update_waiting flag is set if a host is to be noted + hostlist list of hosts that this message could be sent to tpname name of the transport Returns: nothing @@ -1204,11 +1490,13 @@ void transport_update_waiting(host_item *hostlist, uschar *tpname) { uschar buffer[256]; -uschar *prevname = US""; +const uschar *prevname = US""; host_item *host; open_db dbblock; open_db *dbm_file; +DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname); + /* Open the database for this transport */ sprintf(CS buffer, "wait-%.200s", tpname); @@ -1216,8 +1504,7 @@ dbm_file = dbfn_open(buffer, O_RDWR, &dbblock, TRUE); if (dbm_file == NULL) return; /* Scan the list of hosts for which this message is waiting, and ensure -that the message id is in each host record for those that have the -update_waiting flag set. */ +that the message id is in each host record. */ for (host = hostlist; host!= NULL; host = host->next) { @@ -1226,10 +1513,6 @@ for (host = hostlist; host!= NULL; host = host->next) uschar *s; int i, host_length; - /* Skip if the update_waiting flag is not set. */ - - if (!host->update_waiting) continue; - /* Skip if this is the same host as we just processed; otherwise remember the name for next time. */ @@ -1279,7 +1562,11 @@ for (host = hostlist; host!= NULL; host = host->next) /* If this message is already in a record, no need to update. */ - if (already) continue; + if (already) + { + DEBUG(D_transport) debug_printf("already listed for %s\n", host->name); + continue; + } /* If this record is full, write it out with a new name constructed @@ -1315,6 +1602,7 @@ for (host = hostlist; host!= NULL; host = host->next) /* Update the database */ dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length); + DEBUG(D_transport) debug_printf("added to list for %s\n", host->name); } /* All now done */ @@ -1343,20 +1631,34 @@ Arguments: as set by the caller transport new_message_id set to the message id of a waiting message more set TRUE if there are yet more messages waiting + oicf_func function to call to validate if it is ok to send + to this message_id from the current instance. + oicf_data opaque data for oicf_func Returns: TRUE if new_message_id set; FALSE otherwise */ +typedef struct msgq_s +{ + uschar message_id [MESSAGE_ID_LENGTH + 1]; + BOOL bKeep; +} msgq_t; + BOOL -transport_check_waiting(uschar *transport_name, uschar *hostname, - int local_message_max, uschar *new_message_id, BOOL *more) +transport_check_waiting(const uschar *transport_name, const uschar *hostname, + int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data) { dbdata_wait *host_record; -int host_length, path_len; +int host_length; open_db dbblock; open_db *dbm_file; uschar buffer[256]; +int i; +uschar spool_dir [PATH_MAX]; +uschar spool_file [PATH_MAX]; +struct stat statbuf; + *more = FALSE; DEBUG(D_transport) @@ -1385,8 +1687,7 @@ if (dbm_file == NULL) return FALSE; /* See if there is a record for this host; if not, there's nothing to do. */ -host_record = dbfn_read(dbm_file, hostname); -if (host_record == NULL) +if (!(host_record = dbfn_read(dbm_file, hostname))) { dbfn_close(dbm_file); DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname); @@ -1409,58 +1710,108 @@ until one is found for which a spool file actually exists. If the record gets emptied, delete it and continue with any continuation records that may exist. */ -host_length = host_record->count * MESSAGE_ID_LENGTH; +/* For Bug 1141, I refactored this major portion of the routine, it is risky +but the 1 off will remain without it. This code now allows me to SKIP over +a message I do not want to send out on this run. */ -/* Loop to handle continuation host records in the database */ +sprintf(CS spool_dir, "%s/input/", spool_directory); -for (;;) +host_length = host_record->count * MESSAGE_ID_LENGTH; + +while (1) { - BOOL found = FALSE; + msgq_t *msgq; + int msgq_count = 0; + int msgq_actual = 0; + BOOL bFound = FALSE; + BOOL bContinuation = FALSE; + + /* create an array to read entire message queue into memory for processing */ - sprintf(CS buffer, "%s/input/", spool_directory); - path_len = Ustrlen(buffer); + msgq = (msgq_t*) malloc(sizeof(msgq_t) * host_record->count); + msgq_count = host_record->count; + msgq_actual = msgq_count; - for (host_length -= MESSAGE_ID_LENGTH; host_length >= 0; - host_length -= MESSAGE_ID_LENGTH) + for (i = 0; i < host_record->count; ++i) { - struct stat statbuf; - Ustrncpy(new_message_id, host_record->text + host_length, + msgq[i].bKeep = TRUE; + + Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH), MESSAGE_ID_LENGTH); - new_message_id[MESSAGE_ID_LENGTH] = 0; + msgq[i].message_id[MESSAGE_ID_LENGTH] = 0; + } + /* first thing remove current message id if it exists */ + + for (i = 0; i < msgq_count; ++i) + if (Ustrcmp(msgq[i].message_id, message_id) == 0) + { + msgq[i].bKeep = FALSE; + break; + } + + /* now find the next acceptable message_id */ + + for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep) + { if (split_spool_directory) - sprintf(CS(buffer + path_len), "%c/%s-D", new_message_id[5], new_message_id); + sprintf(CS spool_file, "%s%c/%s-D", + spool_dir, msgq[i].message_id[5], msgq[i].message_id); else - sprintf(CS(buffer + path_len), "%s-D", new_message_id); + sprintf(CS spool_file, "%s%s-D", spool_dir, msgq[i].message_id); - /* The listed message may be the one we are currently processing. If - so, we want to remove it from the list without doing anything else. - If not, do a stat to see if it is an existing message. If it is, break - the loop to handle it. No need to bother about locks; as this is all - "hint" processing, it won't matter if it doesn't exist by the time exim - actually tries to deliver it. */ - - if (Ustrcmp(new_message_id, message_id) != 0 && - Ustat(buffer, &statbuf) == 0) + if (Ustat(spool_file, &statbuf) != 0) + msgq[i].bKeep = FALSE; + else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data)) { - found = TRUE; + Ustrcpy(new_message_id, msgq[i].message_id); + msgq[i].bKeep = FALSE; + bFound = TRUE; break; } } - /* If we have removed all the message ids from the record delete the record. - If there is a continuation record, fetch it and remove it from the file, - as it will be rewritten as the main record. Repeat in the case of an - empty continuation. */ + /* re-count */ + for (msgq_actual = 0, i = 0; i < msgq_count; ++i) + if (msgq[i].bKeep) + msgq_actual++; + + /* reassemble the host record, based on removed message ids, from in + memory queue */ + + if (msgq_actual <= 0) + { + host_length = 0; + host_record->count = 0; + } + else + { + host_length = msgq_actual * MESSAGE_ID_LENGTH; + host_record->count = msgq_actual; + + if (msgq_actual < msgq_count) + { + int new_count; + for (new_count = 0, i = 0; i < msgq_count; ++i) + if (msgq[i].bKeep) + Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH], + msgq[i].message_id, MESSAGE_ID_LENGTH); + + host_record->text[new_count * MESSAGE_ID_LENGTH] = 0; + } + } + +/* Jeremy: check for a continuation record, this code I do not know how to +test but the code should work */ while (host_length <= 0) { int i; - dbdata_wait *newr = NULL; + dbdata_wait * newr = NULL; /* Search for a continuation */ - for (i = host_record->sequence - 1; i >= 0 && newr == NULL; i--) + for (i = host_record->sequence - 1; i >= 0 && !newr; i--) { sprintf(CS buffer, "%.200s:%d", hostname, i); newr = dbfn_read(dbm_file, buffer); @@ -1468,7 +1819,7 @@ for (;;) /* If no continuation, delete the current and break the loop */ - if (newr == NULL) + if (!newr) { dbfn_delete(dbm_file, hostname); break; @@ -1479,11 +1830,15 @@ for (;;) dbfn_delete(dbm_file, buffer); host_record = newr; host_length = host_record->count * MESSAGE_ID_LENGTH; - } - /* If we found an existing message, break the continuation loop. */ + bContinuation = TRUE; + } - if (found) break; + if (bFound) /* Usual exit from main loop */ + { + free (msgq); + break; + } /* If host_length <= 0 we have emptied a record and not found a good message, and there are no continuation records. Otherwise there is a continuation @@ -1495,7 +1850,20 @@ for (;;) DEBUG(D_transport) debug_printf("waiting messages already delivered\n"); return FALSE; } - } + + /* we were not able to find an acceptable message, nor was there a + * continuation record. So bug out, outer logic will clean this up. + */ + + if (!bContinuation) + { + Ustrcpy(new_message_id, message_id); + dbfn_close(dbm_file); + return FALSE; + } + + free(msgq); + } /* we need to process a continuation record */ /* Control gets here when an existing message has been encountered; its id is in new_message_id, and host_length is the revised length of the @@ -1505,6 +1873,7 @@ record if required, close the database, and return TRUE. */ if (host_length > 0) { host_record->count = host_length/MESSAGE_ID_LENGTH; + dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length); *more = TRUE; } @@ -1513,8 +1882,6 @@ dbfn_close(dbm_file); return TRUE; } - - /************************************************* * Deliver waiting message down same socket * *************************************************/ @@ -1534,8 +1901,8 @@ Returns: FALSE if fork fails; TRUE otherwise */ BOOL -transport_pass_socket(uschar *transport_name, uschar *hostname, - uschar *hostaddress, uschar *id, int socket_fd) +transport_pass_socket(const uschar *transport_name, const uschar *hostname, + const uschar *hostaddress, uschar *id, int socket_fd) { pid_t pid; int status; @@ -1545,7 +1912,7 @@ DEBUG(D_transport) debug_printf("transport_pass_socket entered\n"); if ((pid = fork()) == 0) { int i = 16; - uschar **argv; + const uschar **argv; /* Disconnect entirely from the parent process. If we are running in the test harness, wait for a bit to allow the previous process time to finish, @@ -1553,12 +1920,15 @@ if ((pid = fork()) == 0) automatic comparison. */ if ((pid = fork()) != 0) _exit(EXIT_SUCCESS); - if (running_in_test_harness) millisleep(500); + if (running_in_test_harness) sleep(1); /* Set up the calling arguments; use the standard function for the basics, but we have a number of extras that may be added. */ - argv = child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0); + argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0); + + /* Call with the dsn flag */ + if (smtp_use_dsn) argv[i++] = US"-MCD"; if (smtp_authenticated) argv[i++] = US"-MCA"; @@ -1577,9 +1947,9 @@ if ((pid = fork()) == 0) } argv[i++] = US"-MC"; - argv[i++] = transport_name; - argv[i++] = hostname; - argv[i++] = hostaddress; + argv[i++] = US transport_name; + argv[i++] = US hostname; + argv[i++] = US hostaddress; argv[i++] = string_sprintf("%d", continue_sequence + 1); argv[i++] = id; argv[i++] = NULL; @@ -1588,8 +1958,8 @@ if ((pid = fork()) == 0) if (socket_fd != 0) { - dup2(socket_fd, 0); - close(socket_fd); + (void)dup2(socket_fd, 0); + (void)close(socket_fd); } DEBUG(D_exec) debug_print_argv(argv); @@ -1633,7 +2003,7 @@ case, no addresses are passed. Arguments: argvptr pointer to anchor for argv vector - cmd points to the command string + cmd points to the command string (modified IN PLACE) expand_arguments true if expansion is to occur expand_failed error value to set if expansion fails; not relevant if addr == NULL @@ -1647,11 +2017,12 @@ Returns: TRUE if all went well; otherwise an error will be */ BOOL -transport_set_up_command(uschar ***argvptr, uschar *cmd, BOOL expand_arguments, - int expand_failed, address_item *addr, uschar *etext, uschar **errptr) +transport_set_up_command(const uschar ***argvptr, uschar *cmd, + BOOL expand_arguments, int expand_failed, address_item *addr, + uschar *etext, uschar **errptr) { address_item *ad; -uschar **argv; +const uschar **argv; uschar *s, *ss; int address_count = 0; int argcount = 0; @@ -1685,7 +2056,7 @@ while (*s != 0 && argcount < max_args) if (*s != 0) s++; *ss++ = 0; } - else argv[argcount++] = string_dequote(&s); + else argv[argcount++] = string_copy(string_dequote(CUSS &s)); while (isspace(*s)) s++; } @@ -1759,7 +2130,123 @@ if (expand_arguments) memmove(argv + i + 1 + additional, argv + i + 1, (argcount - i)*sizeof(uschar *)); - for (ad = addr; ad != NULL; ad = ad->next) argv[i++] = ad->address; + for (ad = addr; ad != NULL; ad = ad->next) { + argv[i++] = ad->address; + argcount++; + } + + /* Subtract one since we replace $pipe_addresses */ + argcount--; + i--; + } + + /* Handle special case of $address_pipe when af_force_command is set */ + + else if (addr != NULL && testflag(addr,af_force_command) && + (Ustrcmp(argv[i], "$address_pipe") == 0 || + Ustrcmp(argv[i], "${address_pipe}") == 0)) + { + int address_pipe_i; + int address_pipe_argcount = 0; + int address_pipe_max_args; + uschar **address_pipe_argv; + + /* We can never have more then the argv we will be loading into */ + address_pipe_max_args = max_args - argcount + 1; + + DEBUG(D_transport) + debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args); + + /* We allocate an additional for (uschar *)0 */ + address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *)); + + /* +1 because addr->local_part[0] == '|' since af_force_command is set */ + s = expand_string(addr->local_part + 1); + + if (s == NULL || *s == '\0') + { + addr->transport_return = FAIL; + addr->message = string_sprintf("Expansion of \"%s\" " + "from command \"%s\" in %s failed: %s", + (addr->local_part + 1), cmd, etext, expand_string_message); + return FALSE; + } + + while (isspace(*s)) s++; /* strip leading space */ + + while (*s != 0 && address_pipe_argcount < address_pipe_max_args) + { + if (*s == '\'') + { + ss = s + 1; + while (*ss != 0 && *ss != '\'') ss++; + address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++); + while (*s != 0 && *s != '\'') *ss++ = *s++; + if (*s != 0) s++; + *ss++ = 0; + } + else address_pipe_argv[address_pipe_argcount++] = + string_copy(string_dequote(CUSS &s)); + while (isspace(*s)) s++; /* strip space after arg */ + } + + address_pipe_argv[address_pipe_argcount] = (uschar *)0; + + /* If *s != 0 we have run out of argument slots. */ + if (*s != 0) + { + uschar *msg = string_sprintf("Too many arguments in $address_pipe " + "\"%s\" in %s", addr->local_part + 1, etext); + if (addr != NULL) + { + addr->transport_return = FAIL; + addr->message = msg; + } + else *errptr = msg; + return FALSE; + } + + /* address_pipe_argcount - 1 + * because we are replacing $address_pipe in the argument list + * with the first thing it expands to */ + if (argcount + address_pipe_argcount - 1 > max_args) + { + addr->transport_return = FAIL; + addr->message = string_sprintf("Too many arguments to command " + "\"%s\" after expanding $address_pipe in %s", cmd, etext); + return FALSE; + } + + /* If we are not just able to replace the slot that contained + * $address_pipe (address_pipe_argcount == 1) + * We have to move the existing argv by address_pipe_argcount - 1 + * Visually if address_pipe_argcount == 2: + * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0] + * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0] + */ + if (address_pipe_argcount > 1) + memmove( + /* current position + additonal args */ + argv + i + address_pipe_argcount, + /* current position + 1 (for the (uschar *)0 at the end) */ + argv + i + 1, + /* -1 for the (uschar *)0 at the end)*/ + (argcount - i)*sizeof(uschar *) + ); + + /* Now we fill in the slots we just moved argv out of + * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0] + */ + for (address_pipe_i = 0; + address_pipe_argv[address_pipe_i] != (uschar *)0; + address_pipe_i++) + { + argv[i++] = address_pipe_argv[address_pipe_i]; + argcount++; + } + + /* Subtract one since we replace $address_pipe */ + argcount--; i--; } @@ -1767,9 +2254,9 @@ if (expand_arguments) else { - uschar *expanded_arg; + const uschar *expanded_arg; enable_dollar_recipients = allow_dollar_recipients; - expanded_arg = expand_string(argv[i]); + expanded_arg = expand_cstring(argv[i]); enable_dollar_recipients = FALSE; if (expanded_arg == NULL) @@ -1800,4 +2287,6 @@ if (expand_arguments) return TRUE; } +/* vi: aw ai sw=2 +*/ /* End of transport.c */