Copyright updates:
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
1e1ddfac 6/* Copyright (c) The Exim Maintainers 2020 */
059ec3d9
PH
7/* See the file NOTICE for conditions of use and distribution. */
8
9/* General functions concerned with transportation, and generic options for all
10transports. */
11
12
13#include "exim.h"
14
059ec3d9
PH
15/* Generic options for transports, all of which live inside transport_instance
16data blocks and which therefore have the opt_public flag set. Note that there
17are other options living inside this structure which can be set only from
18certain transports. */
13a4b4c1 19#define LOFF(field) OPT_OFF(transport_instance, field)
059ec3d9
PH
20
21optionlist optionlist_transports[] = {
f2ed27cf 22 /* name type value */
059ec3d9 23 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
13a4b4c1 24 LOFF(expand_gid) },
059ec3d9 25 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
13a4b4c1 26 LOFF(expand_uid) },
059ec3d9 27 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
13a4b4c1 28 LOFF(rewrite_existflags) },
059ec3d9 29 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
13a4b4c1 30 LOFF(rewrite_rules) },
059ec3d9 31 { "*set_group", opt_bool|opt_hidden|opt_public,
13a4b4c1 32 LOFF(gid_set) },
059ec3d9 33 { "*set_user", opt_bool|opt_hidden|opt_public,
13a4b4c1 34 LOFF(uid_set) },
059ec3d9 35 { "body_only", opt_bool|opt_public,
13a4b4c1 36 LOFF(body_only) },
059ec3d9 37 { "current_directory", opt_stringptr|opt_public,
13a4b4c1 38 LOFF(current_dir) },
059ec3d9 39 { "debug_print", opt_stringptr | opt_public,
13a4b4c1 40 LOFF(debug_string) },
059ec3d9 41 { "delivery_date_add", opt_bool|opt_public,
13a4b4c1 42 LOFF(delivery_date_add) },
059ec3d9 43 { "disable_logging", opt_bool|opt_public,
13a4b4c1 44 LOFF(disable_logging) },
059ec3d9 45 { "driver", opt_stringptr|opt_public,
13a4b4c1 46 LOFF(driver_name) },
059ec3d9 47 { "envelope_to_add", opt_bool|opt_public,
13a4b4c1 48 LOFF(envelope_to_add) },
0cbf2b82 49#ifndef DISABLE_EVENT
774ef2d7 50 { "event_action", opt_stringptr | opt_public,
13a4b4c1 51 LOFF(event_action) },
774ef2d7 52#endif
059ec3d9 53 { "group", opt_expand_gid|opt_public,
13a4b4c1 54 LOFF(gid) },
846726c5 55 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
13a4b4c1 56 LOFF(add_headers) },
059ec3d9 57 { "headers_only", opt_bool|opt_public,
13a4b4c1 58 LOFF(headers_only) },
846726c5 59 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
13a4b4c1 60 LOFF(remove_headers) },
059ec3d9 61 { "headers_rewrite", opt_rewrite|opt_public,
13a4b4c1 62 LOFF(headers_rewrite) },
059ec3d9 63 { "home_directory", opt_stringptr|opt_public,
13a4b4c1 64 LOFF(home_dir) },
059ec3d9 65 { "initgroups", opt_bool|opt_public,
13a4b4c1 66 LOFF(initgroups) },
fa41615d 67 { "max_parallel", opt_stringptr|opt_public,
13a4b4c1 68 LOFF(max_parallel) },
059ec3d9 69 { "message_size_limit", opt_stringptr|opt_public,
13a4b4c1 70 LOFF(message_size_limit) },
059ec3d9 71 { "rcpt_include_affixes", opt_bool|opt_public,
13a4b4c1 72 LOFF(rcpt_include_affixes) },
059ec3d9 73 { "retry_use_local_part", opt_bool|opt_public,
13a4b4c1 74 LOFF(retry_use_local_part) },
059ec3d9 75 { "return_path", opt_stringptr|opt_public,
13a4b4c1 76 LOFF(return_path) },
059ec3d9 77 { "return_path_add", opt_bool|opt_public,
13a4b4c1 78 LOFF(return_path_add) },
059ec3d9 79 { "shadow_condition", opt_stringptr|opt_public,
13a4b4c1 80 LOFF(shadow_condition) },
059ec3d9 81 { "shadow_transport", opt_stringptr|opt_public,
13a4b4c1 82 LOFF(shadow) },
059ec3d9 83 { "transport_filter", opt_stringptr|opt_public,
13a4b4c1 84 LOFF(filter_command) },
059ec3d9 85 { "transport_filter_timeout", opt_time|opt_public,
13a4b4c1 86 LOFF(filter_timeout) },
059ec3d9 87 { "user", opt_expand_uid|opt_public,
13a4b4c1 88 LOFF(uid) }
059ec3d9
PH
89};
90
c0b9d3e8 91int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9 92
d185889f
JH
93#ifdef MACRO_PREDEF
94
5f69a529 95# include "macro_predef.h"
059ec3d9 96
c0b9d3e8 97void
d185889f 98options_transports(void)
c0b9d3e8 99{
d185889f 100uschar buf[64];
c0b9d3e8 101
d185889f 102options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
c0b9d3e8 103
d7978c0f 104for (transport_info * ti = transports_available; ti->driver_name[0]; ti++)
4945f557 105 {
cab0c277 106 spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", ti->driver_name);
d185889f
JH
107 builtin_macro_create(buf);
108 options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
4945f557 109 }
383832ef 110}
c0b9d3e8 111
d185889f
JH
112#else /*!MACRO_PREDEF*/
113
114/* Structure for keeping list of addresses that have been added to
115Envelope-To:, in order to avoid duplication. */
116
117struct aci {
118 struct aci *next;
119 address_item *ptr;
120 };
121
122
123/* Static data for write_chunk() */
124
125static uschar *chunk_ptr; /* chunk pointer */
126static uschar *nl_check; /* string to look for at line start */
127static int nl_check_length; /* length of same */
128static uschar *nl_escape; /* string to insert */
129static int nl_escape_length; /* length of same */
130static int nl_partial_match; /* length matched at chunk end */
131
132
059ec3d9
PH
133/*************************************************
134* Initialize transport list *
135*************************************************/
136
137/* Read the transports section of the configuration file, and set up a chain of
138transport instances according to its contents. Each transport has generic
139options and may also have its own private options. This function is only ever
140called when transports == NULL. We use generic code in readconf to do most of
141the work. */
142
143void
144transport_init(void)
145{
059ec3d9
PH
146readconf_driver_init(US"transport",
147 (driver_instance **)(&transports), /* chain anchor */
148 (driver_info *)transports_available, /* available drivers */
149 sizeof(transport_info), /* size of info block */
150 &transport_defaults, /* default values for generic options */
151 sizeof(transport_instance), /* size of instance block */
152 optionlist_transports, /* generic options */
153 optionlist_transports_size);
154
155/* Now scan the configured transports and check inconsistencies. A shadow
156transport is permitted only for local transports. */
157
d7978c0f 158for (transport_instance * t = transports; t; t = t->next)
059ec3d9 159 {
7b4c8c1f
JH
160 if (!t->info->local && t->shadow)
161 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
162 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
163
164 if (t->body_only && t->headers_only)
165 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
166 "%s transport: body_only and headers_only are mutually exclusive",
167 t->name);
168 }
169}
170
171
172
173/*************************************************
174* Write block of data *
175*************************************************/
176
73a10da9
JH
177static int
178tpt_write(int fd, uschar * block, int len, BOOL more, int options)
179{
180return
181#ifndef DISABLE_TLS
182 tls_out.active.sock == fd
183 ? tls_write(tls_out.active.tls_ctx, block, len, more) :
184#endif
185#ifdef MSG_MORE
186 more && !(options & topt_not_socket) ? send(fd, block, len, MSG_MORE) :
187#endif
188 write(fd, block, len);
189}
190
059ec3d9
PH
191/* Subroutine called by write_chunk() and at the end of the message actually
192to write a data block. Also called directly by some transports to write
193additional data to the file descriptor (e.g. prefix, suffix).
194
195If a transport wants data transfers to be timed, it sets a non-zero value in
196transport_write_timeout. A non-zero transport_write_timeout causes a timer to
197be set for each block of data written from here. If time runs out, then write()
198fails and provokes an error return. The caller can then inspect sigalrm_seen to
199check for a timeout.
200
201On some systems, if a quota is exceeded during the write, the yield is the
202number of bytes written rather than an immediate error code. This also happens
203on some systems in other cases, for example a pipe that goes away because the
204other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
205get the error codes the first time.
206
207The write() function is also interruptible; the Solaris 2.6 man page says:
208
209 If write() is interrupted by a signal before it writes any
210 data, it will return -1 with errno set to EINTR.
211
212 If write() is interrupted by a signal after it successfully
213 writes some data, it will return the number of bytes written.
214
215To handle these cases, we want to restart the write() to output the remainder
216of the data after a non-negative return from write(), except after a timeout.
217In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
218proper error then occurs. In principle, after an interruption, the second
219write() could suffer the same fate, but we do not want to continue for
220evermore, so stick a maximum repetition count on the loop to act as a
221longstop.
222
223Arguments:
42055a33 224 tctx transport context: file descriptor or string to write to
059ec3d9
PH
225 block block of bytes to write
226 len number of bytes to write
925ac8e4 227 more further data expected soon
059ec3d9
PH
228
229Returns: TRUE on success, FALSE on failure (with errno preserved);
230 transport_count is incremented by the number of bytes written
231*/
232
42055a33 233static BOOL
73a10da9 234transport_write_block_fd(transport_ctx * tctx, uschar * block, int len, BOOL more)
059ec3d9 235{
d7978c0f 236int rc, save_errno;
958541e9 237int local_timeout = transport_write_timeout;
73a10da9 238int connretry = 1;
42055a33 239int fd = tctx->u.fd;
958541e9
PH
240
241/* This loop is for handling incomplete writes and other retries. In most
242normal cases, it is only ever executed once. */
059ec3d9 243
d7978c0f 244for (int i = 0; i < 100; i++)
059ec3d9
PH
245 {
246 DEBUG(D_transport)
42055a33
JH
247 debug_printf("writing data block fd=%d size=%d timeout=%d%s\n",
248 fd, len, local_timeout, more ? " (more expected)" : "");
059ec3d9 249
73a10da9
JH
250 /* When doing TCP Fast Open we may get this far before the 3-way handshake
251 is complete, and write returns ENOTCONN. Detect that, wait for the socket
252 to become writable, and retry once only. */
059ec3d9 253
73a10da9 254 for(;;)
059ec3d9 255 {
73a10da9
JH
256 fd_set fds;
257 /* This code makes use of alarm() in order to implement the timeout. This
258 isn't a very tidy way of doing things. Using non-blocking I/O with select()
259 provides a neater approach. However, I don't know how to do this when TLS is
260 in use. */
42055a33 261
73a10da9
JH
262 if (transport_write_timeout <= 0) /* No timeout wanted */
263 {
264 rc = tpt_write(fd, block, len, more, tctx->options);
265 save_errno = errno;
266 }
267 else /* Timeout wanted. */
059ec3d9 268 {
3a2adc82 269 sigalrm_seen = FALSE;
73a10da9
JH
270 ALARM(local_timeout);
271 rc = tpt_write(fd, block, len, more, tctx->options);
272 save_errno = errno;
273 local_timeout = ALARM_CLR(0);
274 if (sigalrm_seen)
275 {
276 errno = ETIMEDOUT;
277 return FALSE;
278 }
059ec3d9 279 }
73a10da9
JH
280
281 if (rc >= 0 || errno != ENOTCONN || connretry <= 0)
282 break;
283
284 FD_ZERO(&fds); FD_SET(fd, &fds);
285 select(fd+1, NULL, &fds, NULL, NULL); /* could set timout? */
286 connretry--;
059ec3d9
PH
287 }
288
289 /* Hopefully, the most common case is success, so test that first. */
290
291 if (rc == len) { transport_count += len; return TRUE; }
292
958541e9
PH
293 /* A non-negative return code is an incomplete write. Try again for the rest
294 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
295
296 if (rc >= 0)
297 {
298 len -= rc;
299 block += rc;
300 transport_count += rc;
301 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 302 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
303 }
304
305 /* A negative return code with an EINTR error is another form of
306 incomplete write, zero bytes having been written */
307
308 if (save_errno == EINTR)
309 {
310 DEBUG(D_transport)
311 debug_printf("write interrupted before anything written\n");
958541e9 312 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
313 }
314
315 /* A response of EAGAIN from write() is likely only in the case of writing
316 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
317
318 if (save_errno == EAGAIN)
319 {
320 DEBUG(D_transport)
321 debug_printf("write temporarily locked out, waiting 1 sec\n");
322 sleep(1);
958541e9
PH
323
324 /* Before continuing to try another write, check that we haven't run out of
325 time. */
326
327 CHECK_TIMEOUT:
328 if (transport_write_timeout > 0 && local_timeout <= 0)
329 {
330 errno = ETIMEDOUT;
331 return FALSE;
332 }
059ec3d9
PH
333 continue;
334 }
335
336 /* Otherwise there's been an error */
337
338 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
339 strerror(save_errno));
340 errno = save_errno;
341 return FALSE;
342 }
343
344/* We've tried and tried and tried but still failed */
345
346errno = ERRNO_WRITEINCOMPLETE;
347return FALSE;
348}
349
350
42055a33
JH
351BOOL
352transport_write_block(transport_ctx * tctx, uschar *block, int len, BOOL more)
353{
354if (!(tctx->options & topt_output_string))
355 return transport_write_block_fd(tctx, block, len, more);
356
357/* Write to expanding-string. NOTE: not NUL-terminated */
358
359if (!tctx->u.msg)
acec9514 360 tctx->u.msg = string_get(1024);
42055a33 361
acec9514 362tctx->u.msg = string_catn(tctx->u.msg, block, len);
42055a33
JH
363return TRUE;
364}
365
366
059ec3d9
PH
367
368
369/*************************************************
370* Write formatted string *
371*************************************************/
372
373/* This is called by various transports. It is a convenience function.
374
375Arguments:
376 fd file descriptor
377 format string format
378 ... arguments for format
379
380Returns: the yield of transport_write_block()
381*/
382
383BOOL
1ba28e2b 384transport_write_string(int fd, const char *format, ...)
059ec3d9 385{
cab0c277 386transport_ctx tctx = {{0}};
d12746bc 387gstring gs = { .size = big_buffer_size, .ptr = 0, .s = big_buffer };
059ec3d9 388va_list ap;
d12746bc 389
f3ebb786
JH
390/* Use taint-unchecked routines for writing into big_buffer, trusting
391that the result will never be expanded. */
392
059ec3d9 393va_start(ap, format);
f3ebb786 394if (!string_vformat(&gs, SVFMT_TAINT_NOCHK, format, ap))
059ec3d9
PH
395 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
396va_end(ap);
42055a33 397tctx.u.fd = fd;
d12746bc 398return transport_write_block(&tctx, gs.s, gs.ptr, FALSE);
059ec3d9
PH
399}
400
401
402
403
42055a33
JH
404void
405transport_write_reset(int options)
406{
407if (!(options & topt_continuation)) chunk_ptr = deliver_out_buffer;
408nl_partial_match = -1;
409nl_check_length = nl_escape_length = 0;
410}
411
412
413
059ec3d9
PH
414/*************************************************
415* Write character chunk *
416*************************************************/
417
418/* Subroutine used by transport_write_message() to scan character chunks for
419newlines and act appropriately. The object is to minimise the number of writes.
420The output byte stream is buffered up in deliver_out_buffer, which is written
421only when it gets full, thus minimizing write operations and TCP packets.
422
423Static data is used to handle the case when the last character of the previous
424chunk was NL, or matched part of the data that has to be escaped.
425
426Arguments:
42055a33
JH
427 tctx transport context - processing to be done during output,
428 and file descriptor to write to
059ec3d9
PH
429 chunk pointer to data to write
430 len length of data to write
059ec3d9
PH
431
432In addition, the static nl_xxx variables must be set as required.
433
434Returns: TRUE on success, FALSE on failure (with errno preserved)
435*/
436
42055a33
JH
437BOOL
438write_chunk(transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
439{
440uschar *start = chunk;
441uschar *end = chunk + len;
059ec3d9
PH
442int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
443
444/* The assumption is made that the check string will never stretch over move
445than one chunk since the only time there are partial matches is when copying
446the body in large buffers. There is always enough room in the buffer for an
447escape string, since the loop below ensures this for each character it
448processes, and it won't have stuck in the escape string if it left a partial
449match. */
450
451if (nl_partial_match >= 0)
452 {
453 if (nl_check_length > 0 && len >= nl_check_length &&
454 Ustrncmp(start, nl_check + nl_partial_match,
455 nl_check_length - nl_partial_match) == 0)
456 {
457 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
458 chunk_ptr += nl_escape_length;
459 start += nl_check_length - nl_partial_match;
460 }
461
462 /* The partial match was a false one. Insert the characters carried over
463 from the previous chunk. */
464
465 else if (nl_partial_match > 0)
466 {
467 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
468 chunk_ptr += nl_partial_match;
469 }
470
471 nl_partial_match = -1;
472 }
473
474/* Now process the characters in the chunk. Whenever we hit a newline we check
475for possible escaping. The code for the non-NL route should be as fast as
476possible. */
477
d7978c0f 478for (uschar * ptr = start; ptr < end; ptr++)
059ec3d9 479 {
6d5c916c 480 int ch, len;
059ec3d9
PH
481
482 /* Flush the buffer if it has reached the threshold - we want to leave enough
483 room for the next uschar, plus a possible extra CR for an LF, plus the escape
484 string. */
485
6d5c916c 486 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 487 {
58fc5fb2
JH
488 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
489
6d5c916c
JH
490 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
491 from previous SMTP commands. */
492
d315eda1 493 if (tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2 494 {
d2aa036b 495 if ( tctx->chunk_cb(tctx, (unsigned)len, 0) != OK
42055a33 496 || !transport_write_block(tctx, deliver_out_buffer, len, FALSE)
d2aa036b 497 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
58fc5fb2
JH
498 )
499 return FALSE;
500 }
501 else
42055a33 502 if (!transport_write_block(tctx, deliver_out_buffer, len, FALSE))
6d5c916c 503 return FALSE;
059ec3d9
PH
504 chunk_ptr = deliver_out_buffer;
505 }
506
328c5688
JH
507 /* Remove CR before NL if required */
508
509 if ( *ptr == '\r' && ptr[1] == '\n'
d315eda1 510 && !(tctx->options & topt_use_crlf)
8768d548 511 && f.spool_file_wireformat
328c5688
JH
512 )
513 ptr++;
514
059ec3d9
PH
515 if ((ch = *ptr) == '\n')
516 {
517 int left = end - ptr - 1; /* count of chars left after NL */
518
519 /* Insert CR before NL if required */
520
8768d548 521 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
328c5688 522 *chunk_ptr++ = '\r';
059ec3d9 523 *chunk_ptr++ = '\n';
332f5cf3 524 transport_newlines++;
059ec3d9
PH
525
526 /* The check_string test (formerly "from hack") replaces the specific
527 string at the start of a line with an escape string (e.g. "From " becomes
528 ">From " or "." becomes "..". It is a case-sensitive test. The length
529 check above ensures there is always enough room to insert this string. */
530
531 if (nl_check_length > 0)
532 {
533 if (left >= nl_check_length &&
534 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
535 {
536 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
537 chunk_ptr += nl_escape_length;
538 ptr += nl_check_length;
539 }
540
541 /* Handle the case when there isn't enough left to match the whole
542 check string, but there may be a partial match. We remember how many
543 characters matched, and finish processing this chunk. */
544
545 else if (left <= 0) nl_partial_match = 0;
546
547 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
548 {
549 nl_partial_match = left;
550 ptr = end;
551 }
552 }
553 }
554
555 /* Not a NL character */
556
557 else *chunk_ptr++ = ch;
558 }
559
560return TRUE;
561}
562
563
564
565
566/*************************************************
567* Generate address for RCPT TO *
568*************************************************/
569
570/* This function puts together an address for RCPT to, using the caseful
571version of the local part and the caseful version of the domain. If there is no
572prefix or suffix, or if affixes are to be retained, we can just use the
573original address. Otherwise, if there is a prefix but no suffix we can use a
574pointer into the original address. If there is a suffix, however, we have to
575build a new string.
576
577Arguments:
578 addr the address item
579 include_affixes TRUE if affixes are to be included
580
581Returns: a string
582*/
583
584uschar *
585transport_rcpt_address(address_item *addr, BOOL include_affixes)
586{
587uschar *at;
588int plen, slen;
589
590if (include_affixes)
591 {
592 setflag(addr, af_include_affixes); /* Affects logged => line */
593 return addr->address;
594 }
595
759502e5 596if (!addr->suffix)
059ec3d9 597 {
759502e5 598 if (!addr->prefix) return addr->address;
059ec3d9
PH
599 return addr->address + Ustrlen(addr->prefix);
600 }
601
602at = Ustrrchr(addr->address, '@');
759502e5 603plen = addr->prefix ? Ustrlen(addr->prefix) : 0;
059ec3d9
PH
604slen = Ustrlen(addr->suffix);
605
bb07bcd3 606return string_sprintf("%.*s@%s", (int)(at - addr->address - plen - slen),
059ec3d9
PH
607 addr->address + plen, at + 1);
608}
609
610
611/*************************************************
612* Output Envelope-To: address & scan duplicates *
613*************************************************/
614
615/* This function is called from internal_transport_write_message() below, when
616generating an Envelope-To: header line. It checks for duplicates of the given
617address and its ancestors. When one is found, this function calls itself
618recursively, to output the envelope address of the duplicate.
619
620We want to avoid duplication in the list, which can arise for example when
621A->B,C and then both B and C alias to D. This can also happen when there are
622unseen drivers in use. So a list of addresses that have been output is kept in
623the plist variable.
624
625It is also possible to have loops in the address ancestry/duplication graph,
626for example if there are two top level addresses A and B and we have A->B,C and
627B->A. To break the loop, we use a list of processed addresses in the dlist
628variable.
629
630After handling duplication, this function outputs the progenitor of the given
631address.
632
633Arguments:
634 p the address we are interested in
635 pplist address of anchor of the list of addresses not to output
636 pdlist address of anchor of the list of processed addresses
637 first TRUE if this is the first address; set it FALSE afterwards
6d5c916c 638 tctx transport context - processing to be done during output
42055a33 639 and the file descriptor to write to
059ec3d9
PH
640
641Returns: FALSE if writing failed
642*/
643
644static BOOL
645write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
42055a33 646 BOOL *first, transport_ctx * tctx)
059ec3d9
PH
647{
648address_item *pp;
649struct aci *ppp;
650
651/* Do nothing if we have already handled this address. If not, remember it
652so that we don't handle it again. */
653
ff5aac2b 654for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9 655
f3ebb786 656ppp = store_get(sizeof(struct aci), FALSE);
059ec3d9
PH
657ppp->next = *pdlist;
658*pdlist = ppp;
659ppp->ptr = p;
660
661/* Now scan up the ancestry, checking for duplicates at each generation. */
662
663for (pp = p;; pp = pp->parent)
664 {
665 address_item *dup;
ff5aac2b
JH
666 for (dup = addr_duplicate; dup; dup = dup->next)
667 if (dup->dupof == pp) /* a dup of our address */
42055a33 668 if (!write_env_to(dup, pplist, pdlist, first, tctx))
ff5aac2b
JH
669 return FALSE;
670 if (!pp->parent) break;
059ec3d9
PH
671 }
672
673/* Check to see if we have already output the progenitor. */
674
ff5aac2b
JH
675for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
676if (ppp) return TRUE;
059ec3d9
PH
677
678/* Remember what we have output, and output it. */
679
f3ebb786 680ppp = store_get(sizeof(struct aci), FALSE);
059ec3d9
PH
681ppp->next = *pplist;
682*pplist = ppp;
683ppp->ptr = pp;
684
42055a33 685if (!*first && !write_chunk(tctx, US",\n ", 3)) return FALSE;
059ec3d9 686*first = FALSE;
42055a33 687return write_chunk(tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
688}
689
690
691
692
4c04137d 693/* Add/remove/rewrite headers, and send them plus the empty-line separator.
511a6c14
JH
694
695Globals:
696 header_list
697
698Arguments:
699 addr (chain of) addresses (for extra headers), or NULL;
700 only the first address is used
44bc8f0c 701 tctx transport context
59932f7d 702 sendfn function for output (transport or verify)
511a6c14
JH
703
704Returns: TRUE on success; FALSE on failure.
705*/
706BOOL
42055a33
JH
707transport_headers_send(transport_ctx * tctx,
708 BOOL (*sendfn)(transport_ctx * tctx, uschar * s, int len))
511a6c14 709{
6b46ecc6 710const uschar *list;
6d5c916c
JH
711transport_instance * tblock = tctx ? tctx->tblock : NULL;
712address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
713
714/* Then the message's headers. Don't write any that are flagged as "old";
715that means they were rewritten, or are a record of envelope rewriting, or
716were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
717match any entries therein. It is a colon-sep list; expand the items
718separately and squash any empty ones.
d43cbe25 719Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 720
d7978c0f 721for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14 722 {
511a6c14
JH
723 BOOL include_header = TRUE;
724
fb6833e0 725 list = tblock ? tblock->remove_headers : NULL;
d7978c0f 726 for (int i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
727 {
728 if (list)
729 {
730 int sep = ':'; /* This is specified as a colon-separated list */
731 uschar *s, *ss;
8bc732e8 732 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 733 {
76146973
JH
734 int len;
735
736 if (i == 0)
8768d548 737 if (!(s = expand_string(s)) && !f.expand_string_forcedfail)
76146973
JH
738 {
739 errno = ERRNO_CHHEADER_FAIL;
740 return FALSE;
741 }
d4ff61d1 742 len = s ? Ustrlen(s) : 0;
258dfd01
PS
743 if (len && s[len-1] == '*') /* trailing glob */
744 {
745 if (strncmpic(h->text, s, len-1) == 0) break;
746 }
747 else
748 {
749 if (strncmpic(h->text, s, len) != 0) continue;
750 ss = h->text + len;
751 while (*ss == ' ' || *ss == '\t') ss++;
752 if (*ss == ':') break;
753 }
511a6c14 754 }
6b46ecc6 755 if (s) { include_header = FALSE; break; }
511a6c14 756 }
6b46ecc6 757 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
758 }
759
760 /* If this header is to be output, try to rewrite it if there are rewriting
761 rules. */
762
763 if (include_header)
764 {
fb6833e0 765 if (tblock && tblock->rewrite_rules)
511a6c14 766 {
f3ebb786 767 rmark reset_point = store_mark();
511a6c14
JH
768 header_line *hh;
769
6b46ecc6
JH
770 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
771 tblock->rewrite_existflags, FALSE)))
511a6c14 772 {
42055a33 773 if (!sendfn(tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
774 store_reset(reset_point);
775 continue; /* With the next header line */
776 }
777 }
778
779 /* Either no rewriting rules, or it didn't get rewritten */
780
42055a33 781 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
782 }
783
784 /* Header removed */
785
786 else
511a6c14 787 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
511a6c14
JH
788 }
789
790/* Add on any address-specific headers. If there are multiple addresses,
791they will all have the same headers in order to be batched. The headers
792are chained in reverse order of adding (so several addresses from the
793same alias might share some of them) but we want to output them in the
794opposite order. This is a bit tedious, but there shouldn't be very many
795of them. We just walk the list twice, reversing the pointers each time,
796but on the second time, write out the items.
797
798Headers added to an address by a router are guaranteed to end with a newline.
799*/
800
801if (addr)
802 {
d43cbe25 803 header_line *hprev = addr->prop.extra_headers;
d7978c0f
JH
804 header_line *hnext, * h;
805 for (int i = 0; i < 2; i++)
ff5aac2b 806 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
807 {
808 hnext = h->next;
809 h->next = hprev;
810 hprev = h;
811 if (i == 1)
812 {
42055a33 813 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
814 DEBUG(D_transport)
815 debug_printf("added header line(s):\n%s---\n", h->text);
816 }
817 }
511a6c14
JH
818 }
819
76146973
JH
820/* If a string containing additional headers exists it is a newline-sep
821list. Expand each item and write out the result. This is done last so that
822if it (deliberately or accidentally) isn't in header format, it won't mess
823up any other headers. An empty string or a forced expansion failure are
824noops. An added header string from a transport may not end with a newline;
825add one if it does not. */
511a6c14 826
fb6833e0 827if (tblock && (list = CUS tblock->add_headers))
511a6c14 828 {
76146973
JH
829 int sep = '\n';
830 uschar * s;
831
6b46ecc6 832 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 833 if ((s = expand_string(s)))
76146973
JH
834 {
835 int len = Ustrlen(s);
836 if (len > 0)
511a6c14 837 {
42055a33
JH
838 if (!sendfn(tctx, s, len)) return FALSE;
839 if (s[len-1] != '\n' && !sendfn(tctx, US"\n", 1))
76146973
JH
840 return FALSE;
841 DEBUG(D_transport)
842 {
843 debug_printf("added header line:\n%s", s);
844 if (s[len-1] != '\n') debug_printf("\n");
845 debug_printf("---\n");
846 }
511a6c14
JH
847 }
848 }
8768d548 849 else if (!f.expand_string_forcedfail)
ff5aac2b 850 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
851 }
852
853/* Separate headers from body with a blank line */
854
42055a33 855return sendfn(tctx, US"\n", 1);
511a6c14
JH
856}
857
858
059ec3d9
PH
859/*************************************************
860* Write the message *
861*************************************************/
862
863/* This function writes the message to the given file descriptor. The headers
864are in the in-store data structure, and the rest of the message is in the open
865file descriptor deliver_datafile. Make sure we start it at the beginning.
866
867. If add_return_path is TRUE, a "return-path:" header is added to the message,
868 containing the envelope sender's address.
869
870. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
871 giving the top-level envelope address that caused this delivery to happen.
872
873. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
874 message. It gives the time and date that delivery took place.
875
876. If check_string is not null, the start of each line is checked for that
877 string. If it is found, it is replaced by escape_string. This used to be
878 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
879
880. If use_crlf is true, newlines are turned into CRLF (SMTP output).
881
882The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
883any writing or reading error, leaving the code in errno intact. Error exits
884can include timeouts for certain transports, which are requested by setting
885transport_write_timeout non-zero.
886
887Arguments:
6b46ecc6 888 tctx
42055a33
JH
889 (fd, msg) Either and fd, to write the message to,
890 or a string: if null write message to allocated space
891 otherwire take content as headers.
6b46ecc6
JH
892 addr (chain of) addresses (for extra headers), or NULL;
893 only the first address is used
894 tblock optional transport instance block (NULL signifies NULL/0):
895 add_headers a string containing one or more headers to add; it is
896 expanded, and must be in correct RFC 822 format as
897 it is transmitted verbatim; NULL => no additions,
898 and so does empty string or forced expansion fail
899 remove_headers a colon-separated list of headers to remove, or NULL
900 rewrite_rules chain of header rewriting rules
901 rewrite_existflags flags for the rewriting rules
902 options bit-wise options:
903 add_return_path if TRUE, add a "return-path" header
904 add_envelope_to if TRUE, add a "envelope-to" header
905 add_delivery_date if TRUE, add a "delivery-date" header
906 use_crlf if TRUE, turn NL into CR LF
907 end_dot if TRUE, send a terminating "." line at the end
908 no_headers if TRUE, omit the headers
909 no_body if TRUE, omit the body
2d14f397
JH
910 check_string a string to check for at the start of lines, or NULL
911 escape_string a string to insert in front of any check string
912 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
913 it is used when returning messages to their senders,
914 and is approximate rather than exact, owing to chunk
915 buffering
059ec3d9
PH
916
917Returns: TRUE on success; FALSE (with errno) on failure.
918 In addition, the global variable transport_count
919 is incremented by the number of bytes written.
920*/
921
d315eda1 922static BOOL
42055a33 923internal_transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 924{
328c5688 925int len, size = 0;
059ec3d9
PH
926
927/* Initialize pointer in output buffer. */
928
42055a33 929transport_write_reset(tctx->options);
059ec3d9
PH
930
931/* Set up the data for start-of-line data checking and escaping */
932
6b46ecc6 933if (tctx->check_string && tctx->escape_string)
059ec3d9 934 {
6b46ecc6 935 nl_check = tctx->check_string;
059ec3d9 936 nl_check_length = Ustrlen(nl_check);
6b46ecc6 937 nl_escape = tctx->escape_string;
059ec3d9
PH
938 nl_escape_length = Ustrlen(nl_escape);
939 }
059ec3d9 940
328c5688
JH
941/* Whether the escaping mechanism is applied to headers or not is controlled by
942an option (set for SMTP, not otherwise). Negate the length if not wanted till
943after the headers. */
944
945if (!(tctx->options & topt_escape_headers))
946 nl_check_length = -nl_check_length;
947
059ec3d9 948/* Write the headers if required, including any that have to be added. If there
328c5688
JH
949are header rewriting rules, apply them. The datasource is not the -D spoolfile
950so temporarily hide the global that adjusts for its format. */
059ec3d9 951
6b46ecc6 952if (!(tctx->options & topt_no_headers))
059ec3d9 953 {
8768d548
JH
954 BOOL save_wireformat = f.spool_file_wireformat;
955 f.spool_file_wireformat = FALSE;
42055a33 956
059ec3d9
PH
957 /* Add return-path: if requested. */
958
6b46ecc6 959 if (tctx->options & topt_add_return_path)
059ec3d9
PH
960 {
961 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 962 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 963 return_path);
328c5688 964 if (!write_chunk(tctx, buffer, n)) goto bad;
059ec3d9
PH
965 }
966
967 /* Add envelope-to: if requested */
968
6b46ecc6 969 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
970 {
971 BOOL first = TRUE;
059ec3d9
PH
972 struct aci *plist = NULL;
973 struct aci *dlist = NULL;
f3ebb786 974 rmark reset_point = store_mark();
059ec3d9 975
328c5688 976 if (!write_chunk(tctx, US"Envelope-to: ", 13)) goto bad;
059ec3d9
PH
977
978 /* Pick up from all the addresses. The plist and dlist variables are
979 anchors for lists of addresses already handled; they have to be defined at
4c04137d 980 this level because write_env_to() calls itself recursively. */
059ec3d9 981
d7978c0f
JH
982 for (address_item * p = tctx->addr; p; p = p->next)
983 if (!write_env_to(p, &plist, &dlist, &first, tctx))
984 goto bad;
059ec3d9
PH
985
986 /* Add a final newline and reset the store used for tracking duplicates */
987
328c5688 988 if (!write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
989 store_reset(reset_point);
990 }
991
992 /* Add delivery-date: if requested. */
993
6b46ecc6 994 if (tctx->options & topt_add_delivery_date)
059ec3d9 995 {
a5ffa9b4
JH
996 uschar * s = tod_stamp(tod_full);
997
998 if ( !write_chunk(tctx, US"Delivery-date: ", 15)
999 || !write_chunk(tctx, s, Ustrlen(s))
1000 || !write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
1001 }
1002
1003 /* Then the message's headers. Don't write any that are flagged as "old";
1004 that means they were rewritten, or are a record of envelope rewriting, or
1005 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 1006 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 1007 addr is not NULL. */
6b46ecc6 1008
42055a33 1009 if (!transport_headers_send(tctx, &write_chunk))
328c5688
JH
1010 {
1011bad:
8768d548 1012 f.spool_file_wireformat = save_wireformat;
511a6c14 1013 return FALSE;
328c5688
JH
1014 }
1015
8768d548 1016 f.spool_file_wireformat = save_wireformat;
059ec3d9
PH
1017 }
1018
58fc5fb2
JH
1019/* When doing RFC3030 CHUNKING output, work out how much data would be in a
1020last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
1021(optimally, all of the headers - but it does not matter if we already had to
1022flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 1023(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
1024that further use of write_chunk() will not prepend BDATs.
1025The first BDAT written will also first flush any outstanding MAIL and RCPT
1026commands which were buffered thans to PIPELINING.
1027Commands go out (using a send()) from a different buffer to data (using a
1028write()). They might not end up in the same TCP segment, which is
1029suboptimal. */
59932f7d 1030
65de12cc 1031if (tctx->options & topt_use_bdat)
59932f7d 1032 {
e027f545 1033 off_t fsize;
328c5688 1034 int hsize;
e027f545
JH
1035
1036 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
1037 hsize = 0;
65de12cc 1038 if (!(tctx->options & topt_no_body))
59932f7d
JH
1039 {
1040 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
1041 fsize -= SPOOL_DATA_START_OFFSET;
1042 if (size_limit > 0 && fsize > size_limit)
1043 fsize = size_limit;
e027f545 1044 size = hsize + fsize;
8768d548 1045 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
59932f7d 1046 size += body_linecount; /* account for CRLF-expansion */
42055a33
JH
1047
1048 /* With topt_use_bdat we never do dot-stuffing; no need to
1049 account for any expansion due to that. */
59932f7d
JH
1050 }
1051
e027f545
JH
1052 /* If the message is large, emit first a non-LAST chunk with just the
1053 headers, and reap the command responses. This lets us error out early
1054 on RCPT rejects rather than sending megabytes of data. Include headers
1055 on the assumption they are cheap enough and some clever implementations
1056 might errorcheck them too, on-the-fly, and reject that chunk. */
1057
1058 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
1059 {
58fc5fb2 1060 DEBUG(D_transport)
4c04137d 1061 debug_printf("sending small initial BDAT; hsize=%d\n", hsize);
d2aa036b 1062 if ( tctx->chunk_cb(tctx, hsize, 0) != OK
42055a33 1063 || !transport_write_block(tctx, deliver_out_buffer, hsize, FALSE)
d2aa036b 1064 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
e027f545
JH
1065 )
1066 return FALSE;
1067 chunk_ptr = deliver_out_buffer;
1068 size -= hsize;
1069 }
1070
d2aa036b
JH
1071 /* Emit a LAST datachunk command, and unmark the context for further
1072 BDAT commands. */
59932f7d 1073
d2aa036b 1074 if (tctx->chunk_cb(tctx, size, tc_chunk_last) != OK)
511a6c14 1075 return FALSE;
6d5c916c 1076 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
1077 }
1078
1079/* If the body is required, ensure that the data for check strings (formerly
1080the "from hack") is enabled by negating the length if necessary. (It will be
1081negative in cases where it isn't to apply to the headers). Then ensure the body
1082is positioned at the start of its file (following the message id), then write
1083it, applying the size limit if required. */
1084
328c5688
JH
1085/* If we have a wireformat -D file (CRNL lines, non-dotstuffed, no ending dot)
1086and we want to send a body without dotstuffing or ending-dot, in-clear,
1087then we can just dump it using sendfile.
1088This should get used for CHUNKING output and also for writing the -K file for
1089dkim signing, when we had CHUNKING input. */
1090
7d758a6a 1091#ifdef OS_SENDFILE
8768d548 1092if ( f.spool_file_wireformat
328c5688
JH
1093 && !(tctx->options & (topt_no_body | topt_end_dot))
1094 && !nl_check_length
74f1a423 1095 && tls_out.active.sock != tctx->u.fd
328c5688
JH
1096 )
1097 {
1098 ssize_t copied = 0;
1099 off_t offset = SPOOL_DATA_START_OFFSET;
1100
1101 /* Write out any header data in the buffer */
1102
1103 if ((len = chunk_ptr - deliver_out_buffer) > 0)
1104 {
1105 if (!transport_write_block(tctx, deliver_out_buffer, len, TRUE))
1106 return FALSE;
1107 size -= len;
1108 }
1109
1110 DEBUG(D_transport) debug_printf("using sendfile for body\n");
1111
1112 while(size > 0)
1113 {
7d758a6a 1114 if ((copied = os_sendfile(tctx->u.fd, deliver_datafile, &offset, size)) <= 0) break;
328c5688
JH
1115 size -= copied;
1116 }
1117 return copied >= 0;
1118 }
1119#else
1120DEBUG(D_transport) debug_printf("cannot use sendfile for body: no support\n");
1121#endif
1122
1123DEBUG(D_transport)
1124 if (!(tctx->options & topt_no_body))
1125 debug_printf("cannot use sendfile for body: %s\n",
8768d548 1126 !f.spool_file_wireformat ? "spoolfile not wireformat"
328c5688
JH
1127 : tctx->options & topt_end_dot ? "terminating dot wanted"
1128 : nl_check_length ? "dot- or From-stuffing wanted"
1129 : "TLS output wanted");
1130
6b46ecc6 1131if (!(tctx->options & topt_no_body))
059ec3d9 1132 {
05bf16f6 1133 unsigned long size = size_limit > 0 ? size_limit : ULONG_MAX;
e027f545 1134
059ec3d9
PH
1135 nl_check_length = abs(nl_check_length);
1136 nl_partial_match = 0;
d4ff61d1
JH
1137 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1138 return FALSE;
05bf16f6 1139 while ( (len = MIN(DELIVER_IN_BUFFER_SIZE, size)) > 0
59932f7d 1140 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1141 {
42055a33 1142 if (!write_chunk(tctx, deliver_in_buffer, len))
59932f7d 1143 return FALSE;
e027f545 1144 size -= len;
059ec3d9
PH
1145 }
1146
059ec3d9
PH
1147 /* A read error on the body will have left len == -1 and errno set. */
1148
1149 if (len != 0) return FALSE;
c0940526 1150 }
059ec3d9 1151
f64e8b5f 1152/* Finished with the check string, and spool-format consideration */
059ec3d9 1153
c0940526 1154nl_check_length = nl_escape_length = 0;
8768d548 1155f.spool_file_wireformat = FALSE;
c0940526
PP
1156
1157/* If requested, add a terminating "." line (SMTP output). */
1158
42055a33 1159if (tctx->options & topt_end_dot && !write_chunk(tctx, US".\n", 2))
c0940526 1160 return FALSE;
059ec3d9
PH
1161
1162/* Write out any remaining data in the buffer before returning. */
1163
1164return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
42055a33 1165 transport_write_block(tctx, deliver_out_buffer, len, FALSE);
f7572e5a 1166}
80a47a2c 1167
f7572e5a
TK
1168
1169
328c5688 1170
059ec3d9
PH
1171/*************************************************
1172* External interface to write the message *
1173*************************************************/
1174
1175/* If there is no filtering required, call the internal function above to do
1176the real work, passing over all the arguments from this function. Otherwise,
1177set up a filtering process, fork another process to call the internal function
1178to write to the filter, and in this process just suck from the filter and write
42055a33
JH
1179down the fd in the transport context. At the end, tidy up the pipes and the
1180processes.
059ec3d9
PH
1181
1182Arguments: as for internal_transport_write_message() above
1183
1184Returns: TRUE on success; FALSE (with errno) for any failure
1185 transport_count is incremented by the number of bytes written
1186*/
1187
1188BOOL
42055a33 1189transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 1190{
059ec3d9 1191BOOL last_filter_was_NL = TRUE;
8768d548 1192BOOL save_spool_file_wireformat = f.spool_file_wireformat;
96c81511
JH
1193BOOL yield;
1194int rc, len, fd_read, fd_write, save_errno;
806c3df9 1195int pfd[2] = {-1, -1};
059ec3d9 1196pid_t filter_pid, write_pid;
6b46ecc6 1197
8768d548 1198f.transport_filter_timed_out = FALSE;
2e2a30b4 1199
059ec3d9
PH
1200/* If there is no filter command set up, call the internal function that does
1201the actual work, passing it the incoming fd, and return its result. */
1202
6ff55e50
JH
1203if ( !transport_filter_argv
1204 || !*transport_filter_argv
1205 || !**transport_filter_argv
1206 )
42055a33 1207 return internal_transport_write_message(tctx, size_limit);
059ec3d9
PH
1208
1209/* Otherwise the message must be written to a filter process and read back
1210before being written to the incoming fd. First set up the special processing to
1211be done during the copying. */
1212
059ec3d9
PH
1213nl_partial_match = -1;
1214
6b46ecc6 1215if (tctx->check_string && tctx->escape_string)
059ec3d9 1216 {
6b46ecc6 1217 nl_check = tctx->check_string;
059ec3d9 1218 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1219 nl_escape = tctx->escape_string;
059ec3d9
PH
1220 nl_escape_length = Ustrlen(nl_escape);
1221 }
1222else nl_check_length = nl_escape_length = 0;
1223
1224/* Start up a subprocess to run the command. Ensure that our main fd will
1225be closed when the subprocess execs, but remove the flag afterwards.
1226(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1227process to deliver another message.) We get back stdin/stdout file descriptors.
1228If the process creation failed, give an error return. */
1229
1230fd_read = -1;
1231fd_write = -1;
1232save_errno = 0;
1233yield = FALSE;
1234write_pid = (pid_t)(-1);
1235
92b0827a 1236 {
42055a33 1237 int bits = fcntl(tctx->u.fd, F_GETFD);
eb24befc 1238 (void) fcntl(tctx->u.fd, F_SETFD, bits | FD_CLOEXEC);
92b0827a 1239 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
eb24befc
JH
1240 &fd_write, &fd_read, FALSE, US"transport-filter");
1241 (void) fcntl(tctx->u.fd, F_SETFD, bits & ~FD_CLOEXEC);
92b0827a 1242 }
059ec3d9
PH
1243if (filter_pid < 0) goto TIDY_UP; /* errno set */
1244
1245DEBUG(D_transport)
fb6833e0 1246 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1247 (int)filter_pid, fd_write, fd_read);
1248
1249/* Fork subprocess to write the message to the filter, and return the result
1250via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1251smtp dots, or check string processing. */
1252
1253if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
4b01271f 1254if ((write_pid = exim_fork(US"tpt-filter-writer")) == 0)
059ec3d9
PH
1255 {
1256 BOOL rc;
f1e894f3
PH
1257 (void)close(fd_read);
1258 (void)close(pfd[pipe_read]);
059ec3d9 1259 nl_check_length = nl_escape_length = 0;
6b46ecc6 1260
42055a33 1261 tctx->u.fd = fd_write;
fb6833e0 1262 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1263 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6 1264
42055a33 1265 rc = internal_transport_write_message(tctx, size_limit);
6b46ecc6 1266
059ec3d9 1267 save_errno = errno;
847a015a
JH
1268 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1269 != sizeof(BOOL)
1270 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1271 != sizeof(int)
1272 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1273 != sizeof(int)
a55697ac
JH
1274 || write(pfd[pipe_write], (void *)&tctx->addr->delivery_time, sizeof(struct timeval))
1275 != sizeof(struct timeval)
847a015a
JH
1276 )
1277 rc = FALSE; /* compiler quietening */
81022793 1278 exim_underbar_exit(EXIT_SUCCESS);
059ec3d9
PH
1279 }
1280save_errno = errno;
1281
1282/* Parent process: close our copy of the writing subprocess' pipes. */
1283
f1e894f3
PH
1284(void)close(pfd[pipe_write]);
1285(void)close(fd_write);
059ec3d9
PH
1286fd_write = -1;
1287
1288/* Writing process creation failed */
1289
1290if (write_pid < 0)
1291 {
1292 errno = save_errno; /* restore */
1293 goto TIDY_UP;
1294 }
1295
1296/* When testing, let the subprocess get going */
1297
9f01e50d 1298testharness_pause_ms(250);
059ec3d9
PH
1299
1300DEBUG(D_transport)
1301 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1302
1303/* Copy the message from the filter to the output fd. A read error leaves len
1304== -1 and errno set. We need to apply a timeout to the read, to cope with
1305the case when the filter gets stuck, but it can be quite a long one. The
1306default is 5m, but this is now configurable. */
1307
1308DEBUG(D_transport) debug_printf("copying from the filter\n");
1309
1310/* Copy the output of the filter, remembering if the last character was NL. If
1311no data is returned, that counts as "ended with NL" (default setting of the
328c5688
JH
1312variable is TRUE). The output should always be unix-format as we converted
1313any wireformat source on writing input to the filter. */
059ec3d9 1314
8768d548 1315f.spool_file_wireformat = FALSE;
059ec3d9
PH
1316chunk_ptr = deliver_out_buffer;
1317
1318for (;;)
1319 {
1320 sigalrm_seen = FALSE;
c2a1bba0 1321 ALARM(transport_filter_timeout);
059ec3d9 1322 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
c2a1bba0 1323 ALARM_CLR(0);
059ec3d9
PH
1324 if (sigalrm_seen)
1325 {
96c81511 1326 DEBUG(D_transport) debug_printf("timed out reading from filter\n");
059ec3d9 1327 errno = ETIMEDOUT;
8768d548 1328 f.transport_filter_timed_out = TRUE;
059ec3d9
PH
1329 goto TIDY_UP;
1330 }
1331
1332 /* If the read was successful, write the block down the original fd,
1333 remembering whether it ends in \n or not. */
1334
1335 if (len > 0)
1336 {
42055a33 1337 if (!write_chunk(tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1338 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1339 }
1340
1341 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1342
1343 else
1344 {
1345 if (len == 0) yield = TRUE;
1346 break;
1347 }
1348 }
1349
1350/* Tidying up code. If yield = FALSE there has been an error and errno is set
1351to something. Ensure the pipes are all closed and the processes are removed. If
1352there has been an error, kill the processes before waiting for them, just to be
1353sure. Also apply a paranoia timeout. */
1354
1355TIDY_UP:
8768d548 1356f.spool_file_wireformat = save_spool_file_wireformat;
059ec3d9
PH
1357save_errno = errno;
1358
f1e894f3
PH
1359(void)close(fd_read);
1360if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1361
1362if (!yield)
1363 {
1364 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1365 if (write_pid > 0) kill(write_pid, SIGKILL);
1366 }
1367
1368/* Wait for the filter process to complete. */
1369
1370DEBUG(D_transport) debug_printf("waiting for filter process\n");
1371if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1372 {
1373 yield = FALSE;
1374 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1375 tctx->addr->more_errno = rc;
059ec3d9
PH
1376 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1377 }
1378
1379/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1380read the results from its pipe, provided we haven't already had a filter
35af9f61 1381process failure. */
059ec3d9
PH
1382
1383DEBUG(D_transport) debug_printf("waiting for writing process\n");
1384if (write_pid > 0)
1385 {
35af9f61
PH
1386 rc = child_close(write_pid, 30);
1387 if (yield)
8e669ac1 1388 if (rc == 0)
35af9f61
PH
1389 {
1390 BOOL ok;
847a015a 1391 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
4dc2379a
JH
1392 {
1393 DEBUG(D_transport)
1394 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1395 save_errno = ERRNO_FILTER_FAIL;
1396 yield = FALSE;
1397 }
1398 else if (!ok)
35af9f61 1399 {
847a015a
JH
1400 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
1401 dummy = read(pfd[pipe_read], (void *)&tctx->addr->more_errno, sizeof(int));
a55697ac 1402 dummy = read(pfd[pipe_read], (void *)&tctx->addr->delivery_time, sizeof(struct timeval));
cab0c277 1403 dummy = dummy; /* compiler quietening */
35af9f61
PH
1404 yield = FALSE;
1405 }
1406 }
1407 else
059ec3d9 1408 {
059ec3d9 1409 yield = FALSE;
35af9f61 1410 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1411 tctx->addr->more_errno = rc;
35af9f61 1412 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1413 }
059ec3d9 1414 }
f1e894f3 1415(void)close(pfd[pipe_read]);
059ec3d9
PH
1416
1417/* If there have been no problems we can now add the terminating "." if this is
1418SMTP output, turning off escaping beforehand. If the last character from the
1419filter was not NL, insert a NL to make the SMTP protocol work. */
1420
1421if (yield)
1422 {
1423 nl_check_length = nl_escape_length = 0;
8768d548 1424 f.spool_file_wireformat = FALSE;
6b46ecc6 1425 if ( tctx->options & topt_end_dot
ff5aac2b 1426 && ( last_filter_was_NL
42055a33
JH
1427 ? !write_chunk(tctx, US".\n", 2)
1428 : !write_chunk(tctx, US"\n.\n", 3)
ff5aac2b 1429 ) )
059ec3d9 1430 yield = FALSE;
059ec3d9
PH
1431
1432 /* Write out any remaining data in the buffer. */
1433
1434 else
ff5aac2b 1435 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
42055a33 1436 || transport_write_block(tctx, deliver_out_buffer, len, FALSE);
059ec3d9 1437 }
ff5aac2b
JH
1438else
1439 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1440
1441DEBUG(D_transport)
1442 {
1443 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1444 if (!yield)
96c81511 1445 debug_printf(" errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1446 }
1447
1448return yield;
1449}
1450
1451
1452
1453
1454
1455/*************************************************
1456* Update waiting database *
1457*************************************************/
1458
1459/* This is called when an address is deferred by remote transports that are
1460capable of sending more than one message over one connection. A database is
1461maintained for each transport, keeping track of which messages are waiting for
1462which hosts. The transport can then consult this when eventually a successful
1463delivery happens, and if it finds that another message is waiting for the same
1464host, it can fire up a new process to deal with it using the same connection.
1465
1466The database records are keyed by host name. They can get full if there are
1467lots of messages waiting, and so there is a continuation mechanism for them.
1468
1469Each record contains a list of message ids, packed end to end without any
1470zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1471in this record, and the sequence field says if there are any other records for
1472this host. If the sequence field is 0, there are none. If it is 1, then another
1473record with the name <hostname>:0 exists; if it is 2, then two other records
1474with sequence numbers 0 and 1 exist, and so on.
1475
1476Currently, an exhaustive search of all continuation records has to be done to
1477determine whether to add a message id to a given record. This shouldn't be
1478too bad except in extreme cases. I can't figure out a *simple* way of doing
1479better.
1480
1481Old records should eventually get swept up by the exim_tidydb utility.
1482
1483Arguments:
f6c332bd 1484 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1485 tpname name of the transport
1486
1487Returns: nothing
1488*/
1489
1490void
1491transport_update_waiting(host_item *hostlist, uschar *tpname)
1492{
55414b25 1493const uschar *prevname = US"";
059ec3d9
PH
1494open_db dbblock;
1495open_db *dbm_file;
1496
7a0743eb
PH
1497DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1498
059ec3d9
PH
1499/* Open the database for this transport */
1500
36d295f1 1501if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
b10c87b3 1502 O_RDWR, &dbblock, TRUE, TRUE)))
36d295f1 1503 return;
059ec3d9
PH
1504
1505/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1506that the message id is in each host record. */
059ec3d9 1507
d7978c0f 1508for (host_item * host = hostlist; host; host = host->next)
059ec3d9
PH
1509 {
1510 BOOL already = FALSE;
1511 dbdata_wait *host_record;
d7978c0f 1512 int host_length;
36d295f1 1513 uschar buffer[256];
059ec3d9 1514
059ec3d9
PH
1515 /* Skip if this is the same host as we just processed; otherwise remember
1516 the name for next time. */
1517
1518 if (Ustrcmp(prevname, host->name) == 0) continue;
1519 prevname = host->name;
1520
1521 /* Look up the host record; if there isn't one, make an empty one. */
1522
36d295f1 1523 if (!(host_record = dbfn_read(dbm_file, host->name)))
059ec3d9 1524 {
f3ebb786 1525 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH, FALSE);
059ec3d9
PH
1526 host_record->count = host_record->sequence = 0;
1527 }
1528
1529 /* Compute the current length */
1530
1531 host_length = host_record->count * MESSAGE_ID_LENGTH;
1532
1533 /* Search the record to see if the current message is already in it. */
1534
d7978c0f 1535 for (uschar * s = host_record->text; s < host_record->text + host_length;
059ec3d9 1536 s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1537 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1538 { already = TRUE; break; }
059ec3d9
PH
1539
1540 /* If we haven't found this message in the main record, search any
1541 continuation records that exist. */
1542
d7978c0f 1543 for (int i = host_record->sequence - 1; i >= 0 && !already; i--)
059ec3d9
PH
1544 {
1545 dbdata_wait *cont;
1546 sprintf(CS buffer, "%.200s:%d", host->name, i);
36d295f1 1547 if ((cont = dbfn_read(dbm_file, buffer)))
059ec3d9
PH
1548 {
1549 int clen = cont->count * MESSAGE_ID_LENGTH;
d7978c0f 1550 for (uschar * s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1551 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1552 { already = TRUE; break; }
059ec3d9
PH
1553 }
1554 }
1555
1556 /* If this message is already in a record, no need to update. */
1557
7a0743eb
PH
1558 if (already)
1559 {
1560 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1561 continue;
1562 }
059ec3d9
PH
1563
1564
1565 /* If this record is full, write it out with a new name constructed
1566 from the sequence number, increase the sequence number, and empty
ff966302
JH
1567 the record. If we're doing a two-phase queue run initial phase, ping the
1568 daemon to consider running a delivery on this host. */
059ec3d9
PH
1569
1570 if (host_record->count >= WAIT_NAME_MAX)
1571 {
1572 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1573 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
ff966302
JH
1574#ifdef EXPERIMENTAL_QUEUE_RAMP
1575 if (f.queue_2stage && queue_fast_ramp && !queue_run_in_order)
1576 queue_notify_daemon(message_id);
1577#endif
059ec3d9
PH
1578 host_record->sequence++;
1579 host_record->count = 0;
1580 host_length = 0;
1581 }
1582
1583 /* If this record is not full, increase the size of the record to
1584 allow for one new message id. */
1585
1586 else
1587 {
1588 dbdata_wait *newr =
f3ebb786 1589 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH, FALSE);
059ec3d9
PH
1590 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1591 host_record = newr;
1592 }
1593
1594 /* Now add the new name on the end */
1595
1596 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1597 host_record->count++;
1598 host_length += MESSAGE_ID_LENGTH;
1599
1600 /* Update the database */
1601
1602 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1603 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1604 }
1605
1606/* All now done */
1607
1608dbfn_close(dbm_file);
1609}
1610
1611
1612
1613
1614/*************************************************
1615* Test for waiting messages *
1616*************************************************/
1617
1618/* This function is called by a remote transport which uses the previous
1619function to remember which messages are waiting for which remote hosts. It's
1620called after a successful delivery and its job is to check whether there is
1621another message waiting for the same host. However, it doesn't do this if the
1622current continue sequence is greater than the maximum supplied as an argument,
1623or greater than the global connection_max_messages, which, if set, overrides.
1624
1625Arguments:
1626 transport_name name of the transport
1627 hostname name of the host
1628 local_message_max maximum number of messages down one connection
1629 as set by the caller transport
1630 new_message_id set to the message id of a waiting message
1631 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1632 oicf_func function to call to validate if it is ok to send
1633 to this message_id from the current instance.
1634 oicf_data opaque data for oicf_func
059ec3d9
PH
1635
1636Returns: TRUE if new_message_id set; FALSE otherwise
1637*/
1638
a39bd74d
JB
1639typedef struct msgq_s
1640{
1641 uschar message_id [MESSAGE_ID_LENGTH + 1];
1642 BOOL bKeep;
1643} msgq_t;
1644
059ec3d9 1645BOOL
55414b25 1646transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1647 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1648{
1649dbdata_wait *host_record;
0539a19d 1650int host_length;
059ec3d9
PH
1651open_db dbblock;
1652open_db *dbm_file;
059ec3d9 1653
a39bd74d 1654int i;
a39bd74d 1655struct stat statbuf;
a39bd74d 1656
059ec3d9
PH
1657*more = FALSE;
1658
1659DEBUG(D_transport)
1660 {
1661 debug_printf("transport_check_waiting entered\n");
1662 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1663 continue_sequence, local_message_max, connection_max_messages);
1664 }
1665
1666/* Do nothing if we have hit the maximum number that can be send down one
1667connection. */
1668
1669if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1670if (local_message_max > 0 && continue_sequence >= local_message_max)
1671 {
1672 DEBUG(D_transport)
1673 debug_printf("max messages for one connection reached: returning\n");
1674 return FALSE;
1675 }
1676
1677/* Open the waiting information database. */
1678
36d295f1 1679if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
b10c87b3 1680 O_RDWR, &dbblock, TRUE, TRUE)))
36d295f1 1681 return FALSE;
059ec3d9
PH
1682
1683/* See if there is a record for this host; if not, there's nothing to do. */
1684
789f8a4f 1685if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1686 {
1687 dbfn_close(dbm_file);
1688 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1689 return FALSE;
1690 }
1691
1692/* If the data in the record looks corrupt, just log something and
1693don't try to use it. */
1694
1695if (host_record->count > WAIT_NAME_MAX)
1696 {
1697 dbfn_close(dbm_file);
1698 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1699 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1700 return FALSE;
1701 }
1702
1703/* Scan the message ids in the record from the end towards the beginning,
1704until one is found for which a spool file actually exists. If the record gets
1705emptied, delete it and continue with any continuation records that may exist.
1706*/
1707
a39bd74d
JB
1708/* For Bug 1141, I refactored this major portion of the routine, it is risky
1709but the 1 off will remain without it. This code now allows me to SKIP over
1710a message I do not want to send out on this run. */
059ec3d9 1711
a39bd74d
JB
1712host_length = host_record->count * MESSAGE_ID_LENGTH;
1713
1714while (1)
059ec3d9 1715 {
789f8a4f
JH
1716 msgq_t *msgq;
1717 int msgq_count = 0;
1718 int msgq_actual = 0;
1719 BOOL bFound = FALSE;
1720 BOOL bContinuation = FALSE;
1721
a39bd74d 1722 /* create an array to read entire message queue into memory for processing */
059ec3d9 1723
f3ebb786 1724 msgq = store_get(sizeof(msgq_t) * host_record->count, FALSE);
a39bd74d
JB
1725 msgq_count = host_record->count;
1726 msgq_actual = msgq_count;
059ec3d9 1727
a39bd74d 1728 for (i = 0; i < host_record->count; ++i)
059ec3d9 1729 {
a39bd74d
JB
1730 msgq[i].bKeep = TRUE;
1731
f3ebb786 1732 Ustrncpy_nt(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1733 MESSAGE_ID_LENGTH);
a39bd74d
JB
1734 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1735 }
1736
1737 /* first thing remove current message id if it exists */
96c81511 1738 /*XXX but what if it has un-sent addrs? */
059ec3d9 1739
a39bd74d
JB
1740 for (i = 0; i < msgq_count; ++i)
1741 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1742 {
1743 msgq[i].bKeep = FALSE;
1744 break;
1745 }
1746
1747 /* now find the next acceptable message_id */
1748
a39bd74d
JB
1749 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1750 {
41313d92 1751 uschar subdir[2];
59a93276 1752 uschar * mid = msgq[i].message_id;
41313d92 1753
59a93276
JH
1754 set_subdir_str(subdir, mid, 0);
1755 if (Ustat(spool_fname(US"input", subdir, mid, US"-D"), &statbuf) != 0)
a39bd74d 1756 msgq[i].bKeep = FALSE;
59a93276 1757 else if (!oicf_func || oicf_func(mid, oicf_data))
059ec3d9 1758 {
59a93276 1759 Ustrcpy_nt(new_message_id, mid);
a39bd74d
JB
1760 msgq[i].bKeep = FALSE;
1761 bFound = TRUE;
059ec3d9
PH
1762 break;
1763 }
1764 }
1765
a39bd74d
JB
1766 /* re-count */
1767 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1768 if (msgq[i].bKeep)
1769 msgq_actual++;
1770
1771 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1772 memory queue */
a39bd74d
JB
1773
1774 if (msgq_actual <= 0)
1775 {
1776 host_length = 0;
1777 host_record->count = 0;
1778 }
1779 else
1780 {
1781 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1782 host_record->count = msgq_actual;
1783
1784 if (msgq_actual < msgq_count)
1785 {
1786 int new_count;
1787 for (new_count = 0, i = 0; i < msgq_count; ++i)
1788 if (msgq[i].bKeep)
1789 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1790 msgq[i].message_id, MESSAGE_ID_LENGTH);
1791
1792 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1793 }
1794 }
1795
36d295f1 1796 /* Check for a continuation record. */
a39bd74d 1797
059ec3d9
PH
1798 while (host_length <= 0)
1799 {
a39bd74d 1800 dbdata_wait * newr = NULL;
36d295f1 1801 uschar buffer[256];
059ec3d9
PH
1802
1803 /* Search for a continuation */
1804
d7978c0f 1805 for (int i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1806 {
1807 sprintf(CS buffer, "%.200s:%d", hostname, i);
1808 newr = dbfn_read(dbm_file, buffer);
1809 }
1810
1811 /* If no continuation, delete the current and break the loop */
1812
a39bd74d 1813 if (!newr)
059ec3d9
PH
1814 {
1815 dbfn_delete(dbm_file, hostname);
1816 break;
1817 }
1818
1819 /* Else replace the current with the continuation */
1820
1821 dbfn_delete(dbm_file, buffer);
1822 host_record = newr;
1823 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1824
a39bd74d
JB
1825 bContinuation = TRUE;
1826 }
059ec3d9 1827
789f8a4f 1828 if (bFound) /* Usual exit from main loop */
a39bd74d 1829 break;
059ec3d9
PH
1830
1831 /* If host_length <= 0 we have emptied a record and not found a good message,
1832 and there are no continuation records. Otherwise there is a continuation
1833 record to process. */
1834
1835 if (host_length <= 0)
1836 {
1837 dbfn_close(dbm_file);
1838 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1839 return FALSE;
1840 }
a39bd74d
JB
1841
1842 /* we were not able to find an acceptable message, nor was there a
1843 * continuation record. So bug out, outer logic will clean this up.
1844 */
1845
1846 if (!bContinuation)
1847 {
789f8a4f 1848 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1849 dbfn_close(dbm_file);
1850 return FALSE;
1851 }
789f8a4f 1852 } /* we need to process a continuation record */
059ec3d9
PH
1853
1854/* Control gets here when an existing message has been encountered; its
1855id is in new_message_id, and host_length is the revised length of the
1856host record. If it is zero, the record has been removed. Update the
1857record if required, close the database, and return TRUE. */
1858
1859if (host_length > 0)
1860 {
1861 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1862
059ec3d9
PH
1863 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1864 *more = TRUE;
1865 }
1866
1867dbfn_close(dbm_file);
1868return TRUE;
1869}
1870
059ec3d9
PH
1871/*************************************************
1872* Deliver waiting message down same socket *
1873*************************************************/
1874
57cc2785
JH
1875/* Just the regain-root-privilege exec portion */
1876void
1877transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
1878 const uschar *hostaddress, uschar *id, int socket_fd)
1879{
57cc2785
JH
1880int i = 20;
1881const uschar **argv;
1882
1883/* Set up the calling arguments; use the standard function for the basics,
1884but we have a number of extras that may be added. */
1885
1886argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
1887
8768d548 1888if (f.smtp_authenticated) argv[i++] = US"-MCA";
14de8063
JH
1889if (smtp_peer_options & OPTION_CHUNKING) argv[i++] = US"-MCK";
1890if (smtp_peer_options & OPTION_DSN) argv[i++] = US"-MCD";
1891if (smtp_peer_options & OPTION_PIPE) argv[i++] = US"-MCP";
1892if (smtp_peer_options & OPTION_SIZE) argv[i++] = US"-MCS";
01603eec 1893#ifndef DISABLE_TLS
14de8063 1894if (smtp_peer_options & OPTION_TLS)
74f1a423 1895 if (tls_out.active.sock >= 0 || continue_proxy_cipher)
57cc2785
JH
1896 {
1897 argv[i++] = US"-MCt";
1898 argv[i++] = sending_ip_address;
1899 argv[i++] = string_sprintf("%d", sending_port);
74f1a423 1900 argv[i++] = tls_out.active.sock >= 0 ? tls_out.cipher : continue_proxy_cipher;
57cc2785
JH
1901 }
1902 else
1903 argv[i++] = US"-MCT";
1904#endif
1905
1906if (queue_run_pid != (pid_t)0)
1907 {
1908 argv[i++] = US"-MCQ";
1909 argv[i++] = string_sprintf("%d", queue_run_pid);
1910 argv[i++] = string_sprintf("%d", queue_run_pipe);
1911 }
1912
1913argv[i++] = US"-MC";
1914argv[i++] = US transport_name;
1915argv[i++] = US hostname;
1916argv[i++] = US hostaddress;
1917argv[i++] = string_sprintf("%d", continue_sequence + 1);
1918argv[i++] = id;
1919argv[i++] = NULL;
1920
1921/* Arrange for the channel to be on stdin. */
1922
1923if (socket_fd != 0)
1924 {
1925 (void)dup2(socket_fd, 0);
1926 (void)close(socket_fd);
1927 }
1928
1929DEBUG(D_exec) debug_print_argv(argv);
1930exim_nullstd(); /* Ensure std{out,err} exist */
1931execv(CS argv[0], (char *const *)argv);
1932
1933DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
1934_exit(errno); /* Note: must be _exit(), NOT exit() */
1935}
1936
1937
1938
059ec3d9
PH
1939/* Fork a new exim process to deliver the message, and do a re-exec, both to
1940get a clean delivery process, and to regain root privilege in cases where it
1941has been given away.
1942
1943Arguments:
1944 transport_name to pass to the new process
1945 hostname ditto
1946 hostaddress ditto
1947 id the new message to process
1948 socket_fd the connected socket
1949
1950Returns: FALSE if fork fails; TRUE otherwise
1951*/
1952
1953BOOL
55414b25
JH
1954transport_pass_socket(const uschar *transport_name, const uschar *hostname,
1955 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
1956{
1957pid_t pid;
1958int status;
1959
1960DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
1961
4b01271f 1962if ((pid = exim_fork(US"continued-transport-interproc")) == 0)
059ec3d9 1963 {
059ec3d9
PH
1964 /* Disconnect entirely from the parent process. If we are running in the
1965 test harness, wait for a bit to allow the previous process time to finish,
1966 write the log, etc., so that the output is always in the same order for
1967 automatic comparison. */
1968
8e9fdd63 1969 if ((pid = exim_fork(US"continued-transport")) != 0)
b7d3afcf 1970 _exit(EXIT_SUCCESS);
4b01271f 1971 testharness_pause_ms(1000);
059ec3d9 1972
57cc2785
JH
1973 transport_do_pass_socket(transport_name, hostname, hostaddress,
1974 id, socket_fd);
059ec3d9
PH
1975 }
1976
1977/* If the process creation succeeded, wait for the first-level child, which
1978immediately exits, leaving the second level process entirely disconnected from
1979this one. */
1980
1981if (pid > 0)
1982 {
1983 int rc;
1984 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
059ec3d9
PH
1985 return TRUE;
1986 }
1987else
1988 {
1989 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
1990 strerror(errno));
1991 return FALSE;
1992 }
1993}
1994
1995
1996
1997/*************************************************
1998* Set up direct (non-shell) command *
1999*************************************************/
2000
2001/* This function is called when a command line is to be parsed and executed
2002directly, without the use of /bin/sh. It is called by the pipe transport,
2003the queryprogram router, and also from the main delivery code when setting up a
2004transport filter process. The code for ETRN also makes use of this; in that
2005case, no addresses are passed.
2006
2007Arguments:
2008 argvptr pointer to anchor for argv vector
55414b25 2009 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
2010 expand_arguments true if expansion is to occur
2011 expand_failed error value to set if expansion fails; not relevant if
2012 addr == NULL
2013 addr chain of addresses, or NULL
2014 etext text for use in error messages
2015 errptr where to put error message if addr is NULL;
2016 otherwise it is put in the first address
2017
2018Returns: TRUE if all went well; otherwise an error will be
2019 set in the first address and FALSE returned
2020*/
2021
2022BOOL
55414b25
JH
2023transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2024 BOOL expand_arguments, int expand_failed, address_item *addr,
2025 uschar *etext, uschar **errptr)
059ec3d9 2026{
55414b25 2027const uschar **argv;
059ec3d9
PH
2028uschar *s, *ss;
2029int address_count = 0;
2030int argcount = 0;
d7978c0f 2031int max_args;
059ec3d9
PH
2032
2033/* Get store in which to build an argument list. Count the number of addresses
2034supplied, and allow for that many arguments, plus an additional 60, which
2035should be enough for anybody. Multiple addresses happen only when the local
2036delivery batch option is set. */
2037
d7978c0f 2038for (address_item * ad = addr; ad; ad = ad->next) address_count++;
059ec3d9 2039max_args = address_count + 60;
f3ebb786 2040*argvptr = argv = store_get((max_args+1)*sizeof(uschar *), FALSE);
059ec3d9
PH
2041
2042/* Split the command up into arguments terminated by white space. Lose
2043trailing space at the start and end. Double-quoted arguments can contain \\ and
2044\" escapes and so can be handled by the standard function; single-quoted
2045arguments are verbatim. Copy each argument into a new string. */
2046
2047s = cmd;
2048while (isspace(*s)) s++;
2049
f3ebb786 2050for (; *s != 0 && argcount < max_args; argcount++)
059ec3d9
PH
2051 {
2052 if (*s == '\'')
2053 {
2054 ss = s + 1;
2055 while (*ss != 0 && *ss != '\'') ss++;
f3ebb786 2056 argv[argcount] = ss = store_get(ss - s++, is_tainted(cmd));
059ec3d9
PH
2057 while (*s != 0 && *s != '\'') *ss++ = *s++;
2058 if (*s != 0) s++;
2059 *ss++ = 0;
2060 }
f3ebb786
JH
2061 else
2062 argv[argcount] = string_dequote(CUSS &s);
059ec3d9
PH
2063 while (isspace(*s)) s++;
2064 }
2065
5903c6ff 2066argv[argcount] = US 0;
059ec3d9
PH
2067
2068/* If *s != 0 we have run out of argument slots. */
2069
2070if (*s != 0)
2071 {
2072 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2073 "%s", cmd, etext);
2074 if (addr != NULL)
2075 {
2076 addr->transport_return = FAIL;
2077 addr->message = msg;
2078 }
2079 else *errptr = msg;
2080 return FALSE;
2081 }
2082
2083/* Expand each individual argument if required. Expansion happens for pipes set
2084up in filter files and with directly-supplied commands. It does not happen if
2085the pipe comes from a traditional .forward file. A failing expansion is a big
2086disaster if the command came from Exim's configuration; if it came from a user
2087it is just a normal failure. The expand_failed value is used as the error value
2088to cater for these two cases.
2089
2090An argument consisting just of the text "$pipe_addresses" is treated specially.
2091It is not passed to the general expansion function. Instead, it is replaced by
2092a number of arguments, one for each address. This avoids problems with shell
2093metacharacters and spaces in addresses.
2094
2095If the parent of the top address has an original part of "system-filter", this
2096pipe was set up by the system filter, and we can permit the expansion of
2097$recipients. */
2098
2099DEBUG(D_transport)
2100 {
2101 debug_printf("direct command:\n");
f3ebb786
JH
2102 for (int i = 0; argv[i]; i++)
2103 debug_printf(" argv[%d] = '%s'\n", i, string_printing(argv[i]));
059ec3d9
PH
2104 }
2105
2106if (expand_arguments)
2107 {
2108 BOOL allow_dollar_recipients = addr != NULL &&
2109 addr->parent != NULL &&
2110 Ustrcmp(addr->parent->address, "system-filter") == 0;
2111
d7978c0f 2112 for (int i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2113 {
2114
2115 /* Handle special fudge for passing an address list */
2116
2117 if (addr != NULL &&
2118 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2119 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2120 {
2121 int additional;
2122
2123 if (argcount + address_count - 1 > max_args)
2124 {
2125 addr->transport_return = FAIL;
2126 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2127 "in %s", cmd, etext);
2128 return FALSE;
2129 }
2130
2131 additional = address_count - 1;
2132 if (additional > 0)
2133 memmove(argv + i + 1 + additional, argv + i + 1,
2134 (argcount - i)*sizeof(uschar *));
2135
d7978c0f
JH
2136 for (address_item * ad = addr; ad; ad = ad->next)
2137 {
2138 argv[i++] = ad->address;
2139 argcount++;
2140 }
09792322 2141
700d22f3
PP
2142 /* Subtract one since we replace $pipe_addresses */
2143 argcount--;
2144 i--;
09792322
NK
2145 }
2146
2147 /* Handle special case of $address_pipe when af_force_command is set */
2148
2149 else if (addr != NULL && testflag(addr,af_force_command) &&
2150 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2151 Ustrcmp(argv[i], "${address_pipe}") == 0))
2152 {
09792322
NK
2153 int address_pipe_argcount = 0;
2154 int address_pipe_max_args;
2155 uschar **address_pipe_argv;
f3ebb786 2156 BOOL tainted;
09792322
NK
2157
2158 /* We can never have more then the argv we will be loading into */
2159 address_pipe_max_args = max_args - argcount + 1;
2160
2161 DEBUG(D_transport)
700d22f3 2162 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2163
2164 /* We allocate an additional for (uschar *)0 */
f3ebb786 2165 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *), FALSE);
09792322
NK
2166
2167 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2168 s = expand_string(addr->local_part + 1);
f3ebb786 2169 tainted = is_tainted(s);
09792322 2170
700d22f3
PP
2171 if (s == NULL || *s == '\0')
2172 {
2173 addr->transport_return = FAIL;
2174 addr->message = string_sprintf("Expansion of \"%s\" "
2175 "from command \"%s\" in %s failed: %s",
2176 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2177 return FALSE;
700d22f3 2178 }
09792322
NK
2179
2180 while (isspace(*s)) s++; /* strip leading space */
2181
2182 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2183 {
2184 if (*s == '\'')
2185 {
2186 ss = s + 1;
2187 while (*ss != 0 && *ss != '\'') ss++;
f3ebb786 2188 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++, tainted);
09792322
NK
2189 while (*s != 0 && *s != '\'') *ss++ = *s++;
2190 if (*s != 0) s++;
2191 *ss++ = 0;
2192 }
55414b25
JH
2193 else address_pipe_argv[address_pipe_argcount++] =
2194 string_copy(string_dequote(CUSS &s));
09792322
NK
2195 while (isspace(*s)) s++; /* strip space after arg */
2196 }
2197
5903c6ff 2198 address_pipe_argv[address_pipe_argcount] = US 0;
09792322
NK
2199
2200 /* If *s != 0 we have run out of argument slots. */
2201 if (*s != 0)
2202 {
2203 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2204 "\"%s\" in %s", addr->local_part + 1, etext);
2205 if (addr != NULL)
2206 {
2207 addr->transport_return = FAIL;
2208 addr->message = msg;
2209 }
2210 else *errptr = msg;
2211 return FALSE;
2212 }
2213
700d22f3
PP
2214 /* address_pipe_argcount - 1
2215 * because we are replacing $address_pipe in the argument list
2216 * with the first thing it expands to */
09792322
NK
2217 if (argcount + address_pipe_argcount - 1 > max_args)
2218 {
2219 addr->transport_return = FAIL;
2220 addr->message = string_sprintf("Too many arguments to command "
2221 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2222 return FALSE;
2223 }
2224
2225 /* If we are not just able to replace the slot that contained
2226 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2227 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2228 * Visually if address_pipe_argcount == 2:
2229 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2230 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2231 */
2232 if (address_pipe_argcount > 1)
2233 memmove(
4c04137d 2234 /* current position + additional args */
09792322
NK
2235 argv + i + address_pipe_argcount,
2236 /* current position + 1 (for the (uschar *)0 at the end) */
2237 argv + i + 1,
2238 /* -1 for the (uschar *)0 at the end)*/
2239 (argcount - i)*sizeof(uschar *)
2240 );
2241
2242 /* Now we fill in the slots we just moved argv out of
2243 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2244 */
d7978c0f 2245 for (int address_pipe_i = 0;
5903c6ff 2246 address_pipe_argv[address_pipe_i] != US 0;
700d22f3
PP
2247 address_pipe_i++)
2248 {
2249 argv[i++] = address_pipe_argv[address_pipe_i];
2250 argcount++;
09792322
NK
2251 }
2252
700d22f3
PP
2253 /* Subtract one since we replace $address_pipe */
2254 argcount--;
2255 i--;
059ec3d9
PH
2256 }
2257
2258 /* Handle normal expansion string */
2259
2260 else
2261 {
55414b25 2262 const uschar *expanded_arg;
8768d548 2263 f.enable_dollar_recipients = allow_dollar_recipients;
55414b25 2264 expanded_arg = expand_cstring(argv[i]);
8768d548 2265 f.enable_dollar_recipients = FALSE;
059ec3d9 2266
f3ebb786 2267 if (!expanded_arg)
059ec3d9
PH
2268 {
2269 uschar *msg = string_sprintf("Expansion of \"%s\" "
2270 "from command \"%s\" in %s failed: %s",
2271 argv[i], cmd, etext, expand_string_message);
f3ebb786 2272 if (addr)
059ec3d9
PH
2273 {
2274 addr->transport_return = expand_failed;
2275 addr->message = msg;
2276 }
2277 else *errptr = msg;
2278 return FALSE;
2279 }
2280 argv[i] = expanded_arg;
2281 }
2282 }
2283
2284 DEBUG(D_transport)
2285 {
2286 debug_printf("direct command after expansion:\n");
d7978c0f 2287 for (int i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2288 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2289 }
2290 }
2291
2292return TRUE;
2293}
2294
d185889f 2295#endif /*!MACRO_PREDEF*/
511a6c14
JH
2296/* vi: aw ai sw=2
2297*/
059ec3d9 2298/* End of transport.c */