tidying
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
80fea873 5/* Copyright (c) University of Cambridge 1995 - 2016 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* General functions concerned with transportation, and generic options for all
9transports. */
10
11
12#include "exim.h"
13
059ec3d9
PH
14/* Generic options for transports, all of which live inside transport_instance
15data blocks and which therefore have the opt_public flag set. Note that there
16are other options living inside this structure which can be set only from
17certain transports. */
18
19optionlist optionlist_transports[] = {
20 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
21 (void *)offsetof(transport_instance, expand_gid) },
22 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
23 (void *)offsetof(transport_instance, expand_uid) },
24 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
25 (void *)offsetof(transport_instance, rewrite_existflags) },
26 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
27 (void *)offsetof(transport_instance, rewrite_rules) },
28 { "*set_group", opt_bool|opt_hidden|opt_public,
29 (void *)offsetof(transport_instance, gid_set) },
30 { "*set_user", opt_bool|opt_hidden|opt_public,
31 (void *)offsetof(transport_instance, uid_set) },
32 { "body_only", opt_bool|opt_public,
33 (void *)offsetof(transport_instance, body_only) },
34 { "current_directory", opt_stringptr|opt_public,
35 (void *)offsetof(transport_instance, current_dir) },
36 { "debug_print", opt_stringptr | opt_public,
37 (void *)offsetof(transport_instance, debug_string) },
38 { "delivery_date_add", opt_bool|opt_public,
39 (void *)(offsetof(transport_instance, delivery_date_add)) },
40 { "disable_logging", opt_bool|opt_public,
41 (void *)(offsetof(transport_instance, disable_logging)) },
42 { "driver", opt_stringptr|opt_public,
43 (void *)offsetof(transport_instance, driver_name) },
44 { "envelope_to_add", opt_bool|opt_public,
45 (void *)(offsetof(transport_instance, envelope_to_add)) },
0cbf2b82 46#ifndef DISABLE_EVENT
774ef2d7
JH
47 { "event_action", opt_stringptr | opt_public,
48 (void *)offsetof(transport_instance, event_action) },
49#endif
059ec3d9
PH
50 { "group", opt_expand_gid|opt_public,
51 (void *)offsetof(transport_instance, gid) },
846726c5 52 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
53 (void *)offsetof(transport_instance, add_headers) },
54 { "headers_only", opt_bool|opt_public,
55 (void *)offsetof(transport_instance, headers_only) },
846726c5 56 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
57 (void *)offsetof(transport_instance, remove_headers) },
58 { "headers_rewrite", opt_rewrite|opt_public,
59 (void *)offsetof(transport_instance, headers_rewrite) },
60 { "home_directory", opt_stringptr|opt_public,
61 (void *)offsetof(transport_instance, home_dir) },
62 { "initgroups", opt_bool|opt_public,
63 (void *)offsetof(transport_instance, initgroups) },
fa41615d
JH
64 { "max_parallel", opt_stringptr|opt_public,
65 (void *)offsetof(transport_instance, max_parallel) },
059ec3d9
PH
66 { "message_size_limit", opt_stringptr|opt_public,
67 (void *)offsetof(transport_instance, message_size_limit) },
68 { "rcpt_include_affixes", opt_bool|opt_public,
69 (void *)offsetof(transport_instance, rcpt_include_affixes) },
70 { "retry_use_local_part", opt_bool|opt_public,
71 (void *)offsetof(transport_instance, retry_use_local_part) },
72 { "return_path", opt_stringptr|opt_public,
73 (void *)(offsetof(transport_instance, return_path)) },
74 { "return_path_add", opt_bool|opt_public,
75 (void *)(offsetof(transport_instance, return_path_add)) },
76 { "shadow_condition", opt_stringptr|opt_public,
77 (void *)offsetof(transport_instance, shadow_condition) },
78 { "shadow_transport", opt_stringptr|opt_public,
79 (void *)offsetof(transport_instance, shadow) },
80 { "transport_filter", opt_stringptr|opt_public,
81 (void *)offsetof(transport_instance, filter_command) },
82 { "transport_filter_timeout", opt_time|opt_public,
83 (void *)offsetof(transport_instance, filter_timeout) },
84 { "user", opt_expand_uid|opt_public,
85 (void *)offsetof(transport_instance, uid) }
86};
87
c0b9d3e8 88int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9 89
d185889f
JH
90#ifdef MACRO_PREDEF
91
92# include "macro_predef.h"
059ec3d9 93
c0b9d3e8 94void
d185889f 95options_transports(void)
c0b9d3e8
JH
96{
97struct transport_info * ti;
d185889f 98uschar buf[64];
c0b9d3e8 99
d185889f 100options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
c0b9d3e8
JH
101
102for (ti = transports_available; ti->driver_name[0]; ti++)
4945f557 103 {
cab0c277 104 spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", ti->driver_name);
d185889f
JH
105 builtin_macro_create(buf);
106 options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
4945f557 107 }
383832ef 108}
c0b9d3e8 109
d185889f
JH
110#else /*!MACRO_PREDEF*/
111
112/* Structure for keeping list of addresses that have been added to
113Envelope-To:, in order to avoid duplication. */
114
115struct aci {
116 struct aci *next;
117 address_item *ptr;
118 };
119
120
121/* Static data for write_chunk() */
122
123static uschar *chunk_ptr; /* chunk pointer */
124static uschar *nl_check; /* string to look for at line start */
125static int nl_check_length; /* length of same */
126static uschar *nl_escape; /* string to insert */
127static int nl_escape_length; /* length of same */
128static int nl_partial_match; /* length matched at chunk end */
129
130
059ec3d9
PH
131/*************************************************
132* Initialize transport list *
133*************************************************/
134
135/* Read the transports section of the configuration file, and set up a chain of
136transport instances according to its contents. Each transport has generic
137options and may also have its own private options. This function is only ever
138called when transports == NULL. We use generic code in readconf to do most of
139the work. */
140
141void
142transport_init(void)
143{
144transport_instance *t;
145
146readconf_driver_init(US"transport",
147 (driver_instance **)(&transports), /* chain anchor */
148 (driver_info *)transports_available, /* available drivers */
149 sizeof(transport_info), /* size of info block */
150 &transport_defaults, /* default values for generic options */
151 sizeof(transport_instance), /* size of instance block */
152 optionlist_transports, /* generic options */
153 optionlist_transports_size);
154
155/* Now scan the configured transports and check inconsistencies. A shadow
156transport is permitted only for local transports. */
157
7b4c8c1f 158for (t = transports; t; t = t->next)
059ec3d9 159 {
7b4c8c1f
JH
160 if (!t->info->local && t->shadow)
161 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
162 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
163
164 if (t->body_only && t->headers_only)
165 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
166 "%s transport: body_only and headers_only are mutually exclusive",
167 t->name);
168 }
169}
170
171
172
173/*************************************************
174* Write block of data *
175*************************************************/
176
177/* Subroutine called by write_chunk() and at the end of the message actually
178to write a data block. Also called directly by some transports to write
179additional data to the file descriptor (e.g. prefix, suffix).
180
181If a transport wants data transfers to be timed, it sets a non-zero value in
182transport_write_timeout. A non-zero transport_write_timeout causes a timer to
183be set for each block of data written from here. If time runs out, then write()
184fails and provokes an error return. The caller can then inspect sigalrm_seen to
185check for a timeout.
186
187On some systems, if a quota is exceeded during the write, the yield is the
188number of bytes written rather than an immediate error code. This also happens
189on some systems in other cases, for example a pipe that goes away because the
190other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
191get the error codes the first time.
192
193The write() function is also interruptible; the Solaris 2.6 man page says:
194
195 If write() is interrupted by a signal before it writes any
196 data, it will return -1 with errno set to EINTR.
197
198 If write() is interrupted by a signal after it successfully
199 writes some data, it will return the number of bytes written.
200
201To handle these cases, we want to restart the write() to output the remainder
202of the data after a non-negative return from write(), except after a timeout.
203In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
204proper error then occurs. In principle, after an interruption, the second
205write() could suffer the same fate, but we do not want to continue for
206evermore, so stick a maximum repetition count on the loop to act as a
207longstop.
208
209Arguments:
42055a33 210 tctx transport context: file descriptor or string to write to
059ec3d9
PH
211 block block of bytes to write
212 len number of bytes to write
925ac8e4 213 more further data expected soon
059ec3d9
PH
214
215Returns: TRUE on success, FALSE on failure (with errno preserved);
216 transport_count is incremented by the number of bytes written
217*/
218
42055a33
JH
219static BOOL
220transport_write_block_fd(transport_ctx * tctx, uschar *block, int len, BOOL more)
059ec3d9
PH
221{
222int i, rc, save_errno;
958541e9 223int local_timeout = transport_write_timeout;
42055a33 224int fd = tctx->u.fd;
958541e9
PH
225
226/* This loop is for handling incomplete writes and other retries. In most
227normal cases, it is only ever executed once. */
059ec3d9
PH
228
229for (i = 0; i < 100; i++)
230 {
231 DEBUG(D_transport)
42055a33
JH
232 debug_printf("writing data block fd=%d size=%d timeout=%d%s\n",
233 fd, len, local_timeout, more ? " (more expected)" : "");
059ec3d9 234
958541e9
PH
235 /* This code makes use of alarm() in order to implement the timeout. This
236 isn't a very tidy way of doing things. Using non-blocking I/O with select()
237 provides a neater approach. However, I don't know how to do this when TLS is
238 in use. */
059ec3d9 239
958541e9
PH
240 if (transport_write_timeout <= 0) /* No timeout wanted */
241 {
42055a33
JH
242 rc =
243#ifdef SUPPORT_TLS
925ac8e4 244 tls_out.active == fd ? tls_write(FALSE, block, len, more) :
42055a33
JH
245#endif
246#ifdef MSG_MORE
247 more ? send(fd, block, len, MSG_MORE) :
248#endif
249 write(fd, block, len);
958541e9
PH
250 save_errno = errno;
251 }
059ec3d9 252
958541e9 253 /* Timeout wanted. */
059ec3d9 254
958541e9 255 else
059ec3d9 256 {
958541e9 257 alarm(local_timeout);
42055a33
JH
258
259 rc =
59932f7d 260#ifdef SUPPORT_TLS
925ac8e4 261 tls_out.active == fd ? tls_write(FALSE, block, len, more) :
42055a33
JH
262#endif
263#ifdef MSG_MORE
264 more ? send(fd, block, len, MSG_MORE) :
59932f7d 265#endif
42055a33
JH
266 write(fd, block, len);
267
958541e9
PH
268 save_errno = errno;
269 local_timeout = alarm(0);
059ec3d9
PH
270 if (sigalrm_seen)
271 {
272 errno = ETIMEDOUT;
273 return FALSE;
274 }
275 }
276
277 /* Hopefully, the most common case is success, so test that first. */
278
279 if (rc == len) { transport_count += len; return TRUE; }
280
958541e9
PH
281 /* A non-negative return code is an incomplete write. Try again for the rest
282 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
283
284 if (rc >= 0)
285 {
286 len -= rc;
287 block += rc;
288 transport_count += rc;
289 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 290 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
291 }
292
293 /* A negative return code with an EINTR error is another form of
294 incomplete write, zero bytes having been written */
295
296 if (save_errno == EINTR)
297 {
298 DEBUG(D_transport)
299 debug_printf("write interrupted before anything written\n");
958541e9 300 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
301 }
302
303 /* A response of EAGAIN from write() is likely only in the case of writing
304 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
305
306 if (save_errno == EAGAIN)
307 {
308 DEBUG(D_transport)
309 debug_printf("write temporarily locked out, waiting 1 sec\n");
310 sleep(1);
958541e9
PH
311
312 /* Before continuing to try another write, check that we haven't run out of
313 time. */
314
315 CHECK_TIMEOUT:
316 if (transport_write_timeout > 0 && local_timeout <= 0)
317 {
318 errno = ETIMEDOUT;
319 return FALSE;
320 }
059ec3d9
PH
321 continue;
322 }
323
324 /* Otherwise there's been an error */
325
326 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
327 strerror(save_errno));
328 errno = save_errno;
329 return FALSE;
330 }
331
332/* We've tried and tried and tried but still failed */
333
334errno = ERRNO_WRITEINCOMPLETE;
335return FALSE;
336}
337
338
42055a33
JH
339BOOL
340transport_write_block(transport_ctx * tctx, uschar *block, int len, BOOL more)
341{
342if (!(tctx->options & topt_output_string))
343 return transport_write_block_fd(tctx, block, len, more);
344
345/* Write to expanding-string. NOTE: not NUL-terminated */
346
347if (!tctx->u.msg)
348 {
349 tctx->u.msg = store_get(tctx->msg_size = 1024);
350 tctx->msg_ptr = 0;
351 }
352
353tctx->u.msg = string_catn(tctx->u.msg, &tctx->msg_size, &tctx->msg_ptr, block, len);
354return TRUE;
355}
356
357
059ec3d9
PH
358
359
360/*************************************************
361* Write formatted string *
362*************************************************/
363
364/* This is called by various transports. It is a convenience function.
365
366Arguments:
367 fd file descriptor
368 format string format
369 ... arguments for format
370
371Returns: the yield of transport_write_block()
372*/
373
374BOOL
1ba28e2b 375transport_write_string(int fd, const char *format, ...)
059ec3d9 376{
cab0c277 377transport_ctx tctx = {{0}};
059ec3d9
PH
378va_list ap;
379va_start(ap, format);
380if (!string_vformat(big_buffer, big_buffer_size, format, ap))
381 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
382va_end(ap);
42055a33
JH
383tctx.u.fd = fd;
384return transport_write_block(&tctx, big_buffer, Ustrlen(big_buffer), FALSE);
059ec3d9
PH
385}
386
387
388
389
42055a33
JH
390void
391transport_write_reset(int options)
392{
393if (!(options & topt_continuation)) chunk_ptr = deliver_out_buffer;
394nl_partial_match = -1;
395nl_check_length = nl_escape_length = 0;
396}
397
398
399
059ec3d9
PH
400/*************************************************
401* Write character chunk *
402*************************************************/
403
404/* Subroutine used by transport_write_message() to scan character chunks for
405newlines and act appropriately. The object is to minimise the number of writes.
406The output byte stream is buffered up in deliver_out_buffer, which is written
407only when it gets full, thus minimizing write operations and TCP packets.
408
409Static data is used to handle the case when the last character of the previous
410chunk was NL, or matched part of the data that has to be escaped.
411
412Arguments:
42055a33
JH
413 tctx transport context - processing to be done during output,
414 and file descriptor to write to
059ec3d9
PH
415 chunk pointer to data to write
416 len length of data to write
059ec3d9
PH
417
418In addition, the static nl_xxx variables must be set as required.
419
420Returns: TRUE on success, FALSE on failure (with errno preserved)
421*/
422
42055a33
JH
423BOOL
424write_chunk(transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
425{
426uschar *start = chunk;
427uschar *end = chunk + len;
ff5aac2b 428uschar *ptr;
059ec3d9
PH
429int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
430
431/* The assumption is made that the check string will never stretch over move
432than one chunk since the only time there are partial matches is when copying
433the body in large buffers. There is always enough room in the buffer for an
434escape string, since the loop below ensures this for each character it
435processes, and it won't have stuck in the escape string if it left a partial
436match. */
437
438if (nl_partial_match >= 0)
439 {
440 if (nl_check_length > 0 && len >= nl_check_length &&
441 Ustrncmp(start, nl_check + nl_partial_match,
442 nl_check_length - nl_partial_match) == 0)
443 {
444 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
445 chunk_ptr += nl_escape_length;
446 start += nl_check_length - nl_partial_match;
447 }
448
449 /* The partial match was a false one. Insert the characters carried over
450 from the previous chunk. */
451
452 else if (nl_partial_match > 0)
453 {
454 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
455 chunk_ptr += nl_partial_match;
456 }
457
458 nl_partial_match = -1;
459 }
460
461/* Now process the characters in the chunk. Whenever we hit a newline we check
462for possible escaping. The code for the non-NL route should be as fast as
463possible. */
464
465for (ptr = start; ptr < end; ptr++)
466 {
6d5c916c 467 int ch, len;
059ec3d9
PH
468
469 /* Flush the buffer if it has reached the threshold - we want to leave enough
470 room for the next uschar, plus a possible extra CR for an LF, plus the escape
471 string. */
472
6d5c916c 473 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 474 {
58fc5fb2
JH
475 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
476
6d5c916c
JH
477 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
478 from previous SMTP commands. */
479
d315eda1 480 if (tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2 481 {
d2aa036b 482 if ( tctx->chunk_cb(tctx, (unsigned)len, 0) != OK
42055a33 483 || !transport_write_block(tctx, deliver_out_buffer, len, FALSE)
d2aa036b 484 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
58fc5fb2
JH
485 )
486 return FALSE;
487 }
488 else
42055a33 489 if (!transport_write_block(tctx, deliver_out_buffer, len, FALSE))
6d5c916c 490 return FALSE;
059ec3d9
PH
491 chunk_ptr = deliver_out_buffer;
492 }
493
328c5688
JH
494 /* Remove CR before NL if required */
495
496 if ( *ptr == '\r' && ptr[1] == '\n'
d315eda1 497 && !(tctx->options & topt_use_crlf)
328c5688
JH
498 && spool_file_wireformat
499 )
500 ptr++;
501
059ec3d9
PH
502 if ((ch = *ptr) == '\n')
503 {
504 int left = end - ptr - 1; /* count of chars left after NL */
505
506 /* Insert CR before NL if required */
507
d315eda1 508 if (tctx->options & topt_use_crlf && !spool_file_wireformat)
328c5688 509 *chunk_ptr++ = '\r';
059ec3d9 510 *chunk_ptr++ = '\n';
332f5cf3 511 transport_newlines++;
059ec3d9
PH
512
513 /* The check_string test (formerly "from hack") replaces the specific
514 string at the start of a line with an escape string (e.g. "From " becomes
515 ">From " or "." becomes "..". It is a case-sensitive test. The length
516 check above ensures there is always enough room to insert this string. */
517
518 if (nl_check_length > 0)
519 {
520 if (left >= nl_check_length &&
521 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
522 {
523 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
524 chunk_ptr += nl_escape_length;
525 ptr += nl_check_length;
526 }
527
528 /* Handle the case when there isn't enough left to match the whole
529 check string, but there may be a partial match. We remember how many
530 characters matched, and finish processing this chunk. */
531
532 else if (left <= 0) nl_partial_match = 0;
533
534 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
535 {
536 nl_partial_match = left;
537 ptr = end;
538 }
539 }
540 }
541
542 /* Not a NL character */
543
544 else *chunk_ptr++ = ch;
545 }
546
547return TRUE;
548}
549
550
551
552
553/*************************************************
554* Generate address for RCPT TO *
555*************************************************/
556
557/* This function puts together an address for RCPT to, using the caseful
558version of the local part and the caseful version of the domain. If there is no
559prefix or suffix, or if affixes are to be retained, we can just use the
560original address. Otherwise, if there is a prefix but no suffix we can use a
561pointer into the original address. If there is a suffix, however, we have to
562build a new string.
563
564Arguments:
565 addr the address item
566 include_affixes TRUE if affixes are to be included
567
568Returns: a string
569*/
570
571uschar *
572transport_rcpt_address(address_item *addr, BOOL include_affixes)
573{
574uschar *at;
575int plen, slen;
576
577if (include_affixes)
578 {
579 setflag(addr, af_include_affixes); /* Affects logged => line */
580 return addr->address;
581 }
582
583if (addr->suffix == NULL)
584 {
585 if (addr->prefix == NULL) return addr->address;
586 return addr->address + Ustrlen(addr->prefix);
587 }
588
589at = Ustrrchr(addr->address, '@');
590plen = (addr->prefix == NULL)? 0 : Ustrlen(addr->prefix);
591slen = Ustrlen(addr->suffix);
592
593return string_sprintf("%.*s@%s", (at - addr->address - plen - slen),
594 addr->address + plen, at + 1);
595}
596
597
598/*************************************************
599* Output Envelope-To: address & scan duplicates *
600*************************************************/
601
602/* This function is called from internal_transport_write_message() below, when
603generating an Envelope-To: header line. It checks for duplicates of the given
604address and its ancestors. When one is found, this function calls itself
605recursively, to output the envelope address of the duplicate.
606
607We want to avoid duplication in the list, which can arise for example when
608A->B,C and then both B and C alias to D. This can also happen when there are
609unseen drivers in use. So a list of addresses that have been output is kept in
610the plist variable.
611
612It is also possible to have loops in the address ancestry/duplication graph,
613for example if there are two top level addresses A and B and we have A->B,C and
614B->A. To break the loop, we use a list of processed addresses in the dlist
615variable.
616
617After handling duplication, this function outputs the progenitor of the given
618address.
619
620Arguments:
621 p the address we are interested in
622 pplist address of anchor of the list of addresses not to output
623 pdlist address of anchor of the list of processed addresses
624 first TRUE if this is the first address; set it FALSE afterwards
6d5c916c 625 tctx transport context - processing to be done during output
42055a33 626 and the file descriptor to write to
059ec3d9
PH
627
628Returns: FALSE if writing failed
629*/
630
631static BOOL
632write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
42055a33 633 BOOL *first, transport_ctx * tctx)
059ec3d9
PH
634{
635address_item *pp;
636struct aci *ppp;
637
638/* Do nothing if we have already handled this address. If not, remember it
639so that we don't handle it again. */
640
ff5aac2b 641for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9
PH
642
643ppp = store_get(sizeof(struct aci));
644ppp->next = *pdlist;
645*pdlist = ppp;
646ppp->ptr = p;
647
648/* Now scan up the ancestry, checking for duplicates at each generation. */
649
650for (pp = p;; pp = pp->parent)
651 {
652 address_item *dup;
ff5aac2b
JH
653 for (dup = addr_duplicate; dup; dup = dup->next)
654 if (dup->dupof == pp) /* a dup of our address */
42055a33 655 if (!write_env_to(dup, pplist, pdlist, first, tctx))
ff5aac2b
JH
656 return FALSE;
657 if (!pp->parent) break;
059ec3d9
PH
658 }
659
660/* Check to see if we have already output the progenitor. */
661
ff5aac2b
JH
662for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
663if (ppp) return TRUE;
059ec3d9
PH
664
665/* Remember what we have output, and output it. */
666
667ppp = store_get(sizeof(struct aci));
668ppp->next = *pplist;
669*pplist = ppp;
670ppp->ptr = pp;
671
42055a33 672if (!*first && !write_chunk(tctx, US",\n ", 3)) return FALSE;
059ec3d9 673*first = FALSE;
42055a33 674return write_chunk(tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
675}
676
677
678
679
4c04137d 680/* Add/remove/rewrite headers, and send them plus the empty-line separator.
511a6c14
JH
681
682Globals:
683 header_list
684
685Arguments:
686 addr (chain of) addresses (for extra headers), or NULL;
687 only the first address is used
44bc8f0c 688 tctx transport context
59932f7d 689 sendfn function for output (transport or verify)
511a6c14
JH
690
691Returns: TRUE on success; FALSE on failure.
692*/
693BOOL
42055a33
JH
694transport_headers_send(transport_ctx * tctx,
695 BOOL (*sendfn)(transport_ctx * tctx, uschar * s, int len))
511a6c14
JH
696{
697header_line *h;
6b46ecc6 698const uschar *list;
6d5c916c
JH
699transport_instance * tblock = tctx ? tctx->tblock : NULL;
700address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
701
702/* Then the message's headers. Don't write any that are flagged as "old";
703that means they were rewritten, or are a record of envelope rewriting, or
704were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
705match any entries therein. It is a colon-sep list; expand the items
706separately and squash any empty ones.
d43cbe25 707Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 708
ff5aac2b 709for (h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14
JH
710 {
711 int i;
511a6c14
JH
712 BOOL include_header = TRUE;
713
fb6833e0 714 list = tblock ? tblock->remove_headers : NULL;
d43cbe25 715 for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
716 {
717 if (list)
718 {
719 int sep = ':'; /* This is specified as a colon-separated list */
720 uschar *s, *ss;
8bc732e8 721 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 722 {
76146973
JH
723 int len;
724
725 if (i == 0)
726 if (!(s = expand_string(s)) && !expand_string_forcedfail)
727 {
728 errno = ERRNO_CHHEADER_FAIL;
729 return FALSE;
730 }
d4ff61d1 731 len = s ? Ustrlen(s) : 0;
511a6c14
JH
732 if (strncmpic(h->text, s, len) != 0) continue;
733 ss = h->text + len;
734 while (*ss == ' ' || *ss == '\t') ss++;
735 if (*ss == ':') break;
736 }
6b46ecc6 737 if (s) { include_header = FALSE; break; }
511a6c14 738 }
6b46ecc6 739 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
740 }
741
742 /* If this header is to be output, try to rewrite it if there are rewriting
743 rules. */
744
745 if (include_header)
746 {
fb6833e0 747 if (tblock && tblock->rewrite_rules)
511a6c14
JH
748 {
749 void *reset_point = store_get(0);
750 header_line *hh;
751
6b46ecc6
JH
752 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
753 tblock->rewrite_existflags, FALSE)))
511a6c14 754 {
42055a33 755 if (!sendfn(tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
756 store_reset(reset_point);
757 continue; /* With the next header line */
758 }
759 }
760
761 /* Either no rewriting rules, or it didn't get rewritten */
762
42055a33 763 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
764 }
765
766 /* Header removed */
767
768 else
511a6c14 769 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
511a6c14
JH
770 }
771
772/* Add on any address-specific headers. If there are multiple addresses,
773they will all have the same headers in order to be batched. The headers
774are chained in reverse order of adding (so several addresses from the
775same alias might share some of them) but we want to output them in the
776opposite order. This is a bit tedious, but there shouldn't be very many
777of them. We just walk the list twice, reversing the pointers each time,
778but on the second time, write out the items.
779
780Headers added to an address by a router are guaranteed to end with a newline.
781*/
782
783if (addr)
784 {
785 int i;
d43cbe25 786 header_line *hprev = addr->prop.extra_headers;
511a6c14
JH
787 header_line *hnext;
788 for (i = 0; i < 2; i++)
ff5aac2b 789 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
790 {
791 hnext = h->next;
792 h->next = hprev;
793 hprev = h;
794 if (i == 1)
795 {
42055a33 796 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
797 DEBUG(D_transport)
798 debug_printf("added header line(s):\n%s---\n", h->text);
799 }
800 }
511a6c14
JH
801 }
802
76146973
JH
803/* If a string containing additional headers exists it is a newline-sep
804list. Expand each item and write out the result. This is done last so that
805if it (deliberately or accidentally) isn't in header format, it won't mess
806up any other headers. An empty string or a forced expansion failure are
807noops. An added header string from a transport may not end with a newline;
808add one if it does not. */
511a6c14 809
fb6833e0 810if (tblock && (list = CUS tblock->add_headers))
511a6c14 811 {
76146973
JH
812 int sep = '\n';
813 uschar * s;
814
6b46ecc6 815 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 816 if ((s = expand_string(s)))
76146973
JH
817 {
818 int len = Ustrlen(s);
819 if (len > 0)
511a6c14 820 {
42055a33
JH
821 if (!sendfn(tctx, s, len)) return FALSE;
822 if (s[len-1] != '\n' && !sendfn(tctx, US"\n", 1))
76146973
JH
823 return FALSE;
824 DEBUG(D_transport)
825 {
826 debug_printf("added header line:\n%s", s);
827 if (s[len-1] != '\n') debug_printf("\n");
828 debug_printf("---\n");
829 }
511a6c14
JH
830 }
831 }
ff5aac2b
JH
832 else if (!expand_string_forcedfail)
833 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
834 }
835
836/* Separate headers from body with a blank line */
837
42055a33 838return sendfn(tctx, US"\n", 1);
511a6c14
JH
839}
840
841
059ec3d9
PH
842/*************************************************
843* Write the message *
844*************************************************/
845
846/* This function writes the message to the given file descriptor. The headers
847are in the in-store data structure, and the rest of the message is in the open
848file descriptor deliver_datafile. Make sure we start it at the beginning.
849
850. If add_return_path is TRUE, a "return-path:" header is added to the message,
851 containing the envelope sender's address.
852
853. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
854 giving the top-level envelope address that caused this delivery to happen.
855
856. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
857 message. It gives the time and date that delivery took place.
858
859. If check_string is not null, the start of each line is checked for that
860 string. If it is found, it is replaced by escape_string. This used to be
861 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
862
863. If use_crlf is true, newlines are turned into CRLF (SMTP output).
864
865The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
866any writing or reading error, leaving the code in errno intact. Error exits
867can include timeouts for certain transports, which are requested by setting
868transport_write_timeout non-zero.
869
870Arguments:
6b46ecc6 871 tctx
42055a33
JH
872 (fd, msg) Either and fd, to write the message to,
873 or a string: if null write message to allocated space
874 otherwire take content as headers.
6b46ecc6
JH
875 addr (chain of) addresses (for extra headers), or NULL;
876 only the first address is used
877 tblock optional transport instance block (NULL signifies NULL/0):
878 add_headers a string containing one or more headers to add; it is
879 expanded, and must be in correct RFC 822 format as
880 it is transmitted verbatim; NULL => no additions,
881 and so does empty string or forced expansion fail
882 remove_headers a colon-separated list of headers to remove, or NULL
883 rewrite_rules chain of header rewriting rules
884 rewrite_existflags flags for the rewriting rules
885 options bit-wise options:
886 add_return_path if TRUE, add a "return-path" header
887 add_envelope_to if TRUE, add a "envelope-to" header
888 add_delivery_date if TRUE, add a "delivery-date" header
889 use_crlf if TRUE, turn NL into CR LF
890 end_dot if TRUE, send a terminating "." line at the end
891 no_headers if TRUE, omit the headers
892 no_body if TRUE, omit the body
2d14f397
JH
893 check_string a string to check for at the start of lines, or NULL
894 escape_string a string to insert in front of any check string
895 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
896 it is used when returning messages to their senders,
897 and is approximate rather than exact, owing to chunk
898 buffering
059ec3d9
PH
899
900Returns: TRUE on success; FALSE (with errno) on failure.
901 In addition, the global variable transport_count
902 is incremented by the number of bytes written.
903*/
904
d315eda1 905static BOOL
42055a33 906internal_transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 907{
328c5688 908int len, size = 0;
059ec3d9
PH
909
910/* Initialize pointer in output buffer. */
911
42055a33 912transport_write_reset(tctx->options);
059ec3d9
PH
913
914/* Set up the data for start-of-line data checking and escaping */
915
6b46ecc6 916if (tctx->check_string && tctx->escape_string)
059ec3d9 917 {
6b46ecc6 918 nl_check = tctx->check_string;
059ec3d9 919 nl_check_length = Ustrlen(nl_check);
6b46ecc6 920 nl_escape = tctx->escape_string;
059ec3d9
PH
921 nl_escape_length = Ustrlen(nl_escape);
922 }
059ec3d9 923
328c5688
JH
924/* Whether the escaping mechanism is applied to headers or not is controlled by
925an option (set for SMTP, not otherwise). Negate the length if not wanted till
926after the headers. */
927
928if (!(tctx->options & topt_escape_headers))
929 nl_check_length = -nl_check_length;
930
059ec3d9 931/* Write the headers if required, including any that have to be added. If there
328c5688
JH
932are header rewriting rules, apply them. The datasource is not the -D spoolfile
933so temporarily hide the global that adjusts for its format. */
059ec3d9 934
6b46ecc6 935if (!(tctx->options & topt_no_headers))
059ec3d9 936 {
328c5688
JH
937 BOOL save_wireformat = spool_file_wireformat;
938 spool_file_wireformat = FALSE;
42055a33 939
059ec3d9
PH
940 /* Add return-path: if requested. */
941
6b46ecc6 942 if (tctx->options & topt_add_return_path)
059ec3d9
PH
943 {
944 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 945 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 946 return_path);
328c5688 947 if (!write_chunk(tctx, buffer, n)) goto bad;
059ec3d9
PH
948 }
949
950 /* Add envelope-to: if requested */
951
6b46ecc6 952 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
953 {
954 BOOL first = TRUE;
955 address_item *p;
956 struct aci *plist = NULL;
957 struct aci *dlist = NULL;
958 void *reset_point = store_get(0);
959
328c5688 960 if (!write_chunk(tctx, US"Envelope-to: ", 13)) goto bad;
059ec3d9
PH
961
962 /* Pick up from all the addresses. The plist and dlist variables are
963 anchors for lists of addresses already handled; they have to be defined at
4c04137d 964 this level because write_env_to() calls itself recursively. */
059ec3d9 965
6b46ecc6 966 for (p = tctx->addr; p; p = p->next)
328c5688 967 if (!write_env_to(p, &plist, &dlist, &first, tctx)) goto bad;
059ec3d9
PH
968
969 /* Add a final newline and reset the store used for tracking duplicates */
970
328c5688 971 if (!write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
972 store_reset(reset_point);
973 }
974
975 /* Add delivery-date: if requested. */
976
6b46ecc6 977 if (tctx->options & topt_add_delivery_date)
059ec3d9 978 {
a5ffa9b4
JH
979 uschar * s = tod_stamp(tod_full);
980
981 if ( !write_chunk(tctx, US"Delivery-date: ", 15)
982 || !write_chunk(tctx, s, Ustrlen(s))
983 || !write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
984 }
985
986 /* Then the message's headers. Don't write any that are flagged as "old";
987 that means they were rewritten, or are a record of envelope rewriting, or
988 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 989 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 990 addr is not NULL. */
6b46ecc6 991
42055a33 992 if (!transport_headers_send(tctx, &write_chunk))
328c5688
JH
993 {
994bad:
995 spool_file_wireformat = save_wireformat;
511a6c14 996 return FALSE;
328c5688
JH
997 }
998
999 spool_file_wireformat = save_wireformat;
059ec3d9
PH
1000 }
1001
58fc5fb2
JH
1002/* When doing RFC3030 CHUNKING output, work out how much data would be in a
1003last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
1004(optimally, all of the headers - but it does not matter if we already had to
1005flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 1006(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
1007that further use of write_chunk() will not prepend BDATs.
1008The first BDAT written will also first flush any outstanding MAIL and RCPT
1009commands which were buffered thans to PIPELINING.
1010Commands go out (using a send()) from a different buffer to data (using a
1011write()). They might not end up in the same TCP segment, which is
1012suboptimal. */
59932f7d 1013
65de12cc 1014if (tctx->options & topt_use_bdat)
59932f7d 1015 {
e027f545 1016 off_t fsize;
328c5688 1017 int hsize;
e027f545
JH
1018
1019 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
1020 hsize = 0;
65de12cc 1021 if (!(tctx->options & topt_no_body))
59932f7d
JH
1022 {
1023 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
1024 fsize -= SPOOL_DATA_START_OFFSET;
1025 if (size_limit > 0 && fsize > size_limit)
1026 fsize = size_limit;
e027f545 1027 size = hsize + fsize;
328c5688 1028 if (tctx->options & topt_use_crlf && !spool_file_wireformat)
59932f7d 1029 size += body_linecount; /* account for CRLF-expansion */
42055a33
JH
1030
1031 /* With topt_use_bdat we never do dot-stuffing; no need to
1032 account for any expansion due to that. */
59932f7d
JH
1033 }
1034
e027f545
JH
1035 /* If the message is large, emit first a non-LAST chunk with just the
1036 headers, and reap the command responses. This lets us error out early
1037 on RCPT rejects rather than sending megabytes of data. Include headers
1038 on the assumption they are cheap enough and some clever implementations
1039 might errorcheck them too, on-the-fly, and reject that chunk. */
1040
1041 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
1042 {
58fc5fb2 1043 DEBUG(D_transport)
4c04137d 1044 debug_printf("sending small initial BDAT; hsize=%d\n", hsize);
d2aa036b 1045 if ( tctx->chunk_cb(tctx, hsize, 0) != OK
42055a33 1046 || !transport_write_block(tctx, deliver_out_buffer, hsize, FALSE)
d2aa036b 1047 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
e027f545
JH
1048 )
1049 return FALSE;
1050 chunk_ptr = deliver_out_buffer;
1051 size -= hsize;
1052 }
1053
d2aa036b
JH
1054 /* Emit a LAST datachunk command, and unmark the context for further
1055 BDAT commands. */
59932f7d 1056
d2aa036b 1057 if (tctx->chunk_cb(tctx, size, tc_chunk_last) != OK)
511a6c14 1058 return FALSE;
6d5c916c 1059 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
1060 }
1061
1062/* If the body is required, ensure that the data for check strings (formerly
1063the "from hack") is enabled by negating the length if necessary. (It will be
1064negative in cases where it isn't to apply to the headers). Then ensure the body
1065is positioned at the start of its file (following the message id), then write
1066it, applying the size limit if required. */
1067
328c5688
JH
1068/* If we have a wireformat -D file (CRNL lines, non-dotstuffed, no ending dot)
1069and we want to send a body without dotstuffing or ending-dot, in-clear,
1070then we can just dump it using sendfile.
1071This should get used for CHUNKING output and also for writing the -K file for
1072dkim signing, when we had CHUNKING input. */
1073
7d758a6a 1074#ifdef OS_SENDFILE
328c5688
JH
1075if ( spool_file_wireformat
1076 && !(tctx->options & (topt_no_body | topt_end_dot))
1077 && !nl_check_length
1078 && tls_out.active != tctx->u.fd
1079 )
1080 {
1081 ssize_t copied = 0;
1082 off_t offset = SPOOL_DATA_START_OFFSET;
1083
1084 /* Write out any header data in the buffer */
1085
1086 if ((len = chunk_ptr - deliver_out_buffer) > 0)
1087 {
1088 if (!transport_write_block(tctx, deliver_out_buffer, len, TRUE))
1089 return FALSE;
1090 size -= len;
1091 }
1092
1093 DEBUG(D_transport) debug_printf("using sendfile for body\n");
1094
1095 while(size > 0)
1096 {
7d758a6a 1097 if ((copied = os_sendfile(tctx->u.fd, deliver_datafile, &offset, size)) <= 0) break;
328c5688
JH
1098 size -= copied;
1099 }
1100 return copied >= 0;
1101 }
1102#else
1103DEBUG(D_transport) debug_printf("cannot use sendfile for body: no support\n");
1104#endif
1105
1106DEBUG(D_transport)
1107 if (!(tctx->options & topt_no_body))
1108 debug_printf("cannot use sendfile for body: %s\n",
1109 !spool_file_wireformat ? "spoolfile not wireformat"
1110 : tctx->options & topt_end_dot ? "terminating dot wanted"
1111 : nl_check_length ? "dot- or From-stuffing wanted"
1112 : "TLS output wanted");
1113
6b46ecc6 1114if (!(tctx->options & topt_no_body))
059ec3d9 1115 {
e027f545
JH
1116 int size = size_limit;
1117
059ec3d9
PH
1118 nl_check_length = abs(nl_check_length);
1119 nl_partial_match = 0;
d4ff61d1
JH
1120 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1121 return FALSE;
59932f7d
JH
1122 while ( (len = MAX(DELIVER_IN_BUFFER_SIZE, size)) > 0
1123 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1124 {
42055a33 1125 if (!write_chunk(tctx, deliver_in_buffer, len))
59932f7d 1126 return FALSE;
e027f545 1127 size -= len;
059ec3d9
PH
1128 }
1129
059ec3d9
PH
1130 /* A read error on the body will have left len == -1 and errno set. */
1131
1132 if (len != 0) return FALSE;
c0940526 1133 }
059ec3d9 1134
c0940526 1135/* Finished with the check string */
059ec3d9 1136
c0940526
PP
1137nl_check_length = nl_escape_length = 0;
1138
1139/* If requested, add a terminating "." line (SMTP output). */
1140
42055a33 1141if (tctx->options & topt_end_dot && !write_chunk(tctx, US".\n", 2))
c0940526 1142 return FALSE;
059ec3d9
PH
1143
1144/* Write out any remaining data in the buffer before returning. */
1145
1146return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
42055a33 1147 transport_write_block(tctx, deliver_out_buffer, len, FALSE);
f7572e5a 1148}
80a47a2c 1149
f7572e5a
TK
1150
1151
328c5688 1152
059ec3d9
PH
1153/*************************************************
1154* External interface to write the message *
1155*************************************************/
1156
1157/* If there is no filtering required, call the internal function above to do
1158the real work, passing over all the arguments from this function. Otherwise,
1159set up a filtering process, fork another process to call the internal function
1160to write to the filter, and in this process just suck from the filter and write
42055a33
JH
1161down the fd in the transport context. At the end, tidy up the pipes and the
1162processes.
059ec3d9
PH
1163
1164Arguments: as for internal_transport_write_message() above
1165
1166Returns: TRUE on success; FALSE (with errno) for any failure
1167 transport_count is incremented by the number of bytes written
1168*/
1169
1170BOOL
42055a33 1171transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 1172{
059ec3d9 1173BOOL last_filter_was_NL = TRUE;
328c5688 1174BOOL save_spool_file_wireformat = spool_file_wireformat;
059ec3d9 1175int rc, len, yield, fd_read, fd_write, save_errno;
806c3df9 1176int pfd[2] = {-1, -1};
059ec3d9 1177pid_t filter_pid, write_pid;
6b46ecc6 1178
2e2a30b4
PH
1179transport_filter_timed_out = FALSE;
1180
059ec3d9
PH
1181/* If there is no filter command set up, call the internal function that does
1182the actual work, passing it the incoming fd, and return its result. */
1183
6ff55e50
JH
1184if ( !transport_filter_argv
1185 || !*transport_filter_argv
1186 || !**transport_filter_argv
1187 )
42055a33 1188 return internal_transport_write_message(tctx, size_limit);
059ec3d9
PH
1189
1190/* Otherwise the message must be written to a filter process and read back
1191before being written to the incoming fd. First set up the special processing to
1192be done during the copying. */
1193
059ec3d9
PH
1194nl_partial_match = -1;
1195
6b46ecc6 1196if (tctx->check_string && tctx->escape_string)
059ec3d9 1197 {
6b46ecc6 1198 nl_check = tctx->check_string;
059ec3d9 1199 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1200 nl_escape = tctx->escape_string;
059ec3d9
PH
1201 nl_escape_length = Ustrlen(nl_escape);
1202 }
1203else nl_check_length = nl_escape_length = 0;
1204
1205/* Start up a subprocess to run the command. Ensure that our main fd will
1206be closed when the subprocess execs, but remove the flag afterwards.
1207(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1208process to deliver another message.) We get back stdin/stdout file descriptors.
1209If the process creation failed, give an error return. */
1210
1211fd_read = -1;
1212fd_write = -1;
1213save_errno = 0;
1214yield = FALSE;
1215write_pid = (pid_t)(-1);
1216
92b0827a 1217 {
42055a33
JH
1218 int bits = fcntl(tctx->u.fd, F_GETFD);
1219 (void)fcntl(tctx->u.fd, F_SETFD, bits | FD_CLOEXEC);
92b0827a
JH
1220 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
1221 &fd_write, &fd_read, FALSE);
42055a33 1222 (void)fcntl(tctx->u.fd, F_SETFD, bits & ~FD_CLOEXEC);
92b0827a 1223 }
059ec3d9
PH
1224if (filter_pid < 0) goto TIDY_UP; /* errno set */
1225
1226DEBUG(D_transport)
fb6833e0 1227 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1228 (int)filter_pid, fd_write, fd_read);
1229
1230/* Fork subprocess to write the message to the filter, and return the result
1231via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1232smtp dots, or check string processing. */
1233
1234if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
1235if ((write_pid = fork()) == 0)
1236 {
1237 BOOL rc;
f1e894f3
PH
1238 (void)close(fd_read);
1239 (void)close(pfd[pipe_read]);
059ec3d9 1240 nl_check_length = nl_escape_length = 0;
6b46ecc6 1241
42055a33 1242 tctx->u.fd = fd_write;
fb6833e0 1243 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1244 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6 1245
42055a33 1246 rc = internal_transport_write_message(tctx, size_limit);
6b46ecc6 1247
059ec3d9 1248 save_errno = errno;
1ac6b2e7
JH
1249 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1250 != sizeof(BOOL)
1251 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1252 != sizeof(int)
6b46ecc6 1253 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1ac6b2e7 1254 != sizeof(int)
32dfdf8b
JH
1255 || write(pfd[pipe_write], (void *)&tctx->addr->delivery_usec, sizeof(int))
1256 != sizeof(int)
1ac6b2e7
JH
1257 )
1258 rc = FALSE; /* compiler quietening */
059ec3d9
PH
1259 _exit(0);
1260 }
1261save_errno = errno;
1262
1263/* Parent process: close our copy of the writing subprocess' pipes. */
1264
f1e894f3
PH
1265(void)close(pfd[pipe_write]);
1266(void)close(fd_write);
059ec3d9
PH
1267fd_write = -1;
1268
1269/* Writing process creation failed */
1270
1271if (write_pid < 0)
1272 {
1273 errno = save_errno; /* restore */
1274 goto TIDY_UP;
1275 }
1276
1277/* When testing, let the subprocess get going */
1278
1279if (running_in_test_harness) millisleep(250);
1280
1281DEBUG(D_transport)
1282 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1283
1284/* Copy the message from the filter to the output fd. A read error leaves len
1285== -1 and errno set. We need to apply a timeout to the read, to cope with
1286the case when the filter gets stuck, but it can be quite a long one. The
1287default is 5m, but this is now configurable. */
1288
1289DEBUG(D_transport) debug_printf("copying from the filter\n");
1290
1291/* Copy the output of the filter, remembering if the last character was NL. If
1292no data is returned, that counts as "ended with NL" (default setting of the
328c5688
JH
1293variable is TRUE). The output should always be unix-format as we converted
1294any wireformat source on writing input to the filter. */
059ec3d9 1295
328c5688 1296spool_file_wireformat = FALSE;
059ec3d9
PH
1297chunk_ptr = deliver_out_buffer;
1298
1299for (;;)
1300 {
1301 sigalrm_seen = FALSE;
1302 alarm(transport_filter_timeout);
1303 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
1304 alarm(0);
1305 if (sigalrm_seen)
1306 {
1307 errno = ETIMEDOUT;
2e2a30b4 1308 transport_filter_timed_out = TRUE;
059ec3d9
PH
1309 goto TIDY_UP;
1310 }
1311
1312 /* If the read was successful, write the block down the original fd,
1313 remembering whether it ends in \n or not. */
1314
1315 if (len > 0)
1316 {
42055a33 1317 if (!write_chunk(tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1318 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1319 }
1320
1321 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1322
1323 else
1324 {
1325 if (len == 0) yield = TRUE;
1326 break;
1327 }
1328 }
1329
1330/* Tidying up code. If yield = FALSE there has been an error and errno is set
1331to something. Ensure the pipes are all closed and the processes are removed. If
1332there has been an error, kill the processes before waiting for them, just to be
1333sure. Also apply a paranoia timeout. */
1334
1335TIDY_UP:
328c5688 1336spool_file_wireformat = save_spool_file_wireformat;
059ec3d9
PH
1337save_errno = errno;
1338
f1e894f3
PH
1339(void)close(fd_read);
1340if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1341
1342if (!yield)
1343 {
1344 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1345 if (write_pid > 0) kill(write_pid, SIGKILL);
1346 }
1347
1348/* Wait for the filter process to complete. */
1349
1350DEBUG(D_transport) debug_printf("waiting for filter process\n");
1351if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1352 {
1353 yield = FALSE;
1354 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1355 tctx->addr->more_errno = rc;
059ec3d9
PH
1356 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1357 }
1358
1359/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1360read the results from its pipe, provided we haven't already had a filter
35af9f61 1361process failure. */
059ec3d9
PH
1362
1363DEBUG(D_transport) debug_printf("waiting for writing process\n");
1364if (write_pid > 0)
1365 {
35af9f61
PH
1366 rc = child_close(write_pid, 30);
1367 if (yield)
8e669ac1 1368 if (rc == 0)
35af9f61
PH
1369 {
1370 BOOL ok;
4dc2379a
JH
1371 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
1372 {
1373 DEBUG(D_transport)
1374 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1375 save_errno = ERRNO_FILTER_FAIL;
1376 yield = FALSE;
1377 }
1378 else if (!ok)
35af9f61 1379 {
4dc2379a 1380 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
32dfdf8b
JH
1381 dummy = read(pfd[pipe_read], (void *)&tctx->addr->more_errno, sizeof(int));
1382 dummy = read(pfd[pipe_read], (void *)&tctx->addr->delivery_usec, sizeof(int));
cab0c277 1383 dummy = dummy; /* compiler quietening */
35af9f61
PH
1384 yield = FALSE;
1385 }
1386 }
1387 else
059ec3d9 1388 {
059ec3d9 1389 yield = FALSE;
35af9f61 1390 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1391 tctx->addr->more_errno = rc;
35af9f61 1392 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1393 }
059ec3d9 1394 }
f1e894f3 1395(void)close(pfd[pipe_read]);
059ec3d9
PH
1396
1397/* If there have been no problems we can now add the terminating "." if this is
1398SMTP output, turning off escaping beforehand. If the last character from the
1399filter was not NL, insert a NL to make the SMTP protocol work. */
1400
1401if (yield)
1402 {
1403 nl_check_length = nl_escape_length = 0;
6b46ecc6 1404 if ( tctx->options & topt_end_dot
ff5aac2b 1405 && ( last_filter_was_NL
42055a33
JH
1406 ? !write_chunk(tctx, US".\n", 2)
1407 : !write_chunk(tctx, US"\n.\n", 3)
ff5aac2b 1408 ) )
059ec3d9 1409 yield = FALSE;
059ec3d9
PH
1410
1411 /* Write out any remaining data in the buffer. */
1412
1413 else
ff5aac2b 1414 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
42055a33 1415 || transport_write_block(tctx, deliver_out_buffer, len, FALSE);
059ec3d9 1416 }
ff5aac2b
JH
1417else
1418 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1419
1420DEBUG(D_transport)
1421 {
1422 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1423 if (!yield)
6b46ecc6 1424 debug_printf("errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1425 }
1426
1427return yield;
1428}
1429
1430
1431
1432
1433
1434/*************************************************
1435* Update waiting database *
1436*************************************************/
1437
1438/* This is called when an address is deferred by remote transports that are
1439capable of sending more than one message over one connection. A database is
1440maintained for each transport, keeping track of which messages are waiting for
1441which hosts. The transport can then consult this when eventually a successful
1442delivery happens, and if it finds that another message is waiting for the same
1443host, it can fire up a new process to deal with it using the same connection.
1444
1445The database records are keyed by host name. They can get full if there are
1446lots of messages waiting, and so there is a continuation mechanism for them.
1447
1448Each record contains a list of message ids, packed end to end without any
1449zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1450in this record, and the sequence field says if there are any other records for
1451this host. If the sequence field is 0, there are none. If it is 1, then another
1452record with the name <hostname>:0 exists; if it is 2, then two other records
1453with sequence numbers 0 and 1 exist, and so on.
1454
1455Currently, an exhaustive search of all continuation records has to be done to
1456determine whether to add a message id to a given record. This shouldn't be
1457too bad except in extreme cases. I can't figure out a *simple* way of doing
1458better.
1459
1460Old records should eventually get swept up by the exim_tidydb utility.
1461
1462Arguments:
f6c332bd 1463 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1464 tpname name of the transport
1465
1466Returns: nothing
1467*/
1468
1469void
1470transport_update_waiting(host_item *hostlist, uschar *tpname)
1471{
55414b25 1472const uschar *prevname = US"";
059ec3d9
PH
1473host_item *host;
1474open_db dbblock;
1475open_db *dbm_file;
1476
7a0743eb
PH
1477DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1478
059ec3d9
PH
1479/* Open the database for this transport */
1480
36d295f1
JH
1481if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
1482 O_RDWR, &dbblock, TRUE)))
1483 return;
059ec3d9
PH
1484
1485/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1486that the message id is in each host record. */
059ec3d9 1487
36d295f1 1488for (host = hostlist; host; host = host->next)
059ec3d9
PH
1489 {
1490 BOOL already = FALSE;
1491 dbdata_wait *host_record;
1492 uschar *s;
1493 int i, host_length;
36d295f1 1494 uschar buffer[256];
059ec3d9 1495
059ec3d9
PH
1496 /* Skip if this is the same host as we just processed; otherwise remember
1497 the name for next time. */
1498
1499 if (Ustrcmp(prevname, host->name) == 0) continue;
1500 prevname = host->name;
1501
1502 /* Look up the host record; if there isn't one, make an empty one. */
1503
36d295f1 1504 if (!(host_record = dbfn_read(dbm_file, host->name)))
059ec3d9
PH
1505 {
1506 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH);
1507 host_record->count = host_record->sequence = 0;
1508 }
1509
1510 /* Compute the current length */
1511
1512 host_length = host_record->count * MESSAGE_ID_LENGTH;
1513
1514 /* Search the record to see if the current message is already in it. */
1515
1516 for (s = host_record->text; s < host_record->text + host_length;
1517 s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1518 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1519 { already = TRUE; break; }
059ec3d9
PH
1520
1521 /* If we haven't found this message in the main record, search any
1522 continuation records that exist. */
1523
1524 for (i = host_record->sequence - 1; i >= 0 && !already; i--)
1525 {
1526 dbdata_wait *cont;
1527 sprintf(CS buffer, "%.200s:%d", host->name, i);
36d295f1 1528 if ((cont = dbfn_read(dbm_file, buffer)))
059ec3d9
PH
1529 {
1530 int clen = cont->count * MESSAGE_ID_LENGTH;
1531 for (s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1532 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1533 { already = TRUE; break; }
059ec3d9
PH
1534 }
1535 }
1536
1537 /* If this message is already in a record, no need to update. */
1538
7a0743eb
PH
1539 if (already)
1540 {
1541 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1542 continue;
1543 }
059ec3d9
PH
1544
1545
1546 /* If this record is full, write it out with a new name constructed
1547 from the sequence number, increase the sequence number, and empty
1548 the record. */
1549
1550 if (host_record->count >= WAIT_NAME_MAX)
1551 {
1552 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1553 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
1554 host_record->sequence++;
1555 host_record->count = 0;
1556 host_length = 0;
1557 }
1558
1559 /* If this record is not full, increase the size of the record to
1560 allow for one new message id. */
1561
1562 else
1563 {
1564 dbdata_wait *newr =
1565 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH);
1566 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1567 host_record = newr;
1568 }
1569
1570 /* Now add the new name on the end */
1571
1572 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1573 host_record->count++;
1574 host_length += MESSAGE_ID_LENGTH;
1575
1576 /* Update the database */
1577
1578 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1579 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1580 }
1581
1582/* All now done */
1583
1584dbfn_close(dbm_file);
1585}
1586
1587
1588
1589
1590/*************************************************
1591* Test for waiting messages *
1592*************************************************/
1593
1594/* This function is called by a remote transport which uses the previous
1595function to remember which messages are waiting for which remote hosts. It's
1596called after a successful delivery and its job is to check whether there is
1597another message waiting for the same host. However, it doesn't do this if the
1598current continue sequence is greater than the maximum supplied as an argument,
1599or greater than the global connection_max_messages, which, if set, overrides.
1600
1601Arguments:
1602 transport_name name of the transport
1603 hostname name of the host
1604 local_message_max maximum number of messages down one connection
1605 as set by the caller transport
1606 new_message_id set to the message id of a waiting message
1607 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1608 oicf_func function to call to validate if it is ok to send
1609 to this message_id from the current instance.
1610 oicf_data opaque data for oicf_func
059ec3d9
PH
1611
1612Returns: TRUE if new_message_id set; FALSE otherwise
1613*/
1614
a39bd74d
JB
1615typedef struct msgq_s
1616{
1617 uschar message_id [MESSAGE_ID_LENGTH + 1];
1618 BOOL bKeep;
1619} msgq_t;
1620
059ec3d9 1621BOOL
55414b25 1622transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1623 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1624{
1625dbdata_wait *host_record;
0539a19d 1626int host_length;
059ec3d9
PH
1627open_db dbblock;
1628open_db *dbm_file;
059ec3d9 1629
a39bd74d 1630int i;
a39bd74d 1631struct stat statbuf;
a39bd74d 1632
059ec3d9
PH
1633*more = FALSE;
1634
1635DEBUG(D_transport)
1636 {
1637 debug_printf("transport_check_waiting entered\n");
1638 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1639 continue_sequence, local_message_max, connection_max_messages);
1640 }
1641
1642/* Do nothing if we have hit the maximum number that can be send down one
1643connection. */
1644
1645if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1646if (local_message_max > 0 && continue_sequence >= local_message_max)
1647 {
1648 DEBUG(D_transport)
1649 debug_printf("max messages for one connection reached: returning\n");
1650 return FALSE;
1651 }
1652
1653/* Open the waiting information database. */
1654
36d295f1
JH
1655if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
1656 O_RDWR, &dbblock, TRUE)))
1657 return FALSE;
059ec3d9
PH
1658
1659/* See if there is a record for this host; if not, there's nothing to do. */
1660
789f8a4f 1661if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1662 {
1663 dbfn_close(dbm_file);
1664 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1665 return FALSE;
1666 }
1667
1668/* If the data in the record looks corrupt, just log something and
1669don't try to use it. */
1670
1671if (host_record->count > WAIT_NAME_MAX)
1672 {
1673 dbfn_close(dbm_file);
1674 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1675 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1676 return FALSE;
1677 }
1678
1679/* Scan the message ids in the record from the end towards the beginning,
1680until one is found for which a spool file actually exists. If the record gets
1681emptied, delete it and continue with any continuation records that may exist.
1682*/
1683
a39bd74d
JB
1684/* For Bug 1141, I refactored this major portion of the routine, it is risky
1685but the 1 off will remain without it. This code now allows me to SKIP over
1686a message I do not want to send out on this run. */
059ec3d9 1687
a39bd74d
JB
1688host_length = host_record->count * MESSAGE_ID_LENGTH;
1689
1690while (1)
059ec3d9 1691 {
789f8a4f
JH
1692 msgq_t *msgq;
1693 int msgq_count = 0;
1694 int msgq_actual = 0;
1695 BOOL bFound = FALSE;
1696 BOOL bContinuation = FALSE;
1697
a39bd74d 1698 /* create an array to read entire message queue into memory for processing */
059ec3d9 1699
40c90bca 1700 msgq = store_malloc(sizeof(msgq_t) * host_record->count);
a39bd74d
JB
1701 msgq_count = host_record->count;
1702 msgq_actual = msgq_count;
059ec3d9 1703
a39bd74d 1704 for (i = 0; i < host_record->count; ++i)
059ec3d9 1705 {
a39bd74d
JB
1706 msgq[i].bKeep = TRUE;
1707
1708 Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1709 MESSAGE_ID_LENGTH);
a39bd74d
JB
1710 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1711 }
1712
1713 /* first thing remove current message id if it exists */
059ec3d9 1714
a39bd74d
JB
1715 for (i = 0; i < msgq_count; ++i)
1716 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1717 {
1718 msgq[i].bKeep = FALSE;
1719 break;
1720 }
1721
1722 /* now find the next acceptable message_id */
1723
a39bd74d
JB
1724 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1725 {
41313d92
JH
1726 uschar subdir[2];
1727
1728 subdir[0] = split_spool_directory ? msgq[i].message_id[5] : 0;
1729 subdir[1] = 0;
059ec3d9 1730
41313d92
JH
1731 if (Ustat(spool_fname(US"input", subdir, msgq[i].message_id, US"-D"),
1732 &statbuf) != 0)
a39bd74d
JB
1733 msgq[i].bKeep = FALSE;
1734 else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data))
059ec3d9 1735 {
a39bd74d
JB
1736 Ustrcpy(new_message_id, msgq[i].message_id);
1737 msgq[i].bKeep = FALSE;
1738 bFound = TRUE;
059ec3d9
PH
1739 break;
1740 }
1741 }
1742
a39bd74d
JB
1743 /* re-count */
1744 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1745 if (msgq[i].bKeep)
1746 msgq_actual++;
1747
1748 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1749 memory queue */
a39bd74d
JB
1750
1751 if (msgq_actual <= 0)
1752 {
1753 host_length = 0;
1754 host_record->count = 0;
1755 }
1756 else
1757 {
1758 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1759 host_record->count = msgq_actual;
1760
1761 if (msgq_actual < msgq_count)
1762 {
1763 int new_count;
1764 for (new_count = 0, i = 0; i < msgq_count; ++i)
1765 if (msgq[i].bKeep)
1766 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1767 msgq[i].message_id, MESSAGE_ID_LENGTH);
1768
1769 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1770 }
1771 }
1772
36d295f1 1773 /* Check for a continuation record. */
a39bd74d 1774
059ec3d9
PH
1775 while (host_length <= 0)
1776 {
1777 int i;
a39bd74d 1778 dbdata_wait * newr = NULL;
36d295f1 1779 uschar buffer[256];
059ec3d9
PH
1780
1781 /* Search for a continuation */
1782
a39bd74d 1783 for (i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1784 {
1785 sprintf(CS buffer, "%.200s:%d", hostname, i);
1786 newr = dbfn_read(dbm_file, buffer);
1787 }
1788
1789 /* If no continuation, delete the current and break the loop */
1790
a39bd74d 1791 if (!newr)
059ec3d9
PH
1792 {
1793 dbfn_delete(dbm_file, hostname);
1794 break;
1795 }
1796
1797 /* Else replace the current with the continuation */
1798
1799 dbfn_delete(dbm_file, buffer);
1800 host_record = newr;
1801 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1802
a39bd74d
JB
1803 bContinuation = TRUE;
1804 }
059ec3d9 1805
789f8a4f
JH
1806 if (bFound) /* Usual exit from main loop */
1807 {
40c90bca 1808 store_free (msgq);
a39bd74d 1809 break;
789f8a4f 1810 }
059ec3d9
PH
1811
1812 /* If host_length <= 0 we have emptied a record and not found a good message,
1813 and there are no continuation records. Otherwise there is a continuation
1814 record to process. */
1815
1816 if (host_length <= 0)
1817 {
1818 dbfn_close(dbm_file);
1819 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1820 return FALSE;
1821 }
a39bd74d
JB
1822
1823 /* we were not able to find an acceptable message, nor was there a
1824 * continuation record. So bug out, outer logic will clean this up.
1825 */
1826
1827 if (!bContinuation)
1828 {
789f8a4f 1829 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1830 dbfn_close(dbm_file);
1831 return FALSE;
1832 }
a39bd74d 1833
40c90bca 1834 store_free(msgq);
789f8a4f 1835 } /* we need to process a continuation record */
059ec3d9
PH
1836
1837/* Control gets here when an existing message has been encountered; its
1838id is in new_message_id, and host_length is the revised length of the
1839host record. If it is zero, the record has been removed. Update the
1840record if required, close the database, and return TRUE. */
1841
1842if (host_length > 0)
1843 {
1844 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1845
059ec3d9
PH
1846 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1847 *more = TRUE;
1848 }
1849
1850dbfn_close(dbm_file);
1851return TRUE;
1852}
1853
059ec3d9
PH
1854/*************************************************
1855* Deliver waiting message down same socket *
1856*************************************************/
1857
57cc2785
JH
1858/* Just the regain-root-privilege exec portion */
1859void
1860transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
1861 const uschar *hostaddress, uschar *id, int socket_fd)
1862{
57cc2785
JH
1863int i = 20;
1864const uschar **argv;
1865
1866/* Set up the calling arguments; use the standard function for the basics,
1867but we have a number of extras that may be added. */
1868
1869argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
1870
1871if (smtp_authenticated) argv[i++] = US"-MCA";
14de8063
JH
1872if (smtp_peer_options & OPTION_CHUNKING) argv[i++] = US"-MCK";
1873if (smtp_peer_options & OPTION_DSN) argv[i++] = US"-MCD";
1874if (smtp_peer_options & OPTION_PIPE) argv[i++] = US"-MCP";
1875if (smtp_peer_options & OPTION_SIZE) argv[i++] = US"-MCS";
57cc2785 1876#ifdef SUPPORT_TLS
14de8063 1877if (smtp_peer_options & OPTION_TLS)
57cc2785
JH
1878 if (tls_out.active >= 0 || continue_proxy_cipher)
1879 {
1880 argv[i++] = US"-MCt";
1881 argv[i++] = sending_ip_address;
1882 argv[i++] = string_sprintf("%d", sending_port);
1883 argv[i++] = tls_out.active >= 0 ? tls_out.cipher : continue_proxy_cipher;
1884 }
1885 else
1886 argv[i++] = US"-MCT";
1887#endif
1888
1889if (queue_run_pid != (pid_t)0)
1890 {
1891 argv[i++] = US"-MCQ";
1892 argv[i++] = string_sprintf("%d", queue_run_pid);
1893 argv[i++] = string_sprintf("%d", queue_run_pipe);
1894 }
1895
1896argv[i++] = US"-MC";
1897argv[i++] = US transport_name;
1898argv[i++] = US hostname;
1899argv[i++] = US hostaddress;
1900argv[i++] = string_sprintf("%d", continue_sequence + 1);
1901argv[i++] = id;
1902argv[i++] = NULL;
1903
1904/* Arrange for the channel to be on stdin. */
1905
1906if (socket_fd != 0)
1907 {
1908 (void)dup2(socket_fd, 0);
1909 (void)close(socket_fd);
1910 }
1911
1912DEBUG(D_exec) debug_print_argv(argv);
1913exim_nullstd(); /* Ensure std{out,err} exist */
1914execv(CS argv[0], (char *const *)argv);
1915
1916DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
1917_exit(errno); /* Note: must be _exit(), NOT exit() */
1918}
1919
1920
1921
059ec3d9
PH
1922/* Fork a new exim process to deliver the message, and do a re-exec, both to
1923get a clean delivery process, and to regain root privilege in cases where it
1924has been given away.
1925
1926Arguments:
1927 transport_name to pass to the new process
1928 hostname ditto
1929 hostaddress ditto
1930 id the new message to process
1931 socket_fd the connected socket
1932
1933Returns: FALSE if fork fails; TRUE otherwise
1934*/
1935
1936BOOL
55414b25
JH
1937transport_pass_socket(const uschar *transport_name, const uschar *hostname,
1938 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
1939{
1940pid_t pid;
1941int status;
1942
1943DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
1944
1945if ((pid = fork()) == 0)
1946 {
059ec3d9
PH
1947 /* Disconnect entirely from the parent process. If we are running in the
1948 test harness, wait for a bit to allow the previous process time to finish,
1949 write the log, etc., so that the output is always in the same order for
1950 automatic comparison. */
1951
b7d3afcf
JH
1952 if ((pid = fork()) != 0)
1953 {
1954 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (final-pid %d)\n", pid);
1955 _exit(EXIT_SUCCESS);
1956 }
ed0e9820 1957 if (running_in_test_harness) sleep(1);
059ec3d9 1958
57cc2785
JH
1959 transport_do_pass_socket(transport_name, hostname, hostaddress,
1960 id, socket_fd);
059ec3d9
PH
1961 }
1962
1963/* If the process creation succeeded, wait for the first-level child, which
1964immediately exits, leaving the second level process entirely disconnected from
1965this one. */
1966
1967if (pid > 0)
1968 {
1969 int rc;
1970 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
b7d3afcf 1971 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (inter-pid %d)\n", pid);
059ec3d9
PH
1972 return TRUE;
1973 }
1974else
1975 {
1976 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
1977 strerror(errno));
1978 return FALSE;
1979 }
1980}
1981
1982
1983
1984/*************************************************
1985* Set up direct (non-shell) command *
1986*************************************************/
1987
1988/* This function is called when a command line is to be parsed and executed
1989directly, without the use of /bin/sh. It is called by the pipe transport,
1990the queryprogram router, and also from the main delivery code when setting up a
1991transport filter process. The code for ETRN also makes use of this; in that
1992case, no addresses are passed.
1993
1994Arguments:
1995 argvptr pointer to anchor for argv vector
55414b25 1996 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
1997 expand_arguments true if expansion is to occur
1998 expand_failed error value to set if expansion fails; not relevant if
1999 addr == NULL
2000 addr chain of addresses, or NULL
2001 etext text for use in error messages
2002 errptr where to put error message if addr is NULL;
2003 otherwise it is put in the first address
2004
2005Returns: TRUE if all went well; otherwise an error will be
2006 set in the first address and FALSE returned
2007*/
2008
2009BOOL
55414b25
JH
2010transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2011 BOOL expand_arguments, int expand_failed, address_item *addr,
2012 uschar *etext, uschar **errptr)
059ec3d9
PH
2013{
2014address_item *ad;
55414b25 2015const uschar **argv;
059ec3d9
PH
2016uschar *s, *ss;
2017int address_count = 0;
2018int argcount = 0;
2019int i, max_args;
2020
2021/* Get store in which to build an argument list. Count the number of addresses
2022supplied, and allow for that many arguments, plus an additional 60, which
2023should be enough for anybody. Multiple addresses happen only when the local
2024delivery batch option is set. */
2025
2026for (ad = addr; ad != NULL; ad = ad->next) address_count++;
2027max_args = address_count + 60;
2028*argvptr = argv = store_get((max_args+1)*sizeof(uschar *));
2029
2030/* Split the command up into arguments terminated by white space. Lose
2031trailing space at the start and end. Double-quoted arguments can contain \\ and
2032\" escapes and so can be handled by the standard function; single-quoted
2033arguments are verbatim. Copy each argument into a new string. */
2034
2035s = cmd;
2036while (isspace(*s)) s++;
2037
2038while (*s != 0 && argcount < max_args)
2039 {
2040 if (*s == '\'')
2041 {
2042 ss = s + 1;
2043 while (*ss != 0 && *ss != '\'') ss++;
2044 argv[argcount++] = ss = store_get(ss - s++);
2045 while (*s != 0 && *s != '\'') *ss++ = *s++;
2046 if (*s != 0) s++;
2047 *ss++ = 0;
2048 }
55414b25 2049 else argv[argcount++] = string_copy(string_dequote(CUSS &s));
059ec3d9
PH
2050 while (isspace(*s)) s++;
2051 }
2052
2053argv[argcount] = (uschar *)0;
2054
2055/* If *s != 0 we have run out of argument slots. */
2056
2057if (*s != 0)
2058 {
2059 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2060 "%s", cmd, etext);
2061 if (addr != NULL)
2062 {
2063 addr->transport_return = FAIL;
2064 addr->message = msg;
2065 }
2066 else *errptr = msg;
2067 return FALSE;
2068 }
2069
2070/* Expand each individual argument if required. Expansion happens for pipes set
2071up in filter files and with directly-supplied commands. It does not happen if
2072the pipe comes from a traditional .forward file. A failing expansion is a big
2073disaster if the command came from Exim's configuration; if it came from a user
2074it is just a normal failure. The expand_failed value is used as the error value
2075to cater for these two cases.
2076
2077An argument consisting just of the text "$pipe_addresses" is treated specially.
2078It is not passed to the general expansion function. Instead, it is replaced by
2079a number of arguments, one for each address. This avoids problems with shell
2080metacharacters and spaces in addresses.
2081
2082If the parent of the top address has an original part of "system-filter", this
2083pipe was set up by the system filter, and we can permit the expansion of
2084$recipients. */
2085
2086DEBUG(D_transport)
2087 {
2088 debug_printf("direct command:\n");
2089 for (i = 0; argv[i] != (uschar *)0; i++)
2090 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2091 }
2092
2093if (expand_arguments)
2094 {
2095 BOOL allow_dollar_recipients = addr != NULL &&
2096 addr->parent != NULL &&
2097 Ustrcmp(addr->parent->address, "system-filter") == 0;
2098
2099 for (i = 0; argv[i] != (uschar *)0; i++)
2100 {
2101
2102 /* Handle special fudge for passing an address list */
2103
2104 if (addr != NULL &&
2105 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2106 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2107 {
2108 int additional;
2109
2110 if (argcount + address_count - 1 > max_args)
2111 {
2112 addr->transport_return = FAIL;
2113 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2114 "in %s", cmd, etext);
2115 return FALSE;
2116 }
2117
2118 additional = address_count - 1;
2119 if (additional > 0)
2120 memmove(argv + i + 1 + additional, argv + i + 1,
2121 (argcount - i)*sizeof(uschar *));
2122
09792322
NK
2123 for (ad = addr; ad != NULL; ad = ad->next) {
2124 argv[i++] = ad->address;
2125 argcount++;
2126 }
2127
700d22f3
PP
2128 /* Subtract one since we replace $pipe_addresses */
2129 argcount--;
2130 i--;
09792322
NK
2131 }
2132
2133 /* Handle special case of $address_pipe when af_force_command is set */
2134
2135 else if (addr != NULL && testflag(addr,af_force_command) &&
2136 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2137 Ustrcmp(argv[i], "${address_pipe}") == 0))
2138 {
2139 int address_pipe_i;
2140 int address_pipe_argcount = 0;
2141 int address_pipe_max_args;
2142 uschar **address_pipe_argv;
2143
2144 /* We can never have more then the argv we will be loading into */
2145 address_pipe_max_args = max_args - argcount + 1;
2146
2147 DEBUG(D_transport)
700d22f3 2148 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2149
2150 /* We allocate an additional for (uschar *)0 */
2151 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *));
2152
2153 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2154 s = expand_string(addr->local_part + 1);
2155
700d22f3
PP
2156 if (s == NULL || *s == '\0')
2157 {
2158 addr->transport_return = FAIL;
2159 addr->message = string_sprintf("Expansion of \"%s\" "
2160 "from command \"%s\" in %s failed: %s",
2161 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2162 return FALSE;
700d22f3 2163 }
09792322
NK
2164
2165 while (isspace(*s)) s++; /* strip leading space */
2166
2167 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2168 {
2169 if (*s == '\'')
2170 {
2171 ss = s + 1;
2172 while (*ss != 0 && *ss != '\'') ss++;
2173 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++);
2174 while (*s != 0 && *s != '\'') *ss++ = *s++;
2175 if (*s != 0) s++;
2176 *ss++ = 0;
2177 }
55414b25
JH
2178 else address_pipe_argv[address_pipe_argcount++] =
2179 string_copy(string_dequote(CUSS &s));
09792322
NK
2180 while (isspace(*s)) s++; /* strip space after arg */
2181 }
2182
2183 address_pipe_argv[address_pipe_argcount] = (uschar *)0;
2184
2185 /* If *s != 0 we have run out of argument slots. */
2186 if (*s != 0)
2187 {
2188 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2189 "\"%s\" in %s", addr->local_part + 1, etext);
2190 if (addr != NULL)
2191 {
2192 addr->transport_return = FAIL;
2193 addr->message = msg;
2194 }
2195 else *errptr = msg;
2196 return FALSE;
2197 }
2198
700d22f3
PP
2199 /* address_pipe_argcount - 1
2200 * because we are replacing $address_pipe in the argument list
2201 * with the first thing it expands to */
09792322
NK
2202 if (argcount + address_pipe_argcount - 1 > max_args)
2203 {
2204 addr->transport_return = FAIL;
2205 addr->message = string_sprintf("Too many arguments to command "
2206 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2207 return FALSE;
2208 }
2209
2210 /* If we are not just able to replace the slot that contained
2211 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2212 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2213 * Visually if address_pipe_argcount == 2:
2214 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2215 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2216 */
2217 if (address_pipe_argcount > 1)
2218 memmove(
4c04137d 2219 /* current position + additional args */
09792322
NK
2220 argv + i + address_pipe_argcount,
2221 /* current position + 1 (for the (uschar *)0 at the end) */
2222 argv + i + 1,
2223 /* -1 for the (uschar *)0 at the end)*/
2224 (argcount - i)*sizeof(uschar *)
2225 );
2226
2227 /* Now we fill in the slots we just moved argv out of
2228 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2229 */
700d22f3
PP
2230 for (address_pipe_i = 0;
2231 address_pipe_argv[address_pipe_i] != (uschar *)0;
2232 address_pipe_i++)
2233 {
2234 argv[i++] = address_pipe_argv[address_pipe_i];
2235 argcount++;
09792322
NK
2236 }
2237
700d22f3
PP
2238 /* Subtract one since we replace $address_pipe */
2239 argcount--;
2240 i--;
059ec3d9
PH
2241 }
2242
2243 /* Handle normal expansion string */
2244
2245 else
2246 {
55414b25 2247 const uschar *expanded_arg;
059ec3d9 2248 enable_dollar_recipients = allow_dollar_recipients;
55414b25 2249 expanded_arg = expand_cstring(argv[i]);
059ec3d9
PH
2250 enable_dollar_recipients = FALSE;
2251
2252 if (expanded_arg == NULL)
2253 {
2254 uschar *msg = string_sprintf("Expansion of \"%s\" "
2255 "from command \"%s\" in %s failed: %s",
2256 argv[i], cmd, etext, expand_string_message);
2257 if (addr != NULL)
2258 {
2259 addr->transport_return = expand_failed;
2260 addr->message = msg;
2261 }
2262 else *errptr = msg;
2263 return FALSE;
2264 }
2265 argv[i] = expanded_arg;
2266 }
2267 }
2268
2269 DEBUG(D_transport)
2270 {
2271 debug_printf("direct command after expansion:\n");
2272 for (i = 0; argv[i] != (uschar *)0; i++)
2273 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2274 }
2275 }
2276
2277return TRUE;
2278}
2279
d185889f 2280#endif /*!MACRO_PREDEF*/
511a6c14
JH
2281/* vi: aw ai sw=2
2282*/
059ec3d9 2283/* End of transport.c */