Avoid re-expansion in ${sort }
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* General functions concerned with transportation, and generic options for all
9transports. */
10
11
12#include "exim.h"
13
059ec3d9
PH
14/* Generic options for transports, all of which live inside transport_instance
15data blocks and which therefore have the opt_public flag set. Note that there
16are other options living inside this structure which can be set only from
17certain transports. */
18
19optionlist optionlist_transports[] = {
f2ed27cf 20 /* name type value */
059ec3d9
PH
21 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
22 (void *)offsetof(transport_instance, expand_gid) },
23 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
24 (void *)offsetof(transport_instance, expand_uid) },
25 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
26 (void *)offsetof(transport_instance, rewrite_existflags) },
27 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
28 (void *)offsetof(transport_instance, rewrite_rules) },
29 { "*set_group", opt_bool|opt_hidden|opt_public,
30 (void *)offsetof(transport_instance, gid_set) },
31 { "*set_user", opt_bool|opt_hidden|opt_public,
32 (void *)offsetof(transport_instance, uid_set) },
33 { "body_only", opt_bool|opt_public,
34 (void *)offsetof(transport_instance, body_only) },
35 { "current_directory", opt_stringptr|opt_public,
36 (void *)offsetof(transport_instance, current_dir) },
37 { "debug_print", opt_stringptr | opt_public,
38 (void *)offsetof(transport_instance, debug_string) },
39 { "delivery_date_add", opt_bool|opt_public,
40 (void *)(offsetof(transport_instance, delivery_date_add)) },
41 { "disable_logging", opt_bool|opt_public,
42 (void *)(offsetof(transport_instance, disable_logging)) },
43 { "driver", opt_stringptr|opt_public,
44 (void *)offsetof(transport_instance, driver_name) },
45 { "envelope_to_add", opt_bool|opt_public,
46 (void *)(offsetof(transport_instance, envelope_to_add)) },
0cbf2b82 47#ifndef DISABLE_EVENT
774ef2d7
JH
48 { "event_action", opt_stringptr | opt_public,
49 (void *)offsetof(transport_instance, event_action) },
50#endif
059ec3d9
PH
51 { "group", opt_expand_gid|opt_public,
52 (void *)offsetof(transport_instance, gid) },
846726c5 53 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
54 (void *)offsetof(transport_instance, add_headers) },
55 { "headers_only", opt_bool|opt_public,
56 (void *)offsetof(transport_instance, headers_only) },
846726c5 57 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
58 (void *)offsetof(transport_instance, remove_headers) },
59 { "headers_rewrite", opt_rewrite|opt_public,
60 (void *)offsetof(transport_instance, headers_rewrite) },
61 { "home_directory", opt_stringptr|opt_public,
62 (void *)offsetof(transport_instance, home_dir) },
63 { "initgroups", opt_bool|opt_public,
64 (void *)offsetof(transport_instance, initgroups) },
fa41615d
JH
65 { "max_parallel", opt_stringptr|opt_public,
66 (void *)offsetof(transport_instance, max_parallel) },
059ec3d9
PH
67 { "message_size_limit", opt_stringptr|opt_public,
68 (void *)offsetof(transport_instance, message_size_limit) },
69 { "rcpt_include_affixes", opt_bool|opt_public,
70 (void *)offsetof(transport_instance, rcpt_include_affixes) },
71 { "retry_use_local_part", opt_bool|opt_public,
72 (void *)offsetof(transport_instance, retry_use_local_part) },
73 { "return_path", opt_stringptr|opt_public,
74 (void *)(offsetof(transport_instance, return_path)) },
75 { "return_path_add", opt_bool|opt_public,
76 (void *)(offsetof(transport_instance, return_path_add)) },
77 { "shadow_condition", opt_stringptr|opt_public,
78 (void *)offsetof(transport_instance, shadow_condition) },
79 { "shadow_transport", opt_stringptr|opt_public,
80 (void *)offsetof(transport_instance, shadow) },
81 { "transport_filter", opt_stringptr|opt_public,
82 (void *)offsetof(transport_instance, filter_command) },
83 { "transport_filter_timeout", opt_time|opt_public,
84 (void *)offsetof(transport_instance, filter_timeout) },
85 { "user", opt_expand_uid|opt_public,
86 (void *)offsetof(transport_instance, uid) }
87};
88
c0b9d3e8 89int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9 90
d185889f
JH
91#ifdef MACRO_PREDEF
92
5f69a529 93# include "macro_predef.h"
059ec3d9 94
c0b9d3e8 95void
d185889f 96options_transports(void)
c0b9d3e8 97{
d185889f 98uschar buf[64];
c0b9d3e8 99
d185889f 100options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
c0b9d3e8 101
d7978c0f 102for (transport_info * ti = transports_available; ti->driver_name[0]; ti++)
4945f557 103 {
cab0c277 104 spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", ti->driver_name);
d185889f
JH
105 builtin_macro_create(buf);
106 options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
4945f557 107 }
383832ef 108}
c0b9d3e8 109
d185889f
JH
110#else /*!MACRO_PREDEF*/
111
112/* Structure for keeping list of addresses that have been added to
113Envelope-To:, in order to avoid duplication. */
114
115struct aci {
116 struct aci *next;
117 address_item *ptr;
118 };
119
120
121/* Static data for write_chunk() */
122
123static uschar *chunk_ptr; /* chunk pointer */
124static uschar *nl_check; /* string to look for at line start */
125static int nl_check_length; /* length of same */
126static uschar *nl_escape; /* string to insert */
127static int nl_escape_length; /* length of same */
128static int nl_partial_match; /* length matched at chunk end */
129
130
059ec3d9
PH
131/*************************************************
132* Initialize transport list *
133*************************************************/
134
135/* Read the transports section of the configuration file, and set up a chain of
136transport instances according to its contents. Each transport has generic
137options and may also have its own private options. This function is only ever
138called when transports == NULL. We use generic code in readconf to do most of
139the work. */
140
141void
142transport_init(void)
143{
059ec3d9
PH
144readconf_driver_init(US"transport",
145 (driver_instance **)(&transports), /* chain anchor */
146 (driver_info *)transports_available, /* available drivers */
147 sizeof(transport_info), /* size of info block */
148 &transport_defaults, /* default values for generic options */
149 sizeof(transport_instance), /* size of instance block */
150 optionlist_transports, /* generic options */
151 optionlist_transports_size);
152
153/* Now scan the configured transports and check inconsistencies. A shadow
154transport is permitted only for local transports. */
155
d7978c0f 156for (transport_instance * t = transports; t; t = t->next)
059ec3d9 157 {
7b4c8c1f
JH
158 if (!t->info->local && t->shadow)
159 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
160 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
161
162 if (t->body_only && t->headers_only)
163 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
164 "%s transport: body_only and headers_only are mutually exclusive",
165 t->name);
166 }
167}
168
169
170
171/*************************************************
172* Write block of data *
173*************************************************/
174
175/* Subroutine called by write_chunk() and at the end of the message actually
176to write a data block. Also called directly by some transports to write
177additional data to the file descriptor (e.g. prefix, suffix).
178
179If a transport wants data transfers to be timed, it sets a non-zero value in
180transport_write_timeout. A non-zero transport_write_timeout causes a timer to
181be set for each block of data written from here. If time runs out, then write()
182fails and provokes an error return. The caller can then inspect sigalrm_seen to
183check for a timeout.
184
185On some systems, if a quota is exceeded during the write, the yield is the
186number of bytes written rather than an immediate error code. This also happens
187on some systems in other cases, for example a pipe that goes away because the
188other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
189get the error codes the first time.
190
191The write() function is also interruptible; the Solaris 2.6 man page says:
192
193 If write() is interrupted by a signal before it writes any
194 data, it will return -1 with errno set to EINTR.
195
196 If write() is interrupted by a signal after it successfully
197 writes some data, it will return the number of bytes written.
198
199To handle these cases, we want to restart the write() to output the remainder
200of the data after a non-negative return from write(), except after a timeout.
201In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
202proper error then occurs. In principle, after an interruption, the second
203write() could suffer the same fate, but we do not want to continue for
204evermore, so stick a maximum repetition count on the loop to act as a
205longstop.
206
207Arguments:
42055a33 208 tctx transport context: file descriptor or string to write to
059ec3d9
PH
209 block block of bytes to write
210 len number of bytes to write
925ac8e4 211 more further data expected soon
059ec3d9
PH
212
213Returns: TRUE on success, FALSE on failure (with errno preserved);
214 transport_count is incremented by the number of bytes written
215*/
216
42055a33
JH
217static BOOL
218transport_write_block_fd(transport_ctx * tctx, uschar *block, int len, BOOL more)
059ec3d9 219{
d7978c0f 220int rc, save_errno;
958541e9 221int local_timeout = transport_write_timeout;
42055a33 222int fd = tctx->u.fd;
958541e9
PH
223
224/* This loop is for handling incomplete writes and other retries. In most
225normal cases, it is only ever executed once. */
059ec3d9 226
d7978c0f 227for (int i = 0; i < 100; i++)
059ec3d9
PH
228 {
229 DEBUG(D_transport)
42055a33
JH
230 debug_printf("writing data block fd=%d size=%d timeout=%d%s\n",
231 fd, len, local_timeout, more ? " (more expected)" : "");
059ec3d9 232
958541e9
PH
233 /* This code makes use of alarm() in order to implement the timeout. This
234 isn't a very tidy way of doing things. Using non-blocking I/O with select()
235 provides a neater approach. However, I don't know how to do this when TLS is
236 in use. */
059ec3d9 237
958541e9
PH
238 if (transport_write_timeout <= 0) /* No timeout wanted */
239 {
42055a33 240 rc =
01603eec 241#ifndef DISABLE_TLS
74f1a423 242 tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
42055a33
JH
243#endif
244#ifdef MSG_MORE
b3b37076
JH
245 more && !(tctx->options & topt_not_socket)
246 ? send(fd, block, len, MSG_MORE) :
42055a33
JH
247#endif
248 write(fd, block, len);
958541e9
PH
249 save_errno = errno;
250 }
059ec3d9 251
958541e9 252 /* Timeout wanted. */
059ec3d9 253
958541e9 254 else
059ec3d9 255 {
c2a1bba0 256 ALARM(local_timeout);
42055a33
JH
257
258 rc =
01603eec 259#ifndef DISABLE_TLS
74f1a423 260 tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
42055a33
JH
261#endif
262#ifdef MSG_MORE
e01d111e
JH
263 more && !(tctx->options & topt_not_socket)
264 ? send(fd, block, len, MSG_MORE) :
59932f7d 265#endif
42055a33
JH
266 write(fd, block, len);
267
958541e9 268 save_errno = errno;
c2a1bba0 269 local_timeout = ALARM_CLR(0);
059ec3d9
PH
270 if (sigalrm_seen)
271 {
272 errno = ETIMEDOUT;
273 return FALSE;
274 }
275 }
276
277 /* Hopefully, the most common case is success, so test that first. */
278
279 if (rc == len) { transport_count += len; return TRUE; }
280
958541e9
PH
281 /* A non-negative return code is an incomplete write. Try again for the rest
282 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
283
284 if (rc >= 0)
285 {
286 len -= rc;
287 block += rc;
288 transport_count += rc;
289 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 290 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
291 }
292
293 /* A negative return code with an EINTR error is another form of
294 incomplete write, zero bytes having been written */
295
296 if (save_errno == EINTR)
297 {
298 DEBUG(D_transport)
299 debug_printf("write interrupted before anything written\n");
958541e9 300 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
301 }
302
303 /* A response of EAGAIN from write() is likely only in the case of writing
304 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
305
306 if (save_errno == EAGAIN)
307 {
308 DEBUG(D_transport)
309 debug_printf("write temporarily locked out, waiting 1 sec\n");
310 sleep(1);
958541e9
PH
311
312 /* Before continuing to try another write, check that we haven't run out of
313 time. */
314
315 CHECK_TIMEOUT:
316 if (transport_write_timeout > 0 && local_timeout <= 0)
317 {
318 errno = ETIMEDOUT;
319 return FALSE;
320 }
059ec3d9
PH
321 continue;
322 }
323
324 /* Otherwise there's been an error */
325
326 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
327 strerror(save_errno));
328 errno = save_errno;
329 return FALSE;
330 }
331
332/* We've tried and tried and tried but still failed */
333
334errno = ERRNO_WRITEINCOMPLETE;
335return FALSE;
336}
337
338
42055a33
JH
339BOOL
340transport_write_block(transport_ctx * tctx, uschar *block, int len, BOOL more)
341{
342if (!(tctx->options & topt_output_string))
343 return transport_write_block_fd(tctx, block, len, more);
344
345/* Write to expanding-string. NOTE: not NUL-terminated */
346
347if (!tctx->u.msg)
acec9514 348 tctx->u.msg = string_get(1024);
42055a33 349
acec9514 350tctx->u.msg = string_catn(tctx->u.msg, block, len);
42055a33
JH
351return TRUE;
352}
353
354
059ec3d9
PH
355
356
357/*************************************************
358* Write formatted string *
359*************************************************/
360
361/* This is called by various transports. It is a convenience function.
362
363Arguments:
364 fd file descriptor
365 format string format
366 ... arguments for format
367
368Returns: the yield of transport_write_block()
369*/
370
371BOOL
1ba28e2b 372transport_write_string(int fd, const char *format, ...)
059ec3d9 373{
cab0c277 374transport_ctx tctx = {{0}};
d12746bc 375gstring gs = { .size = big_buffer_size, .ptr = 0, .s = big_buffer };
059ec3d9 376va_list ap;
d12746bc 377
059ec3d9 378va_start(ap, format);
d12746bc 379if (!string_vformat(&gs, FALSE, format, ap))
059ec3d9
PH
380 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
381va_end(ap);
42055a33 382tctx.u.fd = fd;
d12746bc 383return transport_write_block(&tctx, gs.s, gs.ptr, FALSE);
059ec3d9
PH
384}
385
386
387
388
42055a33
JH
389void
390transport_write_reset(int options)
391{
392if (!(options & topt_continuation)) chunk_ptr = deliver_out_buffer;
393nl_partial_match = -1;
394nl_check_length = nl_escape_length = 0;
395}
396
397
398
059ec3d9
PH
399/*************************************************
400* Write character chunk *
401*************************************************/
402
403/* Subroutine used by transport_write_message() to scan character chunks for
404newlines and act appropriately. The object is to minimise the number of writes.
405The output byte stream is buffered up in deliver_out_buffer, which is written
406only when it gets full, thus minimizing write operations and TCP packets.
407
408Static data is used to handle the case when the last character of the previous
409chunk was NL, or matched part of the data that has to be escaped.
410
411Arguments:
42055a33
JH
412 tctx transport context - processing to be done during output,
413 and file descriptor to write to
059ec3d9
PH
414 chunk pointer to data to write
415 len length of data to write
059ec3d9
PH
416
417In addition, the static nl_xxx variables must be set as required.
418
419Returns: TRUE on success, FALSE on failure (with errno preserved)
420*/
421
42055a33
JH
422BOOL
423write_chunk(transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
424{
425uschar *start = chunk;
426uschar *end = chunk + len;
059ec3d9
PH
427int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
428
429/* The assumption is made that the check string will never stretch over move
430than one chunk since the only time there are partial matches is when copying
431the body in large buffers. There is always enough room in the buffer for an
432escape string, since the loop below ensures this for each character it
433processes, and it won't have stuck in the escape string if it left a partial
434match. */
435
436if (nl_partial_match >= 0)
437 {
438 if (nl_check_length > 0 && len >= nl_check_length &&
439 Ustrncmp(start, nl_check + nl_partial_match,
440 nl_check_length - nl_partial_match) == 0)
441 {
442 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
443 chunk_ptr += nl_escape_length;
444 start += nl_check_length - nl_partial_match;
445 }
446
447 /* The partial match was a false one. Insert the characters carried over
448 from the previous chunk. */
449
450 else if (nl_partial_match > 0)
451 {
452 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
453 chunk_ptr += nl_partial_match;
454 }
455
456 nl_partial_match = -1;
457 }
458
459/* Now process the characters in the chunk. Whenever we hit a newline we check
460for possible escaping. The code for the non-NL route should be as fast as
461possible. */
462
d7978c0f 463for (uschar * ptr = start; ptr < end; ptr++)
059ec3d9 464 {
6d5c916c 465 int ch, len;
059ec3d9
PH
466
467 /* Flush the buffer if it has reached the threshold - we want to leave enough
468 room for the next uschar, plus a possible extra CR for an LF, plus the escape
469 string. */
470
6d5c916c 471 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 472 {
58fc5fb2
JH
473 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
474
6d5c916c
JH
475 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
476 from previous SMTP commands. */
477
d315eda1 478 if (tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2 479 {
d2aa036b 480 if ( tctx->chunk_cb(tctx, (unsigned)len, 0) != OK
42055a33 481 || !transport_write_block(tctx, deliver_out_buffer, len, FALSE)
d2aa036b 482 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
58fc5fb2
JH
483 )
484 return FALSE;
485 }
486 else
42055a33 487 if (!transport_write_block(tctx, deliver_out_buffer, len, FALSE))
6d5c916c 488 return FALSE;
059ec3d9
PH
489 chunk_ptr = deliver_out_buffer;
490 }
491
328c5688
JH
492 /* Remove CR before NL if required */
493
494 if ( *ptr == '\r' && ptr[1] == '\n'
d315eda1 495 && !(tctx->options & topt_use_crlf)
8768d548 496 && f.spool_file_wireformat
328c5688
JH
497 )
498 ptr++;
499
059ec3d9
PH
500 if ((ch = *ptr) == '\n')
501 {
502 int left = end - ptr - 1; /* count of chars left after NL */
503
504 /* Insert CR before NL if required */
505
8768d548 506 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
328c5688 507 *chunk_ptr++ = '\r';
059ec3d9 508 *chunk_ptr++ = '\n';
332f5cf3 509 transport_newlines++;
059ec3d9
PH
510
511 /* The check_string test (formerly "from hack") replaces the specific
512 string at the start of a line with an escape string (e.g. "From " becomes
513 ">From " or "." becomes "..". It is a case-sensitive test. The length
514 check above ensures there is always enough room to insert this string. */
515
516 if (nl_check_length > 0)
517 {
518 if (left >= nl_check_length &&
519 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
520 {
521 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
522 chunk_ptr += nl_escape_length;
523 ptr += nl_check_length;
524 }
525
526 /* Handle the case when there isn't enough left to match the whole
527 check string, but there may be a partial match. We remember how many
528 characters matched, and finish processing this chunk. */
529
530 else if (left <= 0) nl_partial_match = 0;
531
532 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
533 {
534 nl_partial_match = left;
535 ptr = end;
536 }
537 }
538 }
539
540 /* Not a NL character */
541
542 else *chunk_ptr++ = ch;
543 }
544
545return TRUE;
546}
547
548
549
550
551/*************************************************
552* Generate address for RCPT TO *
553*************************************************/
554
555/* This function puts together an address for RCPT to, using the caseful
556version of the local part and the caseful version of the domain. If there is no
557prefix or suffix, or if affixes are to be retained, we can just use the
558original address. Otherwise, if there is a prefix but no suffix we can use a
559pointer into the original address. If there is a suffix, however, we have to
560build a new string.
561
562Arguments:
563 addr the address item
564 include_affixes TRUE if affixes are to be included
565
566Returns: a string
567*/
568
569uschar *
570transport_rcpt_address(address_item *addr, BOOL include_affixes)
571{
572uschar *at;
573int plen, slen;
574
575if (include_affixes)
576 {
577 setflag(addr, af_include_affixes); /* Affects logged => line */
578 return addr->address;
579 }
580
581if (addr->suffix == NULL)
582 {
583 if (addr->prefix == NULL) return addr->address;
584 return addr->address + Ustrlen(addr->prefix);
585 }
586
587at = Ustrrchr(addr->address, '@');
588plen = (addr->prefix == NULL)? 0 : Ustrlen(addr->prefix);
589slen = Ustrlen(addr->suffix);
590
bb07bcd3 591return string_sprintf("%.*s@%s", (int)(at - addr->address - plen - slen),
059ec3d9
PH
592 addr->address + plen, at + 1);
593}
594
595
596/*************************************************
597* Output Envelope-To: address & scan duplicates *
598*************************************************/
599
600/* This function is called from internal_transport_write_message() below, when
601generating an Envelope-To: header line. It checks for duplicates of the given
602address and its ancestors. When one is found, this function calls itself
603recursively, to output the envelope address of the duplicate.
604
605We want to avoid duplication in the list, which can arise for example when
606A->B,C and then both B and C alias to D. This can also happen when there are
607unseen drivers in use. So a list of addresses that have been output is kept in
608the plist variable.
609
610It is also possible to have loops in the address ancestry/duplication graph,
611for example if there are two top level addresses A and B and we have A->B,C and
612B->A. To break the loop, we use a list of processed addresses in the dlist
613variable.
614
615After handling duplication, this function outputs the progenitor of the given
616address.
617
618Arguments:
619 p the address we are interested in
620 pplist address of anchor of the list of addresses not to output
621 pdlist address of anchor of the list of processed addresses
622 first TRUE if this is the first address; set it FALSE afterwards
6d5c916c 623 tctx transport context - processing to be done during output
42055a33 624 and the file descriptor to write to
059ec3d9
PH
625
626Returns: FALSE if writing failed
627*/
628
629static BOOL
630write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
42055a33 631 BOOL *first, transport_ctx * tctx)
059ec3d9
PH
632{
633address_item *pp;
634struct aci *ppp;
635
636/* Do nothing if we have already handled this address. If not, remember it
637so that we don't handle it again. */
638
ff5aac2b 639for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9
PH
640
641ppp = store_get(sizeof(struct aci));
642ppp->next = *pdlist;
643*pdlist = ppp;
644ppp->ptr = p;
645
646/* Now scan up the ancestry, checking for duplicates at each generation. */
647
648for (pp = p;; pp = pp->parent)
649 {
650 address_item *dup;
ff5aac2b
JH
651 for (dup = addr_duplicate; dup; dup = dup->next)
652 if (dup->dupof == pp) /* a dup of our address */
42055a33 653 if (!write_env_to(dup, pplist, pdlist, first, tctx))
ff5aac2b
JH
654 return FALSE;
655 if (!pp->parent) break;
059ec3d9
PH
656 }
657
658/* Check to see if we have already output the progenitor. */
659
ff5aac2b
JH
660for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
661if (ppp) return TRUE;
059ec3d9
PH
662
663/* Remember what we have output, and output it. */
664
665ppp = store_get(sizeof(struct aci));
666ppp->next = *pplist;
667*pplist = ppp;
668ppp->ptr = pp;
669
42055a33 670if (!*first && !write_chunk(tctx, US",\n ", 3)) return FALSE;
059ec3d9 671*first = FALSE;
42055a33 672return write_chunk(tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
673}
674
675
676
677
4c04137d 678/* Add/remove/rewrite headers, and send them plus the empty-line separator.
511a6c14
JH
679
680Globals:
681 header_list
682
683Arguments:
684 addr (chain of) addresses (for extra headers), or NULL;
685 only the first address is used
44bc8f0c 686 tctx transport context
59932f7d 687 sendfn function for output (transport or verify)
511a6c14
JH
688
689Returns: TRUE on success; FALSE on failure.
690*/
691BOOL
42055a33
JH
692transport_headers_send(transport_ctx * tctx,
693 BOOL (*sendfn)(transport_ctx * tctx, uschar * s, int len))
511a6c14 694{
6b46ecc6 695const uschar *list;
6d5c916c
JH
696transport_instance * tblock = tctx ? tctx->tblock : NULL;
697address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
698
699/* Then the message's headers. Don't write any that are flagged as "old";
700that means they were rewritten, or are a record of envelope rewriting, or
701were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
702match any entries therein. It is a colon-sep list; expand the items
703separately and squash any empty ones.
d43cbe25 704Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 705
d7978c0f 706for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14 707 {
511a6c14
JH
708 BOOL include_header = TRUE;
709
fb6833e0 710 list = tblock ? tblock->remove_headers : NULL;
d7978c0f 711 for (int i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
712 {
713 if (list)
714 {
715 int sep = ':'; /* This is specified as a colon-separated list */
716 uschar *s, *ss;
8bc732e8 717 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 718 {
76146973
JH
719 int len;
720
721 if (i == 0)
8768d548 722 if (!(s = expand_string(s)) && !f.expand_string_forcedfail)
76146973
JH
723 {
724 errno = ERRNO_CHHEADER_FAIL;
725 return FALSE;
726 }
d4ff61d1 727 len = s ? Ustrlen(s) : 0;
511a6c14
JH
728 if (strncmpic(h->text, s, len) != 0) continue;
729 ss = h->text + len;
730 while (*ss == ' ' || *ss == '\t') ss++;
731 if (*ss == ':') break;
732 }
6b46ecc6 733 if (s) { include_header = FALSE; break; }
511a6c14 734 }
6b46ecc6 735 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
736 }
737
738 /* If this header is to be output, try to rewrite it if there are rewriting
739 rules. */
740
741 if (include_header)
742 {
fb6833e0 743 if (tblock && tblock->rewrite_rules)
511a6c14
JH
744 {
745 void *reset_point = store_get(0);
746 header_line *hh;
747
6b46ecc6
JH
748 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
749 tblock->rewrite_existflags, FALSE)))
511a6c14 750 {
42055a33 751 if (!sendfn(tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
752 store_reset(reset_point);
753 continue; /* With the next header line */
754 }
755 }
756
757 /* Either no rewriting rules, or it didn't get rewritten */
758
42055a33 759 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
760 }
761
762 /* Header removed */
763
764 else
511a6c14 765 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
511a6c14
JH
766 }
767
768/* Add on any address-specific headers. If there are multiple addresses,
769they will all have the same headers in order to be batched. The headers
770are chained in reverse order of adding (so several addresses from the
771same alias might share some of them) but we want to output them in the
772opposite order. This is a bit tedious, but there shouldn't be very many
773of them. We just walk the list twice, reversing the pointers each time,
774but on the second time, write out the items.
775
776Headers added to an address by a router are guaranteed to end with a newline.
777*/
778
779if (addr)
780 {
d43cbe25 781 header_line *hprev = addr->prop.extra_headers;
d7978c0f
JH
782 header_line *hnext, * h;
783 for (int i = 0; i < 2; i++)
ff5aac2b 784 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
785 {
786 hnext = h->next;
787 h->next = hprev;
788 hprev = h;
789 if (i == 1)
790 {
42055a33 791 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
792 DEBUG(D_transport)
793 debug_printf("added header line(s):\n%s---\n", h->text);
794 }
795 }
511a6c14
JH
796 }
797
76146973
JH
798/* If a string containing additional headers exists it is a newline-sep
799list. Expand each item and write out the result. This is done last so that
800if it (deliberately or accidentally) isn't in header format, it won't mess
801up any other headers. An empty string or a forced expansion failure are
802noops. An added header string from a transport may not end with a newline;
803add one if it does not. */
511a6c14 804
fb6833e0 805if (tblock && (list = CUS tblock->add_headers))
511a6c14 806 {
76146973
JH
807 int sep = '\n';
808 uschar * s;
809
6b46ecc6 810 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 811 if ((s = expand_string(s)))
76146973
JH
812 {
813 int len = Ustrlen(s);
814 if (len > 0)
511a6c14 815 {
42055a33
JH
816 if (!sendfn(tctx, s, len)) return FALSE;
817 if (s[len-1] != '\n' && !sendfn(tctx, US"\n", 1))
76146973
JH
818 return FALSE;
819 DEBUG(D_transport)
820 {
821 debug_printf("added header line:\n%s", s);
822 if (s[len-1] != '\n') debug_printf("\n");
823 debug_printf("---\n");
824 }
511a6c14
JH
825 }
826 }
8768d548 827 else if (!f.expand_string_forcedfail)
ff5aac2b 828 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
829 }
830
831/* Separate headers from body with a blank line */
832
42055a33 833return sendfn(tctx, US"\n", 1);
511a6c14
JH
834}
835
836
059ec3d9
PH
837/*************************************************
838* Write the message *
839*************************************************/
840
841/* This function writes the message to the given file descriptor. The headers
842are in the in-store data structure, and the rest of the message is in the open
843file descriptor deliver_datafile. Make sure we start it at the beginning.
844
845. If add_return_path is TRUE, a "return-path:" header is added to the message,
846 containing the envelope sender's address.
847
848. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
849 giving the top-level envelope address that caused this delivery to happen.
850
851. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
852 message. It gives the time and date that delivery took place.
853
854. If check_string is not null, the start of each line is checked for that
855 string. If it is found, it is replaced by escape_string. This used to be
856 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
857
858. If use_crlf is true, newlines are turned into CRLF (SMTP output).
859
860The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
861any writing or reading error, leaving the code in errno intact. Error exits
862can include timeouts for certain transports, which are requested by setting
863transport_write_timeout non-zero.
864
865Arguments:
6b46ecc6 866 tctx
42055a33
JH
867 (fd, msg) Either and fd, to write the message to,
868 or a string: if null write message to allocated space
869 otherwire take content as headers.
6b46ecc6
JH
870 addr (chain of) addresses (for extra headers), or NULL;
871 only the first address is used
872 tblock optional transport instance block (NULL signifies NULL/0):
873 add_headers a string containing one or more headers to add; it is
874 expanded, and must be in correct RFC 822 format as
875 it is transmitted verbatim; NULL => no additions,
876 and so does empty string or forced expansion fail
877 remove_headers a colon-separated list of headers to remove, or NULL
878 rewrite_rules chain of header rewriting rules
879 rewrite_existflags flags for the rewriting rules
880 options bit-wise options:
881 add_return_path if TRUE, add a "return-path" header
882 add_envelope_to if TRUE, add a "envelope-to" header
883 add_delivery_date if TRUE, add a "delivery-date" header
884 use_crlf if TRUE, turn NL into CR LF
885 end_dot if TRUE, send a terminating "." line at the end
886 no_headers if TRUE, omit the headers
887 no_body if TRUE, omit the body
2d14f397
JH
888 check_string a string to check for at the start of lines, or NULL
889 escape_string a string to insert in front of any check string
890 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
891 it is used when returning messages to their senders,
892 and is approximate rather than exact, owing to chunk
893 buffering
059ec3d9
PH
894
895Returns: TRUE on success; FALSE (with errno) on failure.
896 In addition, the global variable transport_count
897 is incremented by the number of bytes written.
898*/
899
d315eda1 900static BOOL
42055a33 901internal_transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 902{
328c5688 903int len, size = 0;
059ec3d9
PH
904
905/* Initialize pointer in output buffer. */
906
42055a33 907transport_write_reset(tctx->options);
059ec3d9
PH
908
909/* Set up the data for start-of-line data checking and escaping */
910
6b46ecc6 911if (tctx->check_string && tctx->escape_string)
059ec3d9 912 {
6b46ecc6 913 nl_check = tctx->check_string;
059ec3d9 914 nl_check_length = Ustrlen(nl_check);
6b46ecc6 915 nl_escape = tctx->escape_string;
059ec3d9
PH
916 nl_escape_length = Ustrlen(nl_escape);
917 }
059ec3d9 918
328c5688
JH
919/* Whether the escaping mechanism is applied to headers or not is controlled by
920an option (set for SMTP, not otherwise). Negate the length if not wanted till
921after the headers. */
922
923if (!(tctx->options & topt_escape_headers))
924 nl_check_length = -nl_check_length;
925
059ec3d9 926/* Write the headers if required, including any that have to be added. If there
328c5688
JH
927are header rewriting rules, apply them. The datasource is not the -D spoolfile
928so temporarily hide the global that adjusts for its format. */
059ec3d9 929
6b46ecc6 930if (!(tctx->options & topt_no_headers))
059ec3d9 931 {
8768d548
JH
932 BOOL save_wireformat = f.spool_file_wireformat;
933 f.spool_file_wireformat = FALSE;
42055a33 934
059ec3d9
PH
935 /* Add return-path: if requested. */
936
6b46ecc6 937 if (tctx->options & topt_add_return_path)
059ec3d9
PH
938 {
939 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 940 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 941 return_path);
328c5688 942 if (!write_chunk(tctx, buffer, n)) goto bad;
059ec3d9
PH
943 }
944
945 /* Add envelope-to: if requested */
946
6b46ecc6 947 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
948 {
949 BOOL first = TRUE;
059ec3d9
PH
950 struct aci *plist = NULL;
951 struct aci *dlist = NULL;
952 void *reset_point = store_get(0);
953
328c5688 954 if (!write_chunk(tctx, US"Envelope-to: ", 13)) goto bad;
059ec3d9
PH
955
956 /* Pick up from all the addresses. The plist and dlist variables are
957 anchors for lists of addresses already handled; they have to be defined at
4c04137d 958 this level because write_env_to() calls itself recursively. */
059ec3d9 959
d7978c0f
JH
960 for (address_item * p = tctx->addr; p; p = p->next)
961 if (!write_env_to(p, &plist, &dlist, &first, tctx))
962 goto bad;
059ec3d9
PH
963
964 /* Add a final newline and reset the store used for tracking duplicates */
965
328c5688 966 if (!write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
967 store_reset(reset_point);
968 }
969
970 /* Add delivery-date: if requested. */
971
6b46ecc6 972 if (tctx->options & topt_add_delivery_date)
059ec3d9 973 {
a5ffa9b4
JH
974 uschar * s = tod_stamp(tod_full);
975
976 if ( !write_chunk(tctx, US"Delivery-date: ", 15)
977 || !write_chunk(tctx, s, Ustrlen(s))
978 || !write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
979 }
980
981 /* Then the message's headers. Don't write any that are flagged as "old";
982 that means they were rewritten, or are a record of envelope rewriting, or
983 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 984 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 985 addr is not NULL. */
6b46ecc6 986
42055a33 987 if (!transport_headers_send(tctx, &write_chunk))
328c5688
JH
988 {
989bad:
8768d548 990 f.spool_file_wireformat = save_wireformat;
511a6c14 991 return FALSE;
328c5688
JH
992 }
993
8768d548 994 f.spool_file_wireformat = save_wireformat;
059ec3d9
PH
995 }
996
58fc5fb2
JH
997/* When doing RFC3030 CHUNKING output, work out how much data would be in a
998last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
999(optimally, all of the headers - but it does not matter if we already had to
1000flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 1001(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
1002that further use of write_chunk() will not prepend BDATs.
1003The first BDAT written will also first flush any outstanding MAIL and RCPT
1004commands which were buffered thans to PIPELINING.
1005Commands go out (using a send()) from a different buffer to data (using a
1006write()). They might not end up in the same TCP segment, which is
1007suboptimal. */
59932f7d 1008
65de12cc 1009if (tctx->options & topt_use_bdat)
59932f7d 1010 {
e027f545 1011 off_t fsize;
328c5688 1012 int hsize;
e027f545
JH
1013
1014 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
1015 hsize = 0;
65de12cc 1016 if (!(tctx->options & topt_no_body))
59932f7d
JH
1017 {
1018 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
1019 fsize -= SPOOL_DATA_START_OFFSET;
1020 if (size_limit > 0 && fsize > size_limit)
1021 fsize = size_limit;
e027f545 1022 size = hsize + fsize;
8768d548 1023 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
59932f7d 1024 size += body_linecount; /* account for CRLF-expansion */
42055a33
JH
1025
1026 /* With topt_use_bdat we never do dot-stuffing; no need to
1027 account for any expansion due to that. */
59932f7d
JH
1028 }
1029
e027f545
JH
1030 /* If the message is large, emit first a non-LAST chunk with just the
1031 headers, and reap the command responses. This lets us error out early
1032 on RCPT rejects rather than sending megabytes of data. Include headers
1033 on the assumption they are cheap enough and some clever implementations
1034 might errorcheck them too, on-the-fly, and reject that chunk. */
1035
1036 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
1037 {
58fc5fb2 1038 DEBUG(D_transport)
4c04137d 1039 debug_printf("sending small initial BDAT; hsize=%d\n", hsize);
d2aa036b 1040 if ( tctx->chunk_cb(tctx, hsize, 0) != OK
42055a33 1041 || !transport_write_block(tctx, deliver_out_buffer, hsize, FALSE)
d2aa036b 1042 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
e027f545
JH
1043 )
1044 return FALSE;
1045 chunk_ptr = deliver_out_buffer;
1046 size -= hsize;
1047 }
1048
d2aa036b
JH
1049 /* Emit a LAST datachunk command, and unmark the context for further
1050 BDAT commands. */
59932f7d 1051
d2aa036b 1052 if (tctx->chunk_cb(tctx, size, tc_chunk_last) != OK)
511a6c14 1053 return FALSE;
6d5c916c 1054 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
1055 }
1056
1057/* If the body is required, ensure that the data for check strings (formerly
1058the "from hack") is enabled by negating the length if necessary. (It will be
1059negative in cases where it isn't to apply to the headers). Then ensure the body
1060is positioned at the start of its file (following the message id), then write
1061it, applying the size limit if required. */
1062
328c5688
JH
1063/* If we have a wireformat -D file (CRNL lines, non-dotstuffed, no ending dot)
1064and we want to send a body without dotstuffing or ending-dot, in-clear,
1065then we can just dump it using sendfile.
1066This should get used for CHUNKING output and also for writing the -K file for
1067dkim signing, when we had CHUNKING input. */
1068
7d758a6a 1069#ifdef OS_SENDFILE
8768d548 1070if ( f.spool_file_wireformat
328c5688
JH
1071 && !(tctx->options & (topt_no_body | topt_end_dot))
1072 && !nl_check_length
74f1a423 1073 && tls_out.active.sock != tctx->u.fd
328c5688
JH
1074 )
1075 {
1076 ssize_t copied = 0;
1077 off_t offset = SPOOL_DATA_START_OFFSET;
1078
1079 /* Write out any header data in the buffer */
1080
1081 if ((len = chunk_ptr - deliver_out_buffer) > 0)
1082 {
1083 if (!transport_write_block(tctx, deliver_out_buffer, len, TRUE))
1084 return FALSE;
1085 size -= len;
1086 }
1087
1088 DEBUG(D_transport) debug_printf("using sendfile for body\n");
1089
1090 while(size > 0)
1091 {
7d758a6a 1092 if ((copied = os_sendfile(tctx->u.fd, deliver_datafile, &offset, size)) <= 0) break;
328c5688
JH
1093 size -= copied;
1094 }
1095 return copied >= 0;
1096 }
1097#else
1098DEBUG(D_transport) debug_printf("cannot use sendfile for body: no support\n");
1099#endif
1100
1101DEBUG(D_transport)
1102 if (!(tctx->options & topt_no_body))
1103 debug_printf("cannot use sendfile for body: %s\n",
8768d548 1104 !f.spool_file_wireformat ? "spoolfile not wireformat"
328c5688
JH
1105 : tctx->options & topt_end_dot ? "terminating dot wanted"
1106 : nl_check_length ? "dot- or From-stuffing wanted"
1107 : "TLS output wanted");
1108
6b46ecc6 1109if (!(tctx->options & topt_no_body))
059ec3d9 1110 {
05bf16f6 1111 unsigned long size = size_limit > 0 ? size_limit : ULONG_MAX;
e027f545 1112
059ec3d9
PH
1113 nl_check_length = abs(nl_check_length);
1114 nl_partial_match = 0;
d4ff61d1
JH
1115 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1116 return FALSE;
05bf16f6 1117 while ( (len = MIN(DELIVER_IN_BUFFER_SIZE, size)) > 0
59932f7d 1118 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1119 {
42055a33 1120 if (!write_chunk(tctx, deliver_in_buffer, len))
59932f7d 1121 return FALSE;
e027f545 1122 size -= len;
059ec3d9
PH
1123 }
1124
059ec3d9
PH
1125 /* A read error on the body will have left len == -1 and errno set. */
1126
1127 if (len != 0) return FALSE;
c0940526 1128 }
059ec3d9 1129
f64e8b5f 1130/* Finished with the check string, and spool-format consideration */
059ec3d9 1131
c0940526 1132nl_check_length = nl_escape_length = 0;
8768d548 1133f.spool_file_wireformat = FALSE;
c0940526
PP
1134
1135/* If requested, add a terminating "." line (SMTP output). */
1136
42055a33 1137if (tctx->options & topt_end_dot && !write_chunk(tctx, US".\n", 2))
c0940526 1138 return FALSE;
059ec3d9
PH
1139
1140/* Write out any remaining data in the buffer before returning. */
1141
1142return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
42055a33 1143 transport_write_block(tctx, deliver_out_buffer, len, FALSE);
f7572e5a 1144}
80a47a2c 1145
f7572e5a
TK
1146
1147
328c5688 1148
059ec3d9
PH
1149/*************************************************
1150* External interface to write the message *
1151*************************************************/
1152
1153/* If there is no filtering required, call the internal function above to do
1154the real work, passing over all the arguments from this function. Otherwise,
1155set up a filtering process, fork another process to call the internal function
1156to write to the filter, and in this process just suck from the filter and write
42055a33
JH
1157down the fd in the transport context. At the end, tidy up the pipes and the
1158processes.
059ec3d9
PH
1159
1160Arguments: as for internal_transport_write_message() above
1161
1162Returns: TRUE on success; FALSE (with errno) for any failure
1163 transport_count is incremented by the number of bytes written
1164*/
1165
1166BOOL
42055a33 1167transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 1168{
059ec3d9 1169BOOL last_filter_was_NL = TRUE;
8768d548 1170BOOL save_spool_file_wireformat = f.spool_file_wireformat;
059ec3d9 1171int rc, len, yield, fd_read, fd_write, save_errno;
806c3df9 1172int pfd[2] = {-1, -1};
059ec3d9 1173pid_t filter_pid, write_pid;
6b46ecc6 1174
8768d548 1175f.transport_filter_timed_out = FALSE;
2e2a30b4 1176
059ec3d9
PH
1177/* If there is no filter command set up, call the internal function that does
1178the actual work, passing it the incoming fd, and return its result. */
1179
6ff55e50
JH
1180if ( !transport_filter_argv
1181 || !*transport_filter_argv
1182 || !**transport_filter_argv
1183 )
42055a33 1184 return internal_transport_write_message(tctx, size_limit);
059ec3d9
PH
1185
1186/* Otherwise the message must be written to a filter process and read back
1187before being written to the incoming fd. First set up the special processing to
1188be done during the copying. */
1189
059ec3d9
PH
1190nl_partial_match = -1;
1191
6b46ecc6 1192if (tctx->check_string && tctx->escape_string)
059ec3d9 1193 {
6b46ecc6 1194 nl_check = tctx->check_string;
059ec3d9 1195 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1196 nl_escape = tctx->escape_string;
059ec3d9
PH
1197 nl_escape_length = Ustrlen(nl_escape);
1198 }
1199else nl_check_length = nl_escape_length = 0;
1200
1201/* Start up a subprocess to run the command. Ensure that our main fd will
1202be closed when the subprocess execs, but remove the flag afterwards.
1203(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1204process to deliver another message.) We get back stdin/stdout file descriptors.
1205If the process creation failed, give an error return. */
1206
1207fd_read = -1;
1208fd_write = -1;
1209save_errno = 0;
1210yield = FALSE;
1211write_pid = (pid_t)(-1);
1212
92b0827a 1213 {
42055a33
JH
1214 int bits = fcntl(tctx->u.fd, F_GETFD);
1215 (void)fcntl(tctx->u.fd, F_SETFD, bits | FD_CLOEXEC);
92b0827a
JH
1216 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
1217 &fd_write, &fd_read, FALSE);
42055a33 1218 (void)fcntl(tctx->u.fd, F_SETFD, bits & ~FD_CLOEXEC);
92b0827a 1219 }
059ec3d9
PH
1220if (filter_pid < 0) goto TIDY_UP; /* errno set */
1221
1222DEBUG(D_transport)
fb6833e0 1223 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1224 (int)filter_pid, fd_write, fd_read);
1225
1226/* Fork subprocess to write the message to the filter, and return the result
1227via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1228smtp dots, or check string processing. */
1229
1230if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
1231if ((write_pid = fork()) == 0)
1232 {
1233 BOOL rc;
f1e894f3
PH
1234 (void)close(fd_read);
1235 (void)close(pfd[pipe_read]);
059ec3d9 1236 nl_check_length = nl_escape_length = 0;
6b46ecc6 1237
42055a33 1238 tctx->u.fd = fd_write;
fb6833e0 1239 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1240 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6 1241
42055a33 1242 rc = internal_transport_write_message(tctx, size_limit);
6b46ecc6 1243
059ec3d9 1244 save_errno = errno;
1ac6b2e7
JH
1245 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1246 != sizeof(BOOL)
1247 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1248 != sizeof(int)
6b46ecc6 1249 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1ac6b2e7 1250 != sizeof(int)
32dfdf8b
JH
1251 || write(pfd[pipe_write], (void *)&tctx->addr->delivery_usec, sizeof(int))
1252 != sizeof(int)
1ac6b2e7
JH
1253 )
1254 rc = FALSE; /* compiler quietening */
059ec3d9
PH
1255 _exit(0);
1256 }
1257save_errno = errno;
1258
1259/* Parent process: close our copy of the writing subprocess' pipes. */
1260
f1e894f3
PH
1261(void)close(pfd[pipe_write]);
1262(void)close(fd_write);
059ec3d9
PH
1263fd_write = -1;
1264
1265/* Writing process creation failed */
1266
1267if (write_pid < 0)
1268 {
1269 errno = save_errno; /* restore */
1270 goto TIDY_UP;
1271 }
1272
1273/* When testing, let the subprocess get going */
1274
8768d548 1275if (f.running_in_test_harness) millisleep(250);
059ec3d9
PH
1276
1277DEBUG(D_transport)
1278 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1279
1280/* Copy the message from the filter to the output fd. A read error leaves len
1281== -1 and errno set. We need to apply a timeout to the read, to cope with
1282the case when the filter gets stuck, but it can be quite a long one. The
1283default is 5m, but this is now configurable. */
1284
1285DEBUG(D_transport) debug_printf("copying from the filter\n");
1286
1287/* Copy the output of the filter, remembering if the last character was NL. If
1288no data is returned, that counts as "ended with NL" (default setting of the
328c5688
JH
1289variable is TRUE). The output should always be unix-format as we converted
1290any wireformat source on writing input to the filter. */
059ec3d9 1291
8768d548 1292f.spool_file_wireformat = FALSE;
059ec3d9
PH
1293chunk_ptr = deliver_out_buffer;
1294
1295for (;;)
1296 {
1297 sigalrm_seen = FALSE;
c2a1bba0 1298 ALARM(transport_filter_timeout);
059ec3d9 1299 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
c2a1bba0 1300 ALARM_CLR(0);
059ec3d9
PH
1301 if (sigalrm_seen)
1302 {
1303 errno = ETIMEDOUT;
8768d548 1304 f.transport_filter_timed_out = TRUE;
059ec3d9
PH
1305 goto TIDY_UP;
1306 }
1307
1308 /* If the read was successful, write the block down the original fd,
1309 remembering whether it ends in \n or not. */
1310
1311 if (len > 0)
1312 {
42055a33 1313 if (!write_chunk(tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1314 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1315 }
1316
1317 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1318
1319 else
1320 {
1321 if (len == 0) yield = TRUE;
1322 break;
1323 }
1324 }
1325
1326/* Tidying up code. If yield = FALSE there has been an error and errno is set
1327to something. Ensure the pipes are all closed and the processes are removed. If
1328there has been an error, kill the processes before waiting for them, just to be
1329sure. Also apply a paranoia timeout. */
1330
1331TIDY_UP:
8768d548 1332f.spool_file_wireformat = save_spool_file_wireformat;
059ec3d9
PH
1333save_errno = errno;
1334
f1e894f3
PH
1335(void)close(fd_read);
1336if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1337
1338if (!yield)
1339 {
1340 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1341 if (write_pid > 0) kill(write_pid, SIGKILL);
1342 }
1343
1344/* Wait for the filter process to complete. */
1345
1346DEBUG(D_transport) debug_printf("waiting for filter process\n");
1347if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1348 {
1349 yield = FALSE;
1350 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1351 tctx->addr->more_errno = rc;
059ec3d9
PH
1352 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1353 }
1354
1355/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1356read the results from its pipe, provided we haven't already had a filter
35af9f61 1357process failure. */
059ec3d9
PH
1358
1359DEBUG(D_transport) debug_printf("waiting for writing process\n");
1360if (write_pid > 0)
1361 {
35af9f61
PH
1362 rc = child_close(write_pid, 30);
1363 if (yield)
8e669ac1 1364 if (rc == 0)
35af9f61
PH
1365 {
1366 BOOL ok;
4dc2379a
JH
1367 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
1368 {
1369 DEBUG(D_transport)
1370 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1371 save_errno = ERRNO_FILTER_FAIL;
1372 yield = FALSE;
1373 }
1374 else if (!ok)
35af9f61 1375 {
4dc2379a 1376 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
32dfdf8b
JH
1377 dummy = read(pfd[pipe_read], (void *)&tctx->addr->more_errno, sizeof(int));
1378 dummy = read(pfd[pipe_read], (void *)&tctx->addr->delivery_usec, sizeof(int));
cab0c277 1379 dummy = dummy; /* compiler quietening */
35af9f61
PH
1380 yield = FALSE;
1381 }
1382 }
1383 else
059ec3d9 1384 {
059ec3d9 1385 yield = FALSE;
35af9f61 1386 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1387 tctx->addr->more_errno = rc;
35af9f61 1388 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1389 }
059ec3d9 1390 }
f1e894f3 1391(void)close(pfd[pipe_read]);
059ec3d9
PH
1392
1393/* If there have been no problems we can now add the terminating "." if this is
1394SMTP output, turning off escaping beforehand. If the last character from the
1395filter was not NL, insert a NL to make the SMTP protocol work. */
1396
1397if (yield)
1398 {
1399 nl_check_length = nl_escape_length = 0;
8768d548 1400 f.spool_file_wireformat = FALSE;
6b46ecc6 1401 if ( tctx->options & topt_end_dot
ff5aac2b 1402 && ( last_filter_was_NL
42055a33
JH
1403 ? !write_chunk(tctx, US".\n", 2)
1404 : !write_chunk(tctx, US"\n.\n", 3)
ff5aac2b 1405 ) )
059ec3d9 1406 yield = FALSE;
059ec3d9
PH
1407
1408 /* Write out any remaining data in the buffer. */
1409
1410 else
ff5aac2b 1411 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
42055a33 1412 || transport_write_block(tctx, deliver_out_buffer, len, FALSE);
059ec3d9 1413 }
ff5aac2b
JH
1414else
1415 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1416
1417DEBUG(D_transport)
1418 {
1419 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1420 if (!yield)
6b46ecc6 1421 debug_printf("errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1422 }
1423
1424return yield;
1425}
1426
1427
1428
1429
1430
1431/*************************************************
1432* Update waiting database *
1433*************************************************/
1434
1435/* This is called when an address is deferred by remote transports that are
1436capable of sending more than one message over one connection. A database is
1437maintained for each transport, keeping track of which messages are waiting for
1438which hosts. The transport can then consult this when eventually a successful
1439delivery happens, and if it finds that another message is waiting for the same
1440host, it can fire up a new process to deal with it using the same connection.
1441
1442The database records are keyed by host name. They can get full if there are
1443lots of messages waiting, and so there is a continuation mechanism for them.
1444
1445Each record contains a list of message ids, packed end to end without any
1446zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1447in this record, and the sequence field says if there are any other records for
1448this host. If the sequence field is 0, there are none. If it is 1, then another
1449record with the name <hostname>:0 exists; if it is 2, then two other records
1450with sequence numbers 0 and 1 exist, and so on.
1451
1452Currently, an exhaustive search of all continuation records has to be done to
1453determine whether to add a message id to a given record. This shouldn't be
1454too bad except in extreme cases. I can't figure out a *simple* way of doing
1455better.
1456
1457Old records should eventually get swept up by the exim_tidydb utility.
1458
1459Arguments:
f6c332bd 1460 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1461 tpname name of the transport
1462
1463Returns: nothing
1464*/
1465
1466void
1467transport_update_waiting(host_item *hostlist, uschar *tpname)
1468{
55414b25 1469const uschar *prevname = US"";
059ec3d9
PH
1470open_db dbblock;
1471open_db *dbm_file;
1472
7a0743eb
PH
1473DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1474
059ec3d9
PH
1475/* Open the database for this transport */
1476
36d295f1 1477if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
b10c87b3 1478 O_RDWR, &dbblock, TRUE, TRUE)))
36d295f1 1479 return;
059ec3d9
PH
1480
1481/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1482that the message id is in each host record. */
059ec3d9 1483
d7978c0f 1484for (host_item * host = hostlist; host; host = host->next)
059ec3d9
PH
1485 {
1486 BOOL already = FALSE;
1487 dbdata_wait *host_record;
d7978c0f 1488 int host_length;
36d295f1 1489 uschar buffer[256];
059ec3d9 1490
059ec3d9
PH
1491 /* Skip if this is the same host as we just processed; otherwise remember
1492 the name for next time. */
1493
1494 if (Ustrcmp(prevname, host->name) == 0) continue;
1495 prevname = host->name;
1496
1497 /* Look up the host record; if there isn't one, make an empty one. */
1498
36d295f1 1499 if (!(host_record = dbfn_read(dbm_file, host->name)))
059ec3d9
PH
1500 {
1501 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH);
1502 host_record->count = host_record->sequence = 0;
1503 }
1504
1505 /* Compute the current length */
1506
1507 host_length = host_record->count * MESSAGE_ID_LENGTH;
1508
1509 /* Search the record to see if the current message is already in it. */
1510
d7978c0f 1511 for (uschar * s = host_record->text; s < host_record->text + host_length;
059ec3d9 1512 s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1513 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1514 { already = TRUE; break; }
059ec3d9
PH
1515
1516 /* If we haven't found this message in the main record, search any
1517 continuation records that exist. */
1518
d7978c0f 1519 for (int i = host_record->sequence - 1; i >= 0 && !already; i--)
059ec3d9
PH
1520 {
1521 dbdata_wait *cont;
1522 sprintf(CS buffer, "%.200s:%d", host->name, i);
36d295f1 1523 if ((cont = dbfn_read(dbm_file, buffer)))
059ec3d9
PH
1524 {
1525 int clen = cont->count * MESSAGE_ID_LENGTH;
d7978c0f 1526 for (uschar * s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1527 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1528 { already = TRUE; break; }
059ec3d9
PH
1529 }
1530 }
1531
1532 /* If this message is already in a record, no need to update. */
1533
7a0743eb
PH
1534 if (already)
1535 {
1536 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1537 continue;
1538 }
059ec3d9
PH
1539
1540
1541 /* If this record is full, write it out with a new name constructed
1542 from the sequence number, increase the sequence number, and empty
1543 the record. */
1544
1545 if (host_record->count >= WAIT_NAME_MAX)
1546 {
1547 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1548 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
1549 host_record->sequence++;
1550 host_record->count = 0;
1551 host_length = 0;
1552 }
1553
1554 /* If this record is not full, increase the size of the record to
1555 allow for one new message id. */
1556
1557 else
1558 {
1559 dbdata_wait *newr =
1560 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH);
1561 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1562 host_record = newr;
1563 }
1564
1565 /* Now add the new name on the end */
1566
1567 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1568 host_record->count++;
1569 host_length += MESSAGE_ID_LENGTH;
1570
1571 /* Update the database */
1572
1573 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1574 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1575 }
1576
1577/* All now done */
1578
1579dbfn_close(dbm_file);
1580}
1581
1582
1583
1584
1585/*************************************************
1586* Test for waiting messages *
1587*************************************************/
1588
1589/* This function is called by a remote transport which uses the previous
1590function to remember which messages are waiting for which remote hosts. It's
1591called after a successful delivery and its job is to check whether there is
1592another message waiting for the same host. However, it doesn't do this if the
1593current continue sequence is greater than the maximum supplied as an argument,
1594or greater than the global connection_max_messages, which, if set, overrides.
1595
1596Arguments:
1597 transport_name name of the transport
1598 hostname name of the host
1599 local_message_max maximum number of messages down one connection
1600 as set by the caller transport
1601 new_message_id set to the message id of a waiting message
1602 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1603 oicf_func function to call to validate if it is ok to send
1604 to this message_id from the current instance.
1605 oicf_data opaque data for oicf_func
059ec3d9
PH
1606
1607Returns: TRUE if new_message_id set; FALSE otherwise
1608*/
1609
a39bd74d
JB
1610typedef struct msgq_s
1611{
1612 uschar message_id [MESSAGE_ID_LENGTH + 1];
1613 BOOL bKeep;
1614} msgq_t;
1615
059ec3d9 1616BOOL
55414b25 1617transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1618 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1619{
1620dbdata_wait *host_record;
0539a19d 1621int host_length;
059ec3d9
PH
1622open_db dbblock;
1623open_db *dbm_file;
059ec3d9 1624
a39bd74d 1625int i;
a39bd74d 1626struct stat statbuf;
a39bd74d 1627
059ec3d9
PH
1628*more = FALSE;
1629
1630DEBUG(D_transport)
1631 {
1632 debug_printf("transport_check_waiting entered\n");
1633 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1634 continue_sequence, local_message_max, connection_max_messages);
1635 }
1636
1637/* Do nothing if we have hit the maximum number that can be send down one
1638connection. */
1639
1640if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1641if (local_message_max > 0 && continue_sequence >= local_message_max)
1642 {
1643 DEBUG(D_transport)
1644 debug_printf("max messages for one connection reached: returning\n");
1645 return FALSE;
1646 }
1647
1648/* Open the waiting information database. */
1649
36d295f1 1650if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
b10c87b3 1651 O_RDWR, &dbblock, TRUE, TRUE)))
36d295f1 1652 return FALSE;
059ec3d9
PH
1653
1654/* See if there is a record for this host; if not, there's nothing to do. */
1655
789f8a4f 1656if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1657 {
1658 dbfn_close(dbm_file);
1659 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1660 return FALSE;
1661 }
1662
1663/* If the data in the record looks corrupt, just log something and
1664don't try to use it. */
1665
1666if (host_record->count > WAIT_NAME_MAX)
1667 {
1668 dbfn_close(dbm_file);
1669 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1670 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1671 return FALSE;
1672 }
1673
1674/* Scan the message ids in the record from the end towards the beginning,
1675until one is found for which a spool file actually exists. If the record gets
1676emptied, delete it and continue with any continuation records that may exist.
1677*/
1678
a39bd74d
JB
1679/* For Bug 1141, I refactored this major portion of the routine, it is risky
1680but the 1 off will remain without it. This code now allows me to SKIP over
1681a message I do not want to send out on this run. */
059ec3d9 1682
a39bd74d
JB
1683host_length = host_record->count * MESSAGE_ID_LENGTH;
1684
1685while (1)
059ec3d9 1686 {
789f8a4f
JH
1687 msgq_t *msgq;
1688 int msgq_count = 0;
1689 int msgq_actual = 0;
1690 BOOL bFound = FALSE;
1691 BOOL bContinuation = FALSE;
1692
a39bd74d 1693 /* create an array to read entire message queue into memory for processing */
059ec3d9 1694
40c90bca 1695 msgq = store_malloc(sizeof(msgq_t) * host_record->count);
a39bd74d
JB
1696 msgq_count = host_record->count;
1697 msgq_actual = msgq_count;
059ec3d9 1698
a39bd74d 1699 for (i = 0; i < host_record->count; ++i)
059ec3d9 1700 {
a39bd74d
JB
1701 msgq[i].bKeep = TRUE;
1702
1703 Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1704 MESSAGE_ID_LENGTH);
a39bd74d
JB
1705 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1706 }
1707
1708 /* first thing remove current message id if it exists */
059ec3d9 1709
a39bd74d
JB
1710 for (i = 0; i < msgq_count; ++i)
1711 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1712 {
1713 msgq[i].bKeep = FALSE;
1714 break;
1715 }
1716
1717 /* now find the next acceptable message_id */
1718
a39bd74d
JB
1719 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1720 {
41313d92
JH
1721 uschar subdir[2];
1722
1723 subdir[0] = split_spool_directory ? msgq[i].message_id[5] : 0;
1724 subdir[1] = 0;
059ec3d9 1725
41313d92
JH
1726 if (Ustat(spool_fname(US"input", subdir, msgq[i].message_id, US"-D"),
1727 &statbuf) != 0)
a39bd74d
JB
1728 msgq[i].bKeep = FALSE;
1729 else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data))
059ec3d9 1730 {
a39bd74d
JB
1731 Ustrcpy(new_message_id, msgq[i].message_id);
1732 msgq[i].bKeep = FALSE;
1733 bFound = TRUE;
059ec3d9
PH
1734 break;
1735 }
1736 }
1737
a39bd74d
JB
1738 /* re-count */
1739 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1740 if (msgq[i].bKeep)
1741 msgq_actual++;
1742
1743 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1744 memory queue */
a39bd74d
JB
1745
1746 if (msgq_actual <= 0)
1747 {
1748 host_length = 0;
1749 host_record->count = 0;
1750 }
1751 else
1752 {
1753 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1754 host_record->count = msgq_actual;
1755
1756 if (msgq_actual < msgq_count)
1757 {
1758 int new_count;
1759 for (new_count = 0, i = 0; i < msgq_count; ++i)
1760 if (msgq[i].bKeep)
1761 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1762 msgq[i].message_id, MESSAGE_ID_LENGTH);
1763
1764 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1765 }
1766 }
1767
36d295f1 1768 /* Check for a continuation record. */
a39bd74d 1769
059ec3d9
PH
1770 while (host_length <= 0)
1771 {
a39bd74d 1772 dbdata_wait * newr = NULL;
36d295f1 1773 uschar buffer[256];
059ec3d9
PH
1774
1775 /* Search for a continuation */
1776
d7978c0f 1777 for (int i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1778 {
1779 sprintf(CS buffer, "%.200s:%d", hostname, i);
1780 newr = dbfn_read(dbm_file, buffer);
1781 }
1782
1783 /* If no continuation, delete the current and break the loop */
1784
a39bd74d 1785 if (!newr)
059ec3d9
PH
1786 {
1787 dbfn_delete(dbm_file, hostname);
1788 break;
1789 }
1790
1791 /* Else replace the current with the continuation */
1792
1793 dbfn_delete(dbm_file, buffer);
1794 host_record = newr;
1795 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1796
a39bd74d
JB
1797 bContinuation = TRUE;
1798 }
059ec3d9 1799
789f8a4f
JH
1800 if (bFound) /* Usual exit from main loop */
1801 {
40c90bca 1802 store_free (msgq);
a39bd74d 1803 break;
789f8a4f 1804 }
059ec3d9
PH
1805
1806 /* If host_length <= 0 we have emptied a record and not found a good message,
1807 and there are no continuation records. Otherwise there is a continuation
1808 record to process. */
1809
1810 if (host_length <= 0)
1811 {
1812 dbfn_close(dbm_file);
1813 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1814 return FALSE;
1815 }
a39bd74d
JB
1816
1817 /* we were not able to find an acceptable message, nor was there a
1818 * continuation record. So bug out, outer logic will clean this up.
1819 */
1820
1821 if (!bContinuation)
1822 {
789f8a4f 1823 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1824 dbfn_close(dbm_file);
1825 return FALSE;
1826 }
a39bd74d 1827
40c90bca 1828 store_free(msgq);
789f8a4f 1829 } /* we need to process a continuation record */
059ec3d9
PH
1830
1831/* Control gets here when an existing message has been encountered; its
1832id is in new_message_id, and host_length is the revised length of the
1833host record. If it is zero, the record has been removed. Update the
1834record if required, close the database, and return TRUE. */
1835
1836if (host_length > 0)
1837 {
1838 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1839
059ec3d9
PH
1840 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1841 *more = TRUE;
1842 }
1843
1844dbfn_close(dbm_file);
1845return TRUE;
1846}
1847
059ec3d9
PH
1848/*************************************************
1849* Deliver waiting message down same socket *
1850*************************************************/
1851
57cc2785
JH
1852/* Just the regain-root-privilege exec portion */
1853void
1854transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
1855 const uschar *hostaddress, uschar *id, int socket_fd)
1856{
57cc2785
JH
1857int i = 20;
1858const uschar **argv;
1859
1860/* Set up the calling arguments; use the standard function for the basics,
1861but we have a number of extras that may be added. */
1862
1863argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
1864
8768d548 1865if (f.smtp_authenticated) argv[i++] = US"-MCA";
14de8063
JH
1866if (smtp_peer_options & OPTION_CHUNKING) argv[i++] = US"-MCK";
1867if (smtp_peer_options & OPTION_DSN) argv[i++] = US"-MCD";
1868if (smtp_peer_options & OPTION_PIPE) argv[i++] = US"-MCP";
1869if (smtp_peer_options & OPTION_SIZE) argv[i++] = US"-MCS";
01603eec 1870#ifndef DISABLE_TLS
14de8063 1871if (smtp_peer_options & OPTION_TLS)
74f1a423 1872 if (tls_out.active.sock >= 0 || continue_proxy_cipher)
57cc2785
JH
1873 {
1874 argv[i++] = US"-MCt";
1875 argv[i++] = sending_ip_address;
1876 argv[i++] = string_sprintf("%d", sending_port);
74f1a423 1877 argv[i++] = tls_out.active.sock >= 0 ? tls_out.cipher : continue_proxy_cipher;
57cc2785
JH
1878 }
1879 else
1880 argv[i++] = US"-MCT";
1881#endif
1882
1883if (queue_run_pid != (pid_t)0)
1884 {
1885 argv[i++] = US"-MCQ";
1886 argv[i++] = string_sprintf("%d", queue_run_pid);
1887 argv[i++] = string_sprintf("%d", queue_run_pipe);
1888 }
1889
1890argv[i++] = US"-MC";
1891argv[i++] = US transport_name;
1892argv[i++] = US hostname;
1893argv[i++] = US hostaddress;
1894argv[i++] = string_sprintf("%d", continue_sequence + 1);
1895argv[i++] = id;
1896argv[i++] = NULL;
1897
1898/* Arrange for the channel to be on stdin. */
1899
1900if (socket_fd != 0)
1901 {
1902 (void)dup2(socket_fd, 0);
1903 (void)close(socket_fd);
1904 }
1905
1906DEBUG(D_exec) debug_print_argv(argv);
1907exim_nullstd(); /* Ensure std{out,err} exist */
1908execv(CS argv[0], (char *const *)argv);
1909
1910DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
1911_exit(errno); /* Note: must be _exit(), NOT exit() */
1912}
1913
1914
1915
059ec3d9
PH
1916/* Fork a new exim process to deliver the message, and do a re-exec, both to
1917get a clean delivery process, and to regain root privilege in cases where it
1918has been given away.
1919
1920Arguments:
1921 transport_name to pass to the new process
1922 hostname ditto
1923 hostaddress ditto
1924 id the new message to process
1925 socket_fd the connected socket
1926
1927Returns: FALSE if fork fails; TRUE otherwise
1928*/
1929
1930BOOL
55414b25
JH
1931transport_pass_socket(const uschar *transport_name, const uschar *hostname,
1932 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
1933{
1934pid_t pid;
1935int status;
1936
1937DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
1938
1939if ((pid = fork()) == 0)
1940 {
059ec3d9
PH
1941 /* Disconnect entirely from the parent process. If we are running in the
1942 test harness, wait for a bit to allow the previous process time to finish,
1943 write the log, etc., so that the output is always in the same order for
1944 automatic comparison. */
1945
b7d3afcf
JH
1946 if ((pid = fork()) != 0)
1947 {
1948 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (final-pid %d)\n", pid);
1949 _exit(EXIT_SUCCESS);
1950 }
8768d548 1951 if (f.running_in_test_harness) sleep(1);
059ec3d9 1952
57cc2785
JH
1953 transport_do_pass_socket(transport_name, hostname, hostaddress,
1954 id, socket_fd);
059ec3d9
PH
1955 }
1956
1957/* If the process creation succeeded, wait for the first-level child, which
1958immediately exits, leaving the second level process entirely disconnected from
1959this one. */
1960
1961if (pid > 0)
1962 {
1963 int rc;
1964 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
b7d3afcf 1965 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (inter-pid %d)\n", pid);
059ec3d9
PH
1966 return TRUE;
1967 }
1968else
1969 {
1970 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
1971 strerror(errno));
1972 return FALSE;
1973 }
1974}
1975
1976
1977
1978/*************************************************
1979* Set up direct (non-shell) command *
1980*************************************************/
1981
1982/* This function is called when a command line is to be parsed and executed
1983directly, without the use of /bin/sh. It is called by the pipe transport,
1984the queryprogram router, and also from the main delivery code when setting up a
1985transport filter process. The code for ETRN also makes use of this; in that
1986case, no addresses are passed.
1987
1988Arguments:
1989 argvptr pointer to anchor for argv vector
55414b25 1990 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
1991 expand_arguments true if expansion is to occur
1992 expand_failed error value to set if expansion fails; not relevant if
1993 addr == NULL
1994 addr chain of addresses, or NULL
1995 etext text for use in error messages
1996 errptr where to put error message if addr is NULL;
1997 otherwise it is put in the first address
1998
1999Returns: TRUE if all went well; otherwise an error will be
2000 set in the first address and FALSE returned
2001*/
2002
2003BOOL
55414b25
JH
2004transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2005 BOOL expand_arguments, int expand_failed, address_item *addr,
2006 uschar *etext, uschar **errptr)
059ec3d9 2007{
55414b25 2008const uschar **argv;
059ec3d9
PH
2009uschar *s, *ss;
2010int address_count = 0;
2011int argcount = 0;
d7978c0f 2012int max_args;
059ec3d9
PH
2013
2014/* Get store in which to build an argument list. Count the number of addresses
2015supplied, and allow for that many arguments, plus an additional 60, which
2016should be enough for anybody. Multiple addresses happen only when the local
2017delivery batch option is set. */
2018
d7978c0f 2019for (address_item * ad = addr; ad; ad = ad->next) address_count++;
059ec3d9
PH
2020max_args = address_count + 60;
2021*argvptr = argv = store_get((max_args+1)*sizeof(uschar *));
2022
2023/* Split the command up into arguments terminated by white space. Lose
2024trailing space at the start and end. Double-quoted arguments can contain \\ and
2025\" escapes and so can be handled by the standard function; single-quoted
2026arguments are verbatim. Copy each argument into a new string. */
2027
2028s = cmd;
2029while (isspace(*s)) s++;
2030
2031while (*s != 0 && argcount < max_args)
2032 {
2033 if (*s == '\'')
2034 {
2035 ss = s + 1;
2036 while (*ss != 0 && *ss != '\'') ss++;
2037 argv[argcount++] = ss = store_get(ss - s++);
2038 while (*s != 0 && *s != '\'') *ss++ = *s++;
2039 if (*s != 0) s++;
2040 *ss++ = 0;
2041 }
55414b25 2042 else argv[argcount++] = string_copy(string_dequote(CUSS &s));
059ec3d9
PH
2043 while (isspace(*s)) s++;
2044 }
2045
5903c6ff 2046argv[argcount] = US 0;
059ec3d9
PH
2047
2048/* If *s != 0 we have run out of argument slots. */
2049
2050if (*s != 0)
2051 {
2052 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2053 "%s", cmd, etext);
2054 if (addr != NULL)
2055 {
2056 addr->transport_return = FAIL;
2057 addr->message = msg;
2058 }
2059 else *errptr = msg;
2060 return FALSE;
2061 }
2062
2063/* Expand each individual argument if required. Expansion happens for pipes set
2064up in filter files and with directly-supplied commands. It does not happen if
2065the pipe comes from a traditional .forward file. A failing expansion is a big
2066disaster if the command came from Exim's configuration; if it came from a user
2067it is just a normal failure. The expand_failed value is used as the error value
2068to cater for these two cases.
2069
2070An argument consisting just of the text "$pipe_addresses" is treated specially.
2071It is not passed to the general expansion function. Instead, it is replaced by
2072a number of arguments, one for each address. This avoids problems with shell
2073metacharacters and spaces in addresses.
2074
2075If the parent of the top address has an original part of "system-filter", this
2076pipe was set up by the system filter, and we can permit the expansion of
2077$recipients. */
2078
2079DEBUG(D_transport)
2080 {
2081 debug_printf("direct command:\n");
d7978c0f 2082 for (int i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2083 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2084 }
2085
2086if (expand_arguments)
2087 {
2088 BOOL allow_dollar_recipients = addr != NULL &&
2089 addr->parent != NULL &&
2090 Ustrcmp(addr->parent->address, "system-filter") == 0;
2091
d7978c0f 2092 for (int i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2093 {
2094
2095 /* Handle special fudge for passing an address list */
2096
2097 if (addr != NULL &&
2098 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2099 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2100 {
2101 int additional;
2102
2103 if (argcount + address_count - 1 > max_args)
2104 {
2105 addr->transport_return = FAIL;
2106 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2107 "in %s", cmd, etext);
2108 return FALSE;
2109 }
2110
2111 additional = address_count - 1;
2112 if (additional > 0)
2113 memmove(argv + i + 1 + additional, argv + i + 1,
2114 (argcount - i)*sizeof(uschar *));
2115
d7978c0f
JH
2116 for (address_item * ad = addr; ad; ad = ad->next)
2117 {
2118 argv[i++] = ad->address;
2119 argcount++;
2120 }
09792322 2121
700d22f3
PP
2122 /* Subtract one since we replace $pipe_addresses */
2123 argcount--;
2124 i--;
09792322
NK
2125 }
2126
2127 /* Handle special case of $address_pipe when af_force_command is set */
2128
2129 else if (addr != NULL && testflag(addr,af_force_command) &&
2130 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2131 Ustrcmp(argv[i], "${address_pipe}") == 0))
2132 {
09792322
NK
2133 int address_pipe_argcount = 0;
2134 int address_pipe_max_args;
2135 uschar **address_pipe_argv;
2136
2137 /* We can never have more then the argv we will be loading into */
2138 address_pipe_max_args = max_args - argcount + 1;
2139
2140 DEBUG(D_transport)
700d22f3 2141 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2142
2143 /* We allocate an additional for (uschar *)0 */
2144 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *));
2145
2146 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2147 s = expand_string(addr->local_part + 1);
2148
700d22f3
PP
2149 if (s == NULL || *s == '\0')
2150 {
2151 addr->transport_return = FAIL;
2152 addr->message = string_sprintf("Expansion of \"%s\" "
2153 "from command \"%s\" in %s failed: %s",
2154 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2155 return FALSE;
700d22f3 2156 }
09792322
NK
2157
2158 while (isspace(*s)) s++; /* strip leading space */
2159
2160 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2161 {
2162 if (*s == '\'')
2163 {
2164 ss = s + 1;
2165 while (*ss != 0 && *ss != '\'') ss++;
2166 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++);
2167 while (*s != 0 && *s != '\'') *ss++ = *s++;
2168 if (*s != 0) s++;
2169 *ss++ = 0;
2170 }
55414b25
JH
2171 else address_pipe_argv[address_pipe_argcount++] =
2172 string_copy(string_dequote(CUSS &s));
09792322
NK
2173 while (isspace(*s)) s++; /* strip space after arg */
2174 }
2175
5903c6ff 2176 address_pipe_argv[address_pipe_argcount] = US 0;
09792322
NK
2177
2178 /* If *s != 0 we have run out of argument slots. */
2179 if (*s != 0)
2180 {
2181 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2182 "\"%s\" in %s", addr->local_part + 1, etext);
2183 if (addr != NULL)
2184 {
2185 addr->transport_return = FAIL;
2186 addr->message = msg;
2187 }
2188 else *errptr = msg;
2189 return FALSE;
2190 }
2191
700d22f3
PP
2192 /* address_pipe_argcount - 1
2193 * because we are replacing $address_pipe in the argument list
2194 * with the first thing it expands to */
09792322
NK
2195 if (argcount + address_pipe_argcount - 1 > max_args)
2196 {
2197 addr->transport_return = FAIL;
2198 addr->message = string_sprintf("Too many arguments to command "
2199 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2200 return FALSE;
2201 }
2202
2203 /* If we are not just able to replace the slot that contained
2204 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2205 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2206 * Visually if address_pipe_argcount == 2:
2207 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2208 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2209 */
2210 if (address_pipe_argcount > 1)
2211 memmove(
4c04137d 2212 /* current position + additional args */
09792322
NK
2213 argv + i + address_pipe_argcount,
2214 /* current position + 1 (for the (uschar *)0 at the end) */
2215 argv + i + 1,
2216 /* -1 for the (uschar *)0 at the end)*/
2217 (argcount - i)*sizeof(uschar *)
2218 );
2219
2220 /* Now we fill in the slots we just moved argv out of
2221 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2222 */
d7978c0f 2223 for (int address_pipe_i = 0;
5903c6ff 2224 address_pipe_argv[address_pipe_i] != US 0;
700d22f3
PP
2225 address_pipe_i++)
2226 {
2227 argv[i++] = address_pipe_argv[address_pipe_i];
2228 argcount++;
09792322
NK
2229 }
2230
700d22f3
PP
2231 /* Subtract one since we replace $address_pipe */
2232 argcount--;
2233 i--;
059ec3d9
PH
2234 }
2235
2236 /* Handle normal expansion string */
2237
2238 else
2239 {
55414b25 2240 const uschar *expanded_arg;
8768d548 2241 f.enable_dollar_recipients = allow_dollar_recipients;
55414b25 2242 expanded_arg = expand_cstring(argv[i]);
8768d548 2243 f.enable_dollar_recipients = FALSE;
059ec3d9
PH
2244
2245 if (expanded_arg == NULL)
2246 {
2247 uschar *msg = string_sprintf("Expansion of \"%s\" "
2248 "from command \"%s\" in %s failed: %s",
2249 argv[i], cmd, etext, expand_string_message);
2250 if (addr != NULL)
2251 {
2252 addr->transport_return = expand_failed;
2253 addr->message = msg;
2254 }
2255 else *errptr = msg;
2256 return FALSE;
2257 }
2258 argv[i] = expanded_arg;
2259 }
2260 }
2261
2262 DEBUG(D_transport)
2263 {
2264 debug_printf("direct command after expansion:\n");
d7978c0f 2265 for (int i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2266 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2267 }
2268 }
2269
2270return TRUE;
2271}
2272
d185889f 2273#endif /*!MACRO_PREDEF*/
511a6c14
JH
2274/* vi: aw ai sw=2
2275*/
059ec3d9 2276/* End of transport.c */