Docs: clean for next release
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* General functions concerned with transportation, and generic options for all
9transports. */
10
11
12#include "exim.h"
13
059ec3d9
PH
14/* Generic options for transports, all of which live inside transport_instance
15data blocks and which therefore have the opt_public flag set. Note that there
16are other options living inside this structure which can be set only from
17certain transports. */
18
19optionlist optionlist_transports[] = {
f2ed27cf 20 /* name type value */
059ec3d9
PH
21 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
22 (void *)offsetof(transport_instance, expand_gid) },
23 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
24 (void *)offsetof(transport_instance, expand_uid) },
25 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
26 (void *)offsetof(transport_instance, rewrite_existflags) },
27 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
28 (void *)offsetof(transport_instance, rewrite_rules) },
29 { "*set_group", opt_bool|opt_hidden|opt_public,
30 (void *)offsetof(transport_instance, gid_set) },
31 { "*set_user", opt_bool|opt_hidden|opt_public,
32 (void *)offsetof(transport_instance, uid_set) },
33 { "body_only", opt_bool|opt_public,
34 (void *)offsetof(transport_instance, body_only) },
35 { "current_directory", opt_stringptr|opt_public,
36 (void *)offsetof(transport_instance, current_dir) },
37 { "debug_print", opt_stringptr | opt_public,
38 (void *)offsetof(transport_instance, debug_string) },
39 { "delivery_date_add", opt_bool|opt_public,
40 (void *)(offsetof(transport_instance, delivery_date_add)) },
41 { "disable_logging", opt_bool|opt_public,
42 (void *)(offsetof(transport_instance, disable_logging)) },
43 { "driver", opt_stringptr|opt_public,
44 (void *)offsetof(transport_instance, driver_name) },
45 { "envelope_to_add", opt_bool|opt_public,
46 (void *)(offsetof(transport_instance, envelope_to_add)) },
0cbf2b82 47#ifndef DISABLE_EVENT
774ef2d7
JH
48 { "event_action", opt_stringptr | opt_public,
49 (void *)offsetof(transport_instance, event_action) },
50#endif
059ec3d9
PH
51 { "group", opt_expand_gid|opt_public,
52 (void *)offsetof(transport_instance, gid) },
846726c5 53 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
54 (void *)offsetof(transport_instance, add_headers) },
55 { "headers_only", opt_bool|opt_public,
56 (void *)offsetof(transport_instance, headers_only) },
846726c5 57 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
58 (void *)offsetof(transport_instance, remove_headers) },
59 { "headers_rewrite", opt_rewrite|opt_public,
60 (void *)offsetof(transport_instance, headers_rewrite) },
61 { "home_directory", opt_stringptr|opt_public,
62 (void *)offsetof(transport_instance, home_dir) },
63 { "initgroups", opt_bool|opt_public,
64 (void *)offsetof(transport_instance, initgroups) },
fa41615d
JH
65 { "max_parallel", opt_stringptr|opt_public,
66 (void *)offsetof(transport_instance, max_parallel) },
059ec3d9
PH
67 { "message_size_limit", opt_stringptr|opt_public,
68 (void *)offsetof(transport_instance, message_size_limit) },
69 { "rcpt_include_affixes", opt_bool|opt_public,
70 (void *)offsetof(transport_instance, rcpt_include_affixes) },
71 { "retry_use_local_part", opt_bool|opt_public,
72 (void *)offsetof(transport_instance, retry_use_local_part) },
73 { "return_path", opt_stringptr|opt_public,
74 (void *)(offsetof(transport_instance, return_path)) },
75 { "return_path_add", opt_bool|opt_public,
76 (void *)(offsetof(transport_instance, return_path_add)) },
77 { "shadow_condition", opt_stringptr|opt_public,
78 (void *)offsetof(transport_instance, shadow_condition) },
79 { "shadow_transport", opt_stringptr|opt_public,
80 (void *)offsetof(transport_instance, shadow) },
81 { "transport_filter", opt_stringptr|opt_public,
82 (void *)offsetof(transport_instance, filter_command) },
83 { "transport_filter_timeout", opt_time|opt_public,
84 (void *)offsetof(transport_instance, filter_timeout) },
85 { "user", opt_expand_uid|opt_public,
86 (void *)offsetof(transport_instance, uid) }
87};
88
c0b9d3e8 89int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9 90
d185889f
JH
91#ifdef MACRO_PREDEF
92
5f69a529 93# include "macro_predef.h"
059ec3d9 94
c0b9d3e8 95void
d185889f 96options_transports(void)
c0b9d3e8
JH
97{
98struct transport_info * ti;
d185889f 99uschar buf[64];
c0b9d3e8 100
d185889f 101options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
c0b9d3e8
JH
102
103for (ti = transports_available; ti->driver_name[0]; ti++)
4945f557 104 {
cab0c277 105 spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", ti->driver_name);
d185889f
JH
106 builtin_macro_create(buf);
107 options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
4945f557 108 }
383832ef 109}
c0b9d3e8 110
d185889f
JH
111#else /*!MACRO_PREDEF*/
112
113/* Structure for keeping list of addresses that have been added to
114Envelope-To:, in order to avoid duplication. */
115
116struct aci {
117 struct aci *next;
118 address_item *ptr;
119 };
120
121
122/* Static data for write_chunk() */
123
124static uschar *chunk_ptr; /* chunk pointer */
125static uschar *nl_check; /* string to look for at line start */
126static int nl_check_length; /* length of same */
127static uschar *nl_escape; /* string to insert */
128static int nl_escape_length; /* length of same */
129static int nl_partial_match; /* length matched at chunk end */
130
131
059ec3d9
PH
132/*************************************************
133* Initialize transport list *
134*************************************************/
135
136/* Read the transports section of the configuration file, and set up a chain of
137transport instances according to its contents. Each transport has generic
138options and may also have its own private options. This function is only ever
139called when transports == NULL. We use generic code in readconf to do most of
140the work. */
141
142void
143transport_init(void)
144{
145transport_instance *t;
146
147readconf_driver_init(US"transport",
148 (driver_instance **)(&transports), /* chain anchor */
149 (driver_info *)transports_available, /* available drivers */
150 sizeof(transport_info), /* size of info block */
151 &transport_defaults, /* default values for generic options */
152 sizeof(transport_instance), /* size of instance block */
153 optionlist_transports, /* generic options */
154 optionlist_transports_size);
155
156/* Now scan the configured transports and check inconsistencies. A shadow
157transport is permitted only for local transports. */
158
7b4c8c1f 159for (t = transports; t; t = t->next)
059ec3d9 160 {
7b4c8c1f
JH
161 if (!t->info->local && t->shadow)
162 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
163 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
164
165 if (t->body_only && t->headers_only)
166 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
167 "%s transport: body_only and headers_only are mutually exclusive",
168 t->name);
169 }
170}
171
172
173
174/*************************************************
175* Write block of data *
176*************************************************/
177
178/* Subroutine called by write_chunk() and at the end of the message actually
179to write a data block. Also called directly by some transports to write
180additional data to the file descriptor (e.g. prefix, suffix).
181
182If a transport wants data transfers to be timed, it sets a non-zero value in
183transport_write_timeout. A non-zero transport_write_timeout causes a timer to
184be set for each block of data written from here. If time runs out, then write()
185fails and provokes an error return. The caller can then inspect sigalrm_seen to
186check for a timeout.
187
188On some systems, if a quota is exceeded during the write, the yield is the
189number of bytes written rather than an immediate error code. This also happens
190on some systems in other cases, for example a pipe that goes away because the
191other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
192get the error codes the first time.
193
194The write() function is also interruptible; the Solaris 2.6 man page says:
195
196 If write() is interrupted by a signal before it writes any
197 data, it will return -1 with errno set to EINTR.
198
199 If write() is interrupted by a signal after it successfully
200 writes some data, it will return the number of bytes written.
201
202To handle these cases, we want to restart the write() to output the remainder
203of the data after a non-negative return from write(), except after a timeout.
204In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
205proper error then occurs. In principle, after an interruption, the second
206write() could suffer the same fate, but we do not want to continue for
207evermore, so stick a maximum repetition count on the loop to act as a
208longstop.
209
210Arguments:
42055a33 211 tctx transport context: file descriptor or string to write to
059ec3d9
PH
212 block block of bytes to write
213 len number of bytes to write
925ac8e4 214 more further data expected soon
059ec3d9
PH
215
216Returns: TRUE on success, FALSE on failure (with errno preserved);
217 transport_count is incremented by the number of bytes written
218*/
219
42055a33
JH
220static BOOL
221transport_write_block_fd(transport_ctx * tctx, uschar *block, int len, BOOL more)
059ec3d9
PH
222{
223int i, rc, save_errno;
958541e9 224int local_timeout = transport_write_timeout;
42055a33 225int fd = tctx->u.fd;
958541e9
PH
226
227/* This loop is for handling incomplete writes and other retries. In most
228normal cases, it is only ever executed once. */
059ec3d9
PH
229
230for (i = 0; i < 100; i++)
231 {
232 DEBUG(D_transport)
42055a33
JH
233 debug_printf("writing data block fd=%d size=%d timeout=%d%s\n",
234 fd, len, local_timeout, more ? " (more expected)" : "");
059ec3d9 235
958541e9
PH
236 /* This code makes use of alarm() in order to implement the timeout. This
237 isn't a very tidy way of doing things. Using non-blocking I/O with select()
238 provides a neater approach. However, I don't know how to do this when TLS is
239 in use. */
059ec3d9 240
958541e9
PH
241 if (transport_write_timeout <= 0) /* No timeout wanted */
242 {
42055a33
JH
243 rc =
244#ifdef SUPPORT_TLS
74f1a423 245 tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
42055a33
JH
246#endif
247#ifdef MSG_MORE
b3b37076
JH
248 more && !(tctx->options & topt_not_socket)
249 ? send(fd, block, len, MSG_MORE) :
42055a33
JH
250#endif
251 write(fd, block, len);
958541e9
PH
252 save_errno = errno;
253 }
059ec3d9 254
958541e9 255 /* Timeout wanted. */
059ec3d9 256
958541e9 257 else
059ec3d9 258 {
c2a1bba0 259 ALARM(local_timeout);
42055a33
JH
260
261 rc =
59932f7d 262#ifdef SUPPORT_TLS
74f1a423 263 tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
42055a33
JH
264#endif
265#ifdef MSG_MORE
e01d111e
JH
266 more && !(tctx->options & topt_not_socket)
267 ? send(fd, block, len, MSG_MORE) :
59932f7d 268#endif
42055a33
JH
269 write(fd, block, len);
270
958541e9 271 save_errno = errno;
c2a1bba0 272 local_timeout = ALARM_CLR(0);
059ec3d9
PH
273 if (sigalrm_seen)
274 {
275 errno = ETIMEDOUT;
276 return FALSE;
277 }
278 }
279
280 /* Hopefully, the most common case is success, so test that first. */
281
282 if (rc == len) { transport_count += len; return TRUE; }
283
958541e9
PH
284 /* A non-negative return code is an incomplete write. Try again for the rest
285 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
286
287 if (rc >= 0)
288 {
289 len -= rc;
290 block += rc;
291 transport_count += rc;
292 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 293 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
294 }
295
296 /* A negative return code with an EINTR error is another form of
297 incomplete write, zero bytes having been written */
298
299 if (save_errno == EINTR)
300 {
301 DEBUG(D_transport)
302 debug_printf("write interrupted before anything written\n");
958541e9 303 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
304 }
305
306 /* A response of EAGAIN from write() is likely only in the case of writing
307 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
308
309 if (save_errno == EAGAIN)
310 {
311 DEBUG(D_transport)
312 debug_printf("write temporarily locked out, waiting 1 sec\n");
313 sleep(1);
958541e9
PH
314
315 /* Before continuing to try another write, check that we haven't run out of
316 time. */
317
318 CHECK_TIMEOUT:
319 if (transport_write_timeout > 0 && local_timeout <= 0)
320 {
321 errno = ETIMEDOUT;
322 return FALSE;
323 }
059ec3d9
PH
324 continue;
325 }
326
327 /* Otherwise there's been an error */
328
329 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
330 strerror(save_errno));
331 errno = save_errno;
332 return FALSE;
333 }
334
335/* We've tried and tried and tried but still failed */
336
337errno = ERRNO_WRITEINCOMPLETE;
338return FALSE;
339}
340
341
42055a33
JH
342BOOL
343transport_write_block(transport_ctx * tctx, uschar *block, int len, BOOL more)
344{
345if (!(tctx->options & topt_output_string))
346 return transport_write_block_fd(tctx, block, len, more);
347
348/* Write to expanding-string. NOTE: not NUL-terminated */
349
350if (!tctx->u.msg)
acec9514 351 tctx->u.msg = string_get(1024);
42055a33 352
acec9514 353tctx->u.msg = string_catn(tctx->u.msg, block, len);
42055a33
JH
354return TRUE;
355}
356
357
059ec3d9
PH
358
359
360/*************************************************
361* Write formatted string *
362*************************************************/
363
364/* This is called by various transports. It is a convenience function.
365
366Arguments:
367 fd file descriptor
368 format string format
369 ... arguments for format
370
371Returns: the yield of transport_write_block()
372*/
373
374BOOL
1ba28e2b 375transport_write_string(int fd, const char *format, ...)
059ec3d9 376{
cab0c277 377transport_ctx tctx = {{0}};
d12746bc 378gstring gs = { .size = big_buffer_size, .ptr = 0, .s = big_buffer };
059ec3d9 379va_list ap;
d12746bc 380
059ec3d9 381va_start(ap, format);
d12746bc 382if (!string_vformat(&gs, FALSE, format, ap))
059ec3d9
PH
383 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
384va_end(ap);
42055a33 385tctx.u.fd = fd;
d12746bc 386return transport_write_block(&tctx, gs.s, gs.ptr, FALSE);
059ec3d9
PH
387}
388
389
390
391
42055a33
JH
392void
393transport_write_reset(int options)
394{
395if (!(options & topt_continuation)) chunk_ptr = deliver_out_buffer;
396nl_partial_match = -1;
397nl_check_length = nl_escape_length = 0;
398}
399
400
401
059ec3d9
PH
402/*************************************************
403* Write character chunk *
404*************************************************/
405
406/* Subroutine used by transport_write_message() to scan character chunks for
407newlines and act appropriately. The object is to minimise the number of writes.
408The output byte stream is buffered up in deliver_out_buffer, which is written
409only when it gets full, thus minimizing write operations and TCP packets.
410
411Static data is used to handle the case when the last character of the previous
412chunk was NL, or matched part of the data that has to be escaped.
413
414Arguments:
42055a33
JH
415 tctx transport context - processing to be done during output,
416 and file descriptor to write to
059ec3d9
PH
417 chunk pointer to data to write
418 len length of data to write
059ec3d9
PH
419
420In addition, the static nl_xxx variables must be set as required.
421
422Returns: TRUE on success, FALSE on failure (with errno preserved)
423*/
424
42055a33
JH
425BOOL
426write_chunk(transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
427{
428uschar *start = chunk;
429uschar *end = chunk + len;
ff5aac2b 430uschar *ptr;
059ec3d9
PH
431int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
432
433/* The assumption is made that the check string will never stretch over move
434than one chunk since the only time there are partial matches is when copying
435the body in large buffers. There is always enough room in the buffer for an
436escape string, since the loop below ensures this for each character it
437processes, and it won't have stuck in the escape string if it left a partial
438match. */
439
440if (nl_partial_match >= 0)
441 {
442 if (nl_check_length > 0 && len >= nl_check_length &&
443 Ustrncmp(start, nl_check + nl_partial_match,
444 nl_check_length - nl_partial_match) == 0)
445 {
446 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
447 chunk_ptr += nl_escape_length;
448 start += nl_check_length - nl_partial_match;
449 }
450
451 /* The partial match was a false one. Insert the characters carried over
452 from the previous chunk. */
453
454 else if (nl_partial_match > 0)
455 {
456 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
457 chunk_ptr += nl_partial_match;
458 }
459
460 nl_partial_match = -1;
461 }
462
463/* Now process the characters in the chunk. Whenever we hit a newline we check
464for possible escaping. The code for the non-NL route should be as fast as
465possible. */
466
467for (ptr = start; ptr < end; ptr++)
468 {
6d5c916c 469 int ch, len;
059ec3d9
PH
470
471 /* Flush the buffer if it has reached the threshold - we want to leave enough
472 room for the next uschar, plus a possible extra CR for an LF, plus the escape
473 string. */
474
6d5c916c 475 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 476 {
58fc5fb2
JH
477 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
478
6d5c916c
JH
479 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
480 from previous SMTP commands. */
481
d315eda1 482 if (tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2 483 {
d2aa036b 484 if ( tctx->chunk_cb(tctx, (unsigned)len, 0) != OK
42055a33 485 || !transport_write_block(tctx, deliver_out_buffer, len, FALSE)
d2aa036b 486 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
58fc5fb2
JH
487 )
488 return FALSE;
489 }
490 else
42055a33 491 if (!transport_write_block(tctx, deliver_out_buffer, len, FALSE))
6d5c916c 492 return FALSE;
059ec3d9
PH
493 chunk_ptr = deliver_out_buffer;
494 }
495
328c5688
JH
496 /* Remove CR before NL if required */
497
498 if ( *ptr == '\r' && ptr[1] == '\n'
d315eda1 499 && !(tctx->options & topt_use_crlf)
8768d548 500 && f.spool_file_wireformat
328c5688
JH
501 )
502 ptr++;
503
059ec3d9
PH
504 if ((ch = *ptr) == '\n')
505 {
506 int left = end - ptr - 1; /* count of chars left after NL */
507
508 /* Insert CR before NL if required */
509
8768d548 510 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
328c5688 511 *chunk_ptr++ = '\r';
059ec3d9 512 *chunk_ptr++ = '\n';
332f5cf3 513 transport_newlines++;
059ec3d9
PH
514
515 /* The check_string test (formerly "from hack") replaces the specific
516 string at the start of a line with an escape string (e.g. "From " becomes
517 ">From " or "." becomes "..". It is a case-sensitive test. The length
518 check above ensures there is always enough room to insert this string. */
519
520 if (nl_check_length > 0)
521 {
522 if (left >= nl_check_length &&
523 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
524 {
525 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
526 chunk_ptr += nl_escape_length;
527 ptr += nl_check_length;
528 }
529
530 /* Handle the case when there isn't enough left to match the whole
531 check string, but there may be a partial match. We remember how many
532 characters matched, and finish processing this chunk. */
533
534 else if (left <= 0) nl_partial_match = 0;
535
536 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
537 {
538 nl_partial_match = left;
539 ptr = end;
540 }
541 }
542 }
543
544 /* Not a NL character */
545
546 else *chunk_ptr++ = ch;
547 }
548
549return TRUE;
550}
551
552
553
554
555/*************************************************
556* Generate address for RCPT TO *
557*************************************************/
558
559/* This function puts together an address for RCPT to, using the caseful
560version of the local part and the caseful version of the domain. If there is no
561prefix or suffix, or if affixes are to be retained, we can just use the
562original address. Otherwise, if there is a prefix but no suffix we can use a
563pointer into the original address. If there is a suffix, however, we have to
564build a new string.
565
566Arguments:
567 addr the address item
568 include_affixes TRUE if affixes are to be included
569
570Returns: a string
571*/
572
573uschar *
574transport_rcpt_address(address_item *addr, BOOL include_affixes)
575{
576uschar *at;
577int plen, slen;
578
579if (include_affixes)
580 {
581 setflag(addr, af_include_affixes); /* Affects logged => line */
582 return addr->address;
583 }
584
585if (addr->suffix == NULL)
586 {
587 if (addr->prefix == NULL) return addr->address;
588 return addr->address + Ustrlen(addr->prefix);
589 }
590
591at = Ustrrchr(addr->address, '@');
592plen = (addr->prefix == NULL)? 0 : Ustrlen(addr->prefix);
593slen = Ustrlen(addr->suffix);
594
bb07bcd3 595return string_sprintf("%.*s@%s", (int)(at - addr->address - plen - slen),
059ec3d9
PH
596 addr->address + plen, at + 1);
597}
598
599
600/*************************************************
601* Output Envelope-To: address & scan duplicates *
602*************************************************/
603
604/* This function is called from internal_transport_write_message() below, when
605generating an Envelope-To: header line. It checks for duplicates of the given
606address and its ancestors. When one is found, this function calls itself
607recursively, to output the envelope address of the duplicate.
608
609We want to avoid duplication in the list, which can arise for example when
610A->B,C and then both B and C alias to D. This can also happen when there are
611unseen drivers in use. So a list of addresses that have been output is kept in
612the plist variable.
613
614It is also possible to have loops in the address ancestry/duplication graph,
615for example if there are two top level addresses A and B and we have A->B,C and
616B->A. To break the loop, we use a list of processed addresses in the dlist
617variable.
618
619After handling duplication, this function outputs the progenitor of the given
620address.
621
622Arguments:
623 p the address we are interested in
624 pplist address of anchor of the list of addresses not to output
625 pdlist address of anchor of the list of processed addresses
626 first TRUE if this is the first address; set it FALSE afterwards
6d5c916c 627 tctx transport context - processing to be done during output
42055a33 628 and the file descriptor to write to
059ec3d9
PH
629
630Returns: FALSE if writing failed
631*/
632
633static BOOL
634write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
42055a33 635 BOOL *first, transport_ctx * tctx)
059ec3d9
PH
636{
637address_item *pp;
638struct aci *ppp;
639
640/* Do nothing if we have already handled this address. If not, remember it
641so that we don't handle it again. */
642
ff5aac2b 643for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9
PH
644
645ppp = store_get(sizeof(struct aci));
646ppp->next = *pdlist;
647*pdlist = ppp;
648ppp->ptr = p;
649
650/* Now scan up the ancestry, checking for duplicates at each generation. */
651
652for (pp = p;; pp = pp->parent)
653 {
654 address_item *dup;
ff5aac2b
JH
655 for (dup = addr_duplicate; dup; dup = dup->next)
656 if (dup->dupof == pp) /* a dup of our address */
42055a33 657 if (!write_env_to(dup, pplist, pdlist, first, tctx))
ff5aac2b
JH
658 return FALSE;
659 if (!pp->parent) break;
059ec3d9
PH
660 }
661
662/* Check to see if we have already output the progenitor. */
663
ff5aac2b
JH
664for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
665if (ppp) return TRUE;
059ec3d9
PH
666
667/* Remember what we have output, and output it. */
668
669ppp = store_get(sizeof(struct aci));
670ppp->next = *pplist;
671*pplist = ppp;
672ppp->ptr = pp;
673
42055a33 674if (!*first && !write_chunk(tctx, US",\n ", 3)) return FALSE;
059ec3d9 675*first = FALSE;
42055a33 676return write_chunk(tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
677}
678
679
680
681
4c04137d 682/* Add/remove/rewrite headers, and send them plus the empty-line separator.
511a6c14
JH
683
684Globals:
685 header_list
686
687Arguments:
688 addr (chain of) addresses (for extra headers), or NULL;
689 only the first address is used
44bc8f0c 690 tctx transport context
59932f7d 691 sendfn function for output (transport or verify)
511a6c14
JH
692
693Returns: TRUE on success; FALSE on failure.
694*/
695BOOL
42055a33
JH
696transport_headers_send(transport_ctx * tctx,
697 BOOL (*sendfn)(transport_ctx * tctx, uschar * s, int len))
511a6c14
JH
698{
699header_line *h;
6b46ecc6 700const uschar *list;
6d5c916c
JH
701transport_instance * tblock = tctx ? tctx->tblock : NULL;
702address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
703
704/* Then the message's headers. Don't write any that are flagged as "old";
705that means they were rewritten, or are a record of envelope rewriting, or
706were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
707match any entries therein. It is a colon-sep list; expand the items
708separately and squash any empty ones.
d43cbe25 709Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 710
ff5aac2b 711for (h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14
JH
712 {
713 int i;
511a6c14
JH
714 BOOL include_header = TRUE;
715
fb6833e0 716 list = tblock ? tblock->remove_headers : NULL;
d43cbe25 717 for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
718 {
719 if (list)
720 {
721 int sep = ':'; /* This is specified as a colon-separated list */
722 uschar *s, *ss;
8bc732e8 723 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 724 {
76146973
JH
725 int len;
726
727 if (i == 0)
8768d548 728 if (!(s = expand_string(s)) && !f.expand_string_forcedfail)
76146973
JH
729 {
730 errno = ERRNO_CHHEADER_FAIL;
731 return FALSE;
732 }
d4ff61d1 733 len = s ? Ustrlen(s) : 0;
511a6c14
JH
734 if (strncmpic(h->text, s, len) != 0) continue;
735 ss = h->text + len;
736 while (*ss == ' ' || *ss == '\t') ss++;
737 if (*ss == ':') break;
738 }
6b46ecc6 739 if (s) { include_header = FALSE; break; }
511a6c14 740 }
6b46ecc6 741 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
742 }
743
744 /* If this header is to be output, try to rewrite it if there are rewriting
745 rules. */
746
747 if (include_header)
748 {
fb6833e0 749 if (tblock && tblock->rewrite_rules)
511a6c14
JH
750 {
751 void *reset_point = store_get(0);
752 header_line *hh;
753
6b46ecc6
JH
754 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
755 tblock->rewrite_existflags, FALSE)))
511a6c14 756 {
42055a33 757 if (!sendfn(tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
758 store_reset(reset_point);
759 continue; /* With the next header line */
760 }
761 }
762
763 /* Either no rewriting rules, or it didn't get rewritten */
764
42055a33 765 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
766 }
767
768 /* Header removed */
769
770 else
511a6c14 771 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
511a6c14
JH
772 }
773
774/* Add on any address-specific headers. If there are multiple addresses,
775they will all have the same headers in order to be batched. The headers
776are chained in reverse order of adding (so several addresses from the
777same alias might share some of them) but we want to output them in the
778opposite order. This is a bit tedious, but there shouldn't be very many
779of them. We just walk the list twice, reversing the pointers each time,
780but on the second time, write out the items.
781
782Headers added to an address by a router are guaranteed to end with a newline.
783*/
784
785if (addr)
786 {
787 int i;
d43cbe25 788 header_line *hprev = addr->prop.extra_headers;
511a6c14
JH
789 header_line *hnext;
790 for (i = 0; i < 2; i++)
ff5aac2b 791 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
792 {
793 hnext = h->next;
794 h->next = hprev;
795 hprev = h;
796 if (i == 1)
797 {
42055a33 798 if (!sendfn(tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
799 DEBUG(D_transport)
800 debug_printf("added header line(s):\n%s---\n", h->text);
801 }
802 }
511a6c14
JH
803 }
804
76146973
JH
805/* If a string containing additional headers exists it is a newline-sep
806list. Expand each item and write out the result. This is done last so that
807if it (deliberately or accidentally) isn't in header format, it won't mess
808up any other headers. An empty string or a forced expansion failure are
809noops. An added header string from a transport may not end with a newline;
810add one if it does not. */
511a6c14 811
fb6833e0 812if (tblock && (list = CUS tblock->add_headers))
511a6c14 813 {
76146973
JH
814 int sep = '\n';
815 uschar * s;
816
6b46ecc6 817 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 818 if ((s = expand_string(s)))
76146973
JH
819 {
820 int len = Ustrlen(s);
821 if (len > 0)
511a6c14 822 {
42055a33
JH
823 if (!sendfn(tctx, s, len)) return FALSE;
824 if (s[len-1] != '\n' && !sendfn(tctx, US"\n", 1))
76146973
JH
825 return FALSE;
826 DEBUG(D_transport)
827 {
828 debug_printf("added header line:\n%s", s);
829 if (s[len-1] != '\n') debug_printf("\n");
830 debug_printf("---\n");
831 }
511a6c14
JH
832 }
833 }
8768d548 834 else if (!f.expand_string_forcedfail)
ff5aac2b 835 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
836 }
837
838/* Separate headers from body with a blank line */
839
42055a33 840return sendfn(tctx, US"\n", 1);
511a6c14
JH
841}
842
843
059ec3d9
PH
844/*************************************************
845* Write the message *
846*************************************************/
847
848/* This function writes the message to the given file descriptor. The headers
849are in the in-store data structure, and the rest of the message is in the open
850file descriptor deliver_datafile. Make sure we start it at the beginning.
851
852. If add_return_path is TRUE, a "return-path:" header is added to the message,
853 containing the envelope sender's address.
854
855. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
856 giving the top-level envelope address that caused this delivery to happen.
857
858. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
859 message. It gives the time and date that delivery took place.
860
861. If check_string is not null, the start of each line is checked for that
862 string. If it is found, it is replaced by escape_string. This used to be
863 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
864
865. If use_crlf is true, newlines are turned into CRLF (SMTP output).
866
867The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
868any writing or reading error, leaving the code in errno intact. Error exits
869can include timeouts for certain transports, which are requested by setting
870transport_write_timeout non-zero.
871
872Arguments:
6b46ecc6 873 tctx
42055a33
JH
874 (fd, msg) Either and fd, to write the message to,
875 or a string: if null write message to allocated space
876 otherwire take content as headers.
6b46ecc6
JH
877 addr (chain of) addresses (for extra headers), or NULL;
878 only the first address is used
879 tblock optional transport instance block (NULL signifies NULL/0):
880 add_headers a string containing one or more headers to add; it is
881 expanded, and must be in correct RFC 822 format as
882 it is transmitted verbatim; NULL => no additions,
883 and so does empty string or forced expansion fail
884 remove_headers a colon-separated list of headers to remove, or NULL
885 rewrite_rules chain of header rewriting rules
886 rewrite_existflags flags for the rewriting rules
887 options bit-wise options:
888 add_return_path if TRUE, add a "return-path" header
889 add_envelope_to if TRUE, add a "envelope-to" header
890 add_delivery_date if TRUE, add a "delivery-date" header
891 use_crlf if TRUE, turn NL into CR LF
892 end_dot if TRUE, send a terminating "." line at the end
893 no_headers if TRUE, omit the headers
894 no_body if TRUE, omit the body
2d14f397
JH
895 check_string a string to check for at the start of lines, or NULL
896 escape_string a string to insert in front of any check string
897 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
898 it is used when returning messages to their senders,
899 and is approximate rather than exact, owing to chunk
900 buffering
059ec3d9
PH
901
902Returns: TRUE on success; FALSE (with errno) on failure.
903 In addition, the global variable transport_count
904 is incremented by the number of bytes written.
905*/
906
d315eda1 907static BOOL
42055a33 908internal_transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 909{
328c5688 910int len, size = 0;
059ec3d9
PH
911
912/* Initialize pointer in output buffer. */
913
42055a33 914transport_write_reset(tctx->options);
059ec3d9
PH
915
916/* Set up the data for start-of-line data checking and escaping */
917
6b46ecc6 918if (tctx->check_string && tctx->escape_string)
059ec3d9 919 {
6b46ecc6 920 nl_check = tctx->check_string;
059ec3d9 921 nl_check_length = Ustrlen(nl_check);
6b46ecc6 922 nl_escape = tctx->escape_string;
059ec3d9
PH
923 nl_escape_length = Ustrlen(nl_escape);
924 }
059ec3d9 925
328c5688
JH
926/* Whether the escaping mechanism is applied to headers or not is controlled by
927an option (set for SMTP, not otherwise). Negate the length if not wanted till
928after the headers. */
929
930if (!(tctx->options & topt_escape_headers))
931 nl_check_length = -nl_check_length;
932
059ec3d9 933/* Write the headers if required, including any that have to be added. If there
328c5688
JH
934are header rewriting rules, apply them. The datasource is not the -D spoolfile
935so temporarily hide the global that adjusts for its format. */
059ec3d9 936
6b46ecc6 937if (!(tctx->options & topt_no_headers))
059ec3d9 938 {
8768d548
JH
939 BOOL save_wireformat = f.spool_file_wireformat;
940 f.spool_file_wireformat = FALSE;
42055a33 941
059ec3d9
PH
942 /* Add return-path: if requested. */
943
6b46ecc6 944 if (tctx->options & topt_add_return_path)
059ec3d9
PH
945 {
946 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 947 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 948 return_path);
328c5688 949 if (!write_chunk(tctx, buffer, n)) goto bad;
059ec3d9
PH
950 }
951
952 /* Add envelope-to: if requested */
953
6b46ecc6 954 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
955 {
956 BOOL first = TRUE;
957 address_item *p;
958 struct aci *plist = NULL;
959 struct aci *dlist = NULL;
960 void *reset_point = store_get(0);
961
328c5688 962 if (!write_chunk(tctx, US"Envelope-to: ", 13)) goto bad;
059ec3d9
PH
963
964 /* Pick up from all the addresses. The plist and dlist variables are
965 anchors for lists of addresses already handled; they have to be defined at
4c04137d 966 this level because write_env_to() calls itself recursively. */
059ec3d9 967
6b46ecc6 968 for (p = tctx->addr; p; p = p->next)
328c5688 969 if (!write_env_to(p, &plist, &dlist, &first, tctx)) goto bad;
059ec3d9
PH
970
971 /* Add a final newline and reset the store used for tracking duplicates */
972
328c5688 973 if (!write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
974 store_reset(reset_point);
975 }
976
977 /* Add delivery-date: if requested. */
978
6b46ecc6 979 if (tctx->options & topt_add_delivery_date)
059ec3d9 980 {
a5ffa9b4
JH
981 uschar * s = tod_stamp(tod_full);
982
983 if ( !write_chunk(tctx, US"Delivery-date: ", 15)
984 || !write_chunk(tctx, s, Ustrlen(s))
985 || !write_chunk(tctx, US"\n", 1)) goto bad;
059ec3d9
PH
986 }
987
988 /* Then the message's headers. Don't write any that are flagged as "old";
989 that means they were rewritten, or are a record of envelope rewriting, or
990 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 991 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 992 addr is not NULL. */
6b46ecc6 993
42055a33 994 if (!transport_headers_send(tctx, &write_chunk))
328c5688
JH
995 {
996bad:
8768d548 997 f.spool_file_wireformat = save_wireformat;
511a6c14 998 return FALSE;
328c5688
JH
999 }
1000
8768d548 1001 f.spool_file_wireformat = save_wireformat;
059ec3d9
PH
1002 }
1003
58fc5fb2
JH
1004/* When doing RFC3030 CHUNKING output, work out how much data would be in a
1005last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
1006(optimally, all of the headers - but it does not matter if we already had to
1007flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 1008(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
1009that further use of write_chunk() will not prepend BDATs.
1010The first BDAT written will also first flush any outstanding MAIL and RCPT
1011commands which were buffered thans to PIPELINING.
1012Commands go out (using a send()) from a different buffer to data (using a
1013write()). They might not end up in the same TCP segment, which is
1014suboptimal. */
59932f7d 1015
65de12cc 1016if (tctx->options & topt_use_bdat)
59932f7d 1017 {
e027f545 1018 off_t fsize;
328c5688 1019 int hsize;
e027f545
JH
1020
1021 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
1022 hsize = 0;
65de12cc 1023 if (!(tctx->options & topt_no_body))
59932f7d
JH
1024 {
1025 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
1026 fsize -= SPOOL_DATA_START_OFFSET;
1027 if (size_limit > 0 && fsize > size_limit)
1028 fsize = size_limit;
e027f545 1029 size = hsize + fsize;
8768d548 1030 if (tctx->options & topt_use_crlf && !f.spool_file_wireformat)
59932f7d 1031 size += body_linecount; /* account for CRLF-expansion */
42055a33
JH
1032
1033 /* With topt_use_bdat we never do dot-stuffing; no need to
1034 account for any expansion due to that. */
59932f7d
JH
1035 }
1036
e027f545
JH
1037 /* If the message is large, emit first a non-LAST chunk with just the
1038 headers, and reap the command responses. This lets us error out early
1039 on RCPT rejects rather than sending megabytes of data. Include headers
1040 on the assumption they are cheap enough and some clever implementations
1041 might errorcheck them too, on-the-fly, and reject that chunk. */
1042
1043 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
1044 {
58fc5fb2 1045 DEBUG(D_transport)
4c04137d 1046 debug_printf("sending small initial BDAT; hsize=%d\n", hsize);
d2aa036b 1047 if ( tctx->chunk_cb(tctx, hsize, 0) != OK
42055a33 1048 || !transport_write_block(tctx, deliver_out_buffer, hsize, FALSE)
d2aa036b 1049 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
e027f545
JH
1050 )
1051 return FALSE;
1052 chunk_ptr = deliver_out_buffer;
1053 size -= hsize;
1054 }
1055
d2aa036b
JH
1056 /* Emit a LAST datachunk command, and unmark the context for further
1057 BDAT commands. */
59932f7d 1058
d2aa036b 1059 if (tctx->chunk_cb(tctx, size, tc_chunk_last) != OK)
511a6c14 1060 return FALSE;
6d5c916c 1061 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
1062 }
1063
1064/* If the body is required, ensure that the data for check strings (formerly
1065the "from hack") is enabled by negating the length if necessary. (It will be
1066negative in cases where it isn't to apply to the headers). Then ensure the body
1067is positioned at the start of its file (following the message id), then write
1068it, applying the size limit if required. */
1069
328c5688
JH
1070/* If we have a wireformat -D file (CRNL lines, non-dotstuffed, no ending dot)
1071and we want to send a body without dotstuffing or ending-dot, in-clear,
1072then we can just dump it using sendfile.
1073This should get used for CHUNKING output and also for writing the -K file for
1074dkim signing, when we had CHUNKING input. */
1075
7d758a6a 1076#ifdef OS_SENDFILE
8768d548 1077if ( f.spool_file_wireformat
328c5688
JH
1078 && !(tctx->options & (topt_no_body | topt_end_dot))
1079 && !nl_check_length
74f1a423 1080 && tls_out.active.sock != tctx->u.fd
328c5688
JH
1081 )
1082 {
1083 ssize_t copied = 0;
1084 off_t offset = SPOOL_DATA_START_OFFSET;
1085
1086 /* Write out any header data in the buffer */
1087
1088 if ((len = chunk_ptr - deliver_out_buffer) > 0)
1089 {
1090 if (!transport_write_block(tctx, deliver_out_buffer, len, TRUE))
1091 return FALSE;
1092 size -= len;
1093 }
1094
1095 DEBUG(D_transport) debug_printf("using sendfile for body\n");
1096
1097 while(size > 0)
1098 {
7d758a6a 1099 if ((copied = os_sendfile(tctx->u.fd, deliver_datafile, &offset, size)) <= 0) break;
328c5688
JH
1100 size -= copied;
1101 }
1102 return copied >= 0;
1103 }
1104#else
1105DEBUG(D_transport) debug_printf("cannot use sendfile for body: no support\n");
1106#endif
1107
1108DEBUG(D_transport)
1109 if (!(tctx->options & topt_no_body))
1110 debug_printf("cannot use sendfile for body: %s\n",
8768d548 1111 !f.spool_file_wireformat ? "spoolfile not wireformat"
328c5688
JH
1112 : tctx->options & topt_end_dot ? "terminating dot wanted"
1113 : nl_check_length ? "dot- or From-stuffing wanted"
1114 : "TLS output wanted");
1115
6b46ecc6 1116if (!(tctx->options & topt_no_body))
059ec3d9 1117 {
e027f545
JH
1118 int size = size_limit;
1119
059ec3d9
PH
1120 nl_check_length = abs(nl_check_length);
1121 nl_partial_match = 0;
d4ff61d1
JH
1122 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1123 return FALSE;
59932f7d
JH
1124 while ( (len = MAX(DELIVER_IN_BUFFER_SIZE, size)) > 0
1125 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1126 {
42055a33 1127 if (!write_chunk(tctx, deliver_in_buffer, len))
59932f7d 1128 return FALSE;
e027f545 1129 size -= len;
059ec3d9
PH
1130 }
1131
059ec3d9
PH
1132 /* A read error on the body will have left len == -1 and errno set. */
1133
1134 if (len != 0) return FALSE;
c0940526 1135 }
059ec3d9 1136
f64e8b5f 1137/* Finished with the check string, and spool-format consideration */
059ec3d9 1138
c0940526 1139nl_check_length = nl_escape_length = 0;
8768d548 1140f.spool_file_wireformat = FALSE;
c0940526
PP
1141
1142/* If requested, add a terminating "." line (SMTP output). */
1143
42055a33 1144if (tctx->options & topt_end_dot && !write_chunk(tctx, US".\n", 2))
c0940526 1145 return FALSE;
059ec3d9
PH
1146
1147/* Write out any remaining data in the buffer before returning. */
1148
1149return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
42055a33 1150 transport_write_block(tctx, deliver_out_buffer, len, FALSE);
f7572e5a 1151}
80a47a2c 1152
f7572e5a
TK
1153
1154
328c5688 1155
059ec3d9
PH
1156/*************************************************
1157* External interface to write the message *
1158*************************************************/
1159
1160/* If there is no filtering required, call the internal function above to do
1161the real work, passing over all the arguments from this function. Otherwise,
1162set up a filtering process, fork another process to call the internal function
1163to write to the filter, and in this process just suck from the filter and write
42055a33
JH
1164down the fd in the transport context. At the end, tidy up the pipes and the
1165processes.
059ec3d9
PH
1166
1167Arguments: as for internal_transport_write_message() above
1168
1169Returns: TRUE on success; FALSE (with errno) for any failure
1170 transport_count is incremented by the number of bytes written
1171*/
1172
1173BOOL
42055a33 1174transport_write_message(transport_ctx * tctx, int size_limit)
059ec3d9 1175{
059ec3d9 1176BOOL last_filter_was_NL = TRUE;
8768d548 1177BOOL save_spool_file_wireformat = f.spool_file_wireformat;
059ec3d9 1178int rc, len, yield, fd_read, fd_write, save_errno;
806c3df9 1179int pfd[2] = {-1, -1};
059ec3d9 1180pid_t filter_pid, write_pid;
6b46ecc6 1181
8768d548 1182f.transport_filter_timed_out = FALSE;
2e2a30b4 1183
059ec3d9
PH
1184/* If there is no filter command set up, call the internal function that does
1185the actual work, passing it the incoming fd, and return its result. */
1186
6ff55e50
JH
1187if ( !transport_filter_argv
1188 || !*transport_filter_argv
1189 || !**transport_filter_argv
1190 )
42055a33 1191 return internal_transport_write_message(tctx, size_limit);
059ec3d9
PH
1192
1193/* Otherwise the message must be written to a filter process and read back
1194before being written to the incoming fd. First set up the special processing to
1195be done during the copying. */
1196
059ec3d9
PH
1197nl_partial_match = -1;
1198
6b46ecc6 1199if (tctx->check_string && tctx->escape_string)
059ec3d9 1200 {
6b46ecc6 1201 nl_check = tctx->check_string;
059ec3d9 1202 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1203 nl_escape = tctx->escape_string;
059ec3d9
PH
1204 nl_escape_length = Ustrlen(nl_escape);
1205 }
1206else nl_check_length = nl_escape_length = 0;
1207
1208/* Start up a subprocess to run the command. Ensure that our main fd will
1209be closed when the subprocess execs, but remove the flag afterwards.
1210(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1211process to deliver another message.) We get back stdin/stdout file descriptors.
1212If the process creation failed, give an error return. */
1213
1214fd_read = -1;
1215fd_write = -1;
1216save_errno = 0;
1217yield = FALSE;
1218write_pid = (pid_t)(-1);
1219
92b0827a 1220 {
42055a33
JH
1221 int bits = fcntl(tctx->u.fd, F_GETFD);
1222 (void)fcntl(tctx->u.fd, F_SETFD, bits | FD_CLOEXEC);
92b0827a
JH
1223 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
1224 &fd_write, &fd_read, FALSE);
42055a33 1225 (void)fcntl(tctx->u.fd, F_SETFD, bits & ~FD_CLOEXEC);
92b0827a 1226 }
059ec3d9
PH
1227if (filter_pid < 0) goto TIDY_UP; /* errno set */
1228
1229DEBUG(D_transport)
fb6833e0 1230 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1231 (int)filter_pid, fd_write, fd_read);
1232
1233/* Fork subprocess to write the message to the filter, and return the result
1234via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1235smtp dots, or check string processing. */
1236
1237if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
1238if ((write_pid = fork()) == 0)
1239 {
1240 BOOL rc;
f1e894f3
PH
1241 (void)close(fd_read);
1242 (void)close(pfd[pipe_read]);
059ec3d9 1243 nl_check_length = nl_escape_length = 0;
6b46ecc6 1244
42055a33 1245 tctx->u.fd = fd_write;
fb6833e0 1246 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1247 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6 1248
42055a33 1249 rc = internal_transport_write_message(tctx, size_limit);
6b46ecc6 1250
059ec3d9 1251 save_errno = errno;
1ac6b2e7
JH
1252 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1253 != sizeof(BOOL)
1254 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1255 != sizeof(int)
6b46ecc6 1256 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1ac6b2e7 1257 != sizeof(int)
32dfdf8b
JH
1258 || write(pfd[pipe_write], (void *)&tctx->addr->delivery_usec, sizeof(int))
1259 != sizeof(int)
1ac6b2e7
JH
1260 )
1261 rc = FALSE; /* compiler quietening */
059ec3d9
PH
1262 _exit(0);
1263 }
1264save_errno = errno;
1265
1266/* Parent process: close our copy of the writing subprocess' pipes. */
1267
f1e894f3
PH
1268(void)close(pfd[pipe_write]);
1269(void)close(fd_write);
059ec3d9
PH
1270fd_write = -1;
1271
1272/* Writing process creation failed */
1273
1274if (write_pid < 0)
1275 {
1276 errno = save_errno; /* restore */
1277 goto TIDY_UP;
1278 }
1279
1280/* When testing, let the subprocess get going */
1281
8768d548 1282if (f.running_in_test_harness) millisleep(250);
059ec3d9
PH
1283
1284DEBUG(D_transport)
1285 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1286
1287/* Copy the message from the filter to the output fd. A read error leaves len
1288== -1 and errno set. We need to apply a timeout to the read, to cope with
1289the case when the filter gets stuck, but it can be quite a long one. The
1290default is 5m, but this is now configurable. */
1291
1292DEBUG(D_transport) debug_printf("copying from the filter\n");
1293
1294/* Copy the output of the filter, remembering if the last character was NL. If
1295no data is returned, that counts as "ended with NL" (default setting of the
328c5688
JH
1296variable is TRUE). The output should always be unix-format as we converted
1297any wireformat source on writing input to the filter. */
059ec3d9 1298
8768d548 1299f.spool_file_wireformat = FALSE;
059ec3d9
PH
1300chunk_ptr = deliver_out_buffer;
1301
1302for (;;)
1303 {
1304 sigalrm_seen = FALSE;
c2a1bba0 1305 ALARM(transport_filter_timeout);
059ec3d9 1306 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
c2a1bba0 1307 ALARM_CLR(0);
059ec3d9
PH
1308 if (sigalrm_seen)
1309 {
1310 errno = ETIMEDOUT;
8768d548 1311 f.transport_filter_timed_out = TRUE;
059ec3d9
PH
1312 goto TIDY_UP;
1313 }
1314
1315 /* If the read was successful, write the block down the original fd,
1316 remembering whether it ends in \n or not. */
1317
1318 if (len > 0)
1319 {
42055a33 1320 if (!write_chunk(tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1321 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1322 }
1323
1324 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1325
1326 else
1327 {
1328 if (len == 0) yield = TRUE;
1329 break;
1330 }
1331 }
1332
1333/* Tidying up code. If yield = FALSE there has been an error and errno is set
1334to something. Ensure the pipes are all closed and the processes are removed. If
1335there has been an error, kill the processes before waiting for them, just to be
1336sure. Also apply a paranoia timeout. */
1337
1338TIDY_UP:
8768d548 1339f.spool_file_wireformat = save_spool_file_wireformat;
059ec3d9
PH
1340save_errno = errno;
1341
f1e894f3
PH
1342(void)close(fd_read);
1343if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1344
1345if (!yield)
1346 {
1347 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1348 if (write_pid > 0) kill(write_pid, SIGKILL);
1349 }
1350
1351/* Wait for the filter process to complete. */
1352
1353DEBUG(D_transport) debug_printf("waiting for filter process\n");
1354if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1355 {
1356 yield = FALSE;
1357 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1358 tctx->addr->more_errno = rc;
059ec3d9
PH
1359 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1360 }
1361
1362/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1363read the results from its pipe, provided we haven't already had a filter
35af9f61 1364process failure. */
059ec3d9
PH
1365
1366DEBUG(D_transport) debug_printf("waiting for writing process\n");
1367if (write_pid > 0)
1368 {
35af9f61
PH
1369 rc = child_close(write_pid, 30);
1370 if (yield)
8e669ac1 1371 if (rc == 0)
35af9f61
PH
1372 {
1373 BOOL ok;
4dc2379a
JH
1374 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
1375 {
1376 DEBUG(D_transport)
1377 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1378 save_errno = ERRNO_FILTER_FAIL;
1379 yield = FALSE;
1380 }
1381 else if (!ok)
35af9f61 1382 {
4dc2379a 1383 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
32dfdf8b
JH
1384 dummy = read(pfd[pipe_read], (void *)&tctx->addr->more_errno, sizeof(int));
1385 dummy = read(pfd[pipe_read], (void *)&tctx->addr->delivery_usec, sizeof(int));
cab0c277 1386 dummy = dummy; /* compiler quietening */
35af9f61
PH
1387 yield = FALSE;
1388 }
1389 }
1390 else
059ec3d9 1391 {
059ec3d9 1392 yield = FALSE;
35af9f61 1393 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1394 tctx->addr->more_errno = rc;
35af9f61 1395 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1396 }
059ec3d9 1397 }
f1e894f3 1398(void)close(pfd[pipe_read]);
059ec3d9
PH
1399
1400/* If there have been no problems we can now add the terminating "." if this is
1401SMTP output, turning off escaping beforehand. If the last character from the
1402filter was not NL, insert a NL to make the SMTP protocol work. */
1403
1404if (yield)
1405 {
1406 nl_check_length = nl_escape_length = 0;
8768d548 1407 f.spool_file_wireformat = FALSE;
6b46ecc6 1408 if ( tctx->options & topt_end_dot
ff5aac2b 1409 && ( last_filter_was_NL
42055a33
JH
1410 ? !write_chunk(tctx, US".\n", 2)
1411 : !write_chunk(tctx, US"\n.\n", 3)
ff5aac2b 1412 ) )
059ec3d9 1413 yield = FALSE;
059ec3d9
PH
1414
1415 /* Write out any remaining data in the buffer. */
1416
1417 else
ff5aac2b 1418 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
42055a33 1419 || transport_write_block(tctx, deliver_out_buffer, len, FALSE);
059ec3d9 1420 }
ff5aac2b
JH
1421else
1422 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1423
1424DEBUG(D_transport)
1425 {
1426 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1427 if (!yield)
6b46ecc6 1428 debug_printf("errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1429 }
1430
1431return yield;
1432}
1433
1434
1435
1436
1437
1438/*************************************************
1439* Update waiting database *
1440*************************************************/
1441
1442/* This is called when an address is deferred by remote transports that are
1443capable of sending more than one message over one connection. A database is
1444maintained for each transport, keeping track of which messages are waiting for
1445which hosts. The transport can then consult this when eventually a successful
1446delivery happens, and if it finds that another message is waiting for the same
1447host, it can fire up a new process to deal with it using the same connection.
1448
1449The database records are keyed by host name. They can get full if there are
1450lots of messages waiting, and so there is a continuation mechanism for them.
1451
1452Each record contains a list of message ids, packed end to end without any
1453zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1454in this record, and the sequence field says if there are any other records for
1455this host. If the sequence field is 0, there are none. If it is 1, then another
1456record with the name <hostname>:0 exists; if it is 2, then two other records
1457with sequence numbers 0 and 1 exist, and so on.
1458
1459Currently, an exhaustive search of all continuation records has to be done to
1460determine whether to add a message id to a given record. This shouldn't be
1461too bad except in extreme cases. I can't figure out a *simple* way of doing
1462better.
1463
1464Old records should eventually get swept up by the exim_tidydb utility.
1465
1466Arguments:
f6c332bd 1467 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1468 tpname name of the transport
1469
1470Returns: nothing
1471*/
1472
1473void
1474transport_update_waiting(host_item *hostlist, uschar *tpname)
1475{
55414b25 1476const uschar *prevname = US"";
059ec3d9
PH
1477host_item *host;
1478open_db dbblock;
1479open_db *dbm_file;
1480
7a0743eb
PH
1481DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1482
059ec3d9
PH
1483/* Open the database for this transport */
1484
36d295f1
JH
1485if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
1486 O_RDWR, &dbblock, TRUE)))
1487 return;
059ec3d9
PH
1488
1489/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1490that the message id is in each host record. */
059ec3d9 1491
36d295f1 1492for (host = hostlist; host; host = host->next)
059ec3d9
PH
1493 {
1494 BOOL already = FALSE;
1495 dbdata_wait *host_record;
1496 uschar *s;
1497 int i, host_length;
36d295f1 1498 uschar buffer[256];
059ec3d9 1499
059ec3d9
PH
1500 /* Skip if this is the same host as we just processed; otherwise remember
1501 the name for next time. */
1502
1503 if (Ustrcmp(prevname, host->name) == 0) continue;
1504 prevname = host->name;
1505
1506 /* Look up the host record; if there isn't one, make an empty one. */
1507
36d295f1 1508 if (!(host_record = dbfn_read(dbm_file, host->name)))
059ec3d9
PH
1509 {
1510 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH);
1511 host_record->count = host_record->sequence = 0;
1512 }
1513
1514 /* Compute the current length */
1515
1516 host_length = host_record->count * MESSAGE_ID_LENGTH;
1517
1518 /* Search the record to see if the current message is already in it. */
1519
1520 for (s = host_record->text; s < host_record->text + host_length;
1521 s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1522 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1523 { already = TRUE; break; }
059ec3d9
PH
1524
1525 /* If we haven't found this message in the main record, search any
1526 continuation records that exist. */
1527
1528 for (i = host_record->sequence - 1; i >= 0 && !already; i--)
1529 {
1530 dbdata_wait *cont;
1531 sprintf(CS buffer, "%.200s:%d", host->name, i);
36d295f1 1532 if ((cont = dbfn_read(dbm_file, buffer)))
059ec3d9
PH
1533 {
1534 int clen = cont->count * MESSAGE_ID_LENGTH;
1535 for (s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1536 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1537 { already = TRUE; break; }
059ec3d9
PH
1538 }
1539 }
1540
1541 /* If this message is already in a record, no need to update. */
1542
7a0743eb
PH
1543 if (already)
1544 {
1545 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1546 continue;
1547 }
059ec3d9
PH
1548
1549
1550 /* If this record is full, write it out with a new name constructed
1551 from the sequence number, increase the sequence number, and empty
1552 the record. */
1553
1554 if (host_record->count >= WAIT_NAME_MAX)
1555 {
1556 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1557 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
1558 host_record->sequence++;
1559 host_record->count = 0;
1560 host_length = 0;
1561 }
1562
1563 /* If this record is not full, increase the size of the record to
1564 allow for one new message id. */
1565
1566 else
1567 {
1568 dbdata_wait *newr =
1569 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH);
1570 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1571 host_record = newr;
1572 }
1573
1574 /* Now add the new name on the end */
1575
1576 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1577 host_record->count++;
1578 host_length += MESSAGE_ID_LENGTH;
1579
1580 /* Update the database */
1581
1582 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1583 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1584 }
1585
1586/* All now done */
1587
1588dbfn_close(dbm_file);
1589}
1590
1591
1592
1593
1594/*************************************************
1595* Test for waiting messages *
1596*************************************************/
1597
1598/* This function is called by a remote transport which uses the previous
1599function to remember which messages are waiting for which remote hosts. It's
1600called after a successful delivery and its job is to check whether there is
1601another message waiting for the same host. However, it doesn't do this if the
1602current continue sequence is greater than the maximum supplied as an argument,
1603or greater than the global connection_max_messages, which, if set, overrides.
1604
1605Arguments:
1606 transport_name name of the transport
1607 hostname name of the host
1608 local_message_max maximum number of messages down one connection
1609 as set by the caller transport
1610 new_message_id set to the message id of a waiting message
1611 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1612 oicf_func function to call to validate if it is ok to send
1613 to this message_id from the current instance.
1614 oicf_data opaque data for oicf_func
059ec3d9
PH
1615
1616Returns: TRUE if new_message_id set; FALSE otherwise
1617*/
1618
a39bd74d
JB
1619typedef struct msgq_s
1620{
1621 uschar message_id [MESSAGE_ID_LENGTH + 1];
1622 BOOL bKeep;
1623} msgq_t;
1624
059ec3d9 1625BOOL
55414b25 1626transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1627 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1628{
1629dbdata_wait *host_record;
0539a19d 1630int host_length;
059ec3d9
PH
1631open_db dbblock;
1632open_db *dbm_file;
059ec3d9 1633
a39bd74d 1634int i;
a39bd74d 1635struct stat statbuf;
a39bd74d 1636
059ec3d9
PH
1637*more = FALSE;
1638
1639DEBUG(D_transport)
1640 {
1641 debug_printf("transport_check_waiting entered\n");
1642 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1643 continue_sequence, local_message_max, connection_max_messages);
1644 }
1645
1646/* Do nothing if we have hit the maximum number that can be send down one
1647connection. */
1648
1649if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1650if (local_message_max > 0 && continue_sequence >= local_message_max)
1651 {
1652 DEBUG(D_transport)
1653 debug_printf("max messages for one connection reached: returning\n");
1654 return FALSE;
1655 }
1656
1657/* Open the waiting information database. */
1658
36d295f1
JH
1659if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
1660 O_RDWR, &dbblock, TRUE)))
1661 return FALSE;
059ec3d9
PH
1662
1663/* See if there is a record for this host; if not, there's nothing to do. */
1664
789f8a4f 1665if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1666 {
1667 dbfn_close(dbm_file);
1668 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1669 return FALSE;
1670 }
1671
1672/* If the data in the record looks corrupt, just log something and
1673don't try to use it. */
1674
1675if (host_record->count > WAIT_NAME_MAX)
1676 {
1677 dbfn_close(dbm_file);
1678 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1679 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1680 return FALSE;
1681 }
1682
1683/* Scan the message ids in the record from the end towards the beginning,
1684until one is found for which a spool file actually exists. If the record gets
1685emptied, delete it and continue with any continuation records that may exist.
1686*/
1687
a39bd74d
JB
1688/* For Bug 1141, I refactored this major portion of the routine, it is risky
1689but the 1 off will remain without it. This code now allows me to SKIP over
1690a message I do not want to send out on this run. */
059ec3d9 1691
a39bd74d
JB
1692host_length = host_record->count * MESSAGE_ID_LENGTH;
1693
1694while (1)
059ec3d9 1695 {
789f8a4f
JH
1696 msgq_t *msgq;
1697 int msgq_count = 0;
1698 int msgq_actual = 0;
1699 BOOL bFound = FALSE;
1700 BOOL bContinuation = FALSE;
1701
a39bd74d 1702 /* create an array to read entire message queue into memory for processing */
059ec3d9 1703
40c90bca 1704 msgq = store_malloc(sizeof(msgq_t) * host_record->count);
a39bd74d
JB
1705 msgq_count = host_record->count;
1706 msgq_actual = msgq_count;
059ec3d9 1707
a39bd74d 1708 for (i = 0; i < host_record->count; ++i)
059ec3d9 1709 {
a39bd74d
JB
1710 msgq[i].bKeep = TRUE;
1711
1712 Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1713 MESSAGE_ID_LENGTH);
a39bd74d
JB
1714 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1715 }
1716
1717 /* first thing remove current message id if it exists */
059ec3d9 1718
a39bd74d
JB
1719 for (i = 0; i < msgq_count; ++i)
1720 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1721 {
1722 msgq[i].bKeep = FALSE;
1723 break;
1724 }
1725
1726 /* now find the next acceptable message_id */
1727
a39bd74d
JB
1728 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1729 {
41313d92
JH
1730 uschar subdir[2];
1731
1732 subdir[0] = split_spool_directory ? msgq[i].message_id[5] : 0;
1733 subdir[1] = 0;
059ec3d9 1734
41313d92
JH
1735 if (Ustat(spool_fname(US"input", subdir, msgq[i].message_id, US"-D"),
1736 &statbuf) != 0)
a39bd74d
JB
1737 msgq[i].bKeep = FALSE;
1738 else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data))
059ec3d9 1739 {
a39bd74d
JB
1740 Ustrcpy(new_message_id, msgq[i].message_id);
1741 msgq[i].bKeep = FALSE;
1742 bFound = TRUE;
059ec3d9
PH
1743 break;
1744 }
1745 }
1746
a39bd74d
JB
1747 /* re-count */
1748 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1749 if (msgq[i].bKeep)
1750 msgq_actual++;
1751
1752 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1753 memory queue */
a39bd74d
JB
1754
1755 if (msgq_actual <= 0)
1756 {
1757 host_length = 0;
1758 host_record->count = 0;
1759 }
1760 else
1761 {
1762 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1763 host_record->count = msgq_actual;
1764
1765 if (msgq_actual < msgq_count)
1766 {
1767 int new_count;
1768 for (new_count = 0, i = 0; i < msgq_count; ++i)
1769 if (msgq[i].bKeep)
1770 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1771 msgq[i].message_id, MESSAGE_ID_LENGTH);
1772
1773 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1774 }
1775 }
1776
36d295f1 1777 /* Check for a continuation record. */
a39bd74d 1778
059ec3d9
PH
1779 while (host_length <= 0)
1780 {
1781 int i;
a39bd74d 1782 dbdata_wait * newr = NULL;
36d295f1 1783 uschar buffer[256];
059ec3d9
PH
1784
1785 /* Search for a continuation */
1786
a39bd74d 1787 for (i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1788 {
1789 sprintf(CS buffer, "%.200s:%d", hostname, i);
1790 newr = dbfn_read(dbm_file, buffer);
1791 }
1792
1793 /* If no continuation, delete the current and break the loop */
1794
a39bd74d 1795 if (!newr)
059ec3d9
PH
1796 {
1797 dbfn_delete(dbm_file, hostname);
1798 break;
1799 }
1800
1801 /* Else replace the current with the continuation */
1802
1803 dbfn_delete(dbm_file, buffer);
1804 host_record = newr;
1805 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1806
a39bd74d
JB
1807 bContinuation = TRUE;
1808 }
059ec3d9 1809
789f8a4f
JH
1810 if (bFound) /* Usual exit from main loop */
1811 {
40c90bca 1812 store_free (msgq);
a39bd74d 1813 break;
789f8a4f 1814 }
059ec3d9
PH
1815
1816 /* If host_length <= 0 we have emptied a record and not found a good message,
1817 and there are no continuation records. Otherwise there is a continuation
1818 record to process. */
1819
1820 if (host_length <= 0)
1821 {
1822 dbfn_close(dbm_file);
1823 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1824 return FALSE;
1825 }
a39bd74d
JB
1826
1827 /* we were not able to find an acceptable message, nor was there a
1828 * continuation record. So bug out, outer logic will clean this up.
1829 */
1830
1831 if (!bContinuation)
1832 {
789f8a4f 1833 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1834 dbfn_close(dbm_file);
1835 return FALSE;
1836 }
a39bd74d 1837
40c90bca 1838 store_free(msgq);
789f8a4f 1839 } /* we need to process a continuation record */
059ec3d9
PH
1840
1841/* Control gets here when an existing message has been encountered; its
1842id is in new_message_id, and host_length is the revised length of the
1843host record. If it is zero, the record has been removed. Update the
1844record if required, close the database, and return TRUE. */
1845
1846if (host_length > 0)
1847 {
1848 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1849
059ec3d9
PH
1850 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1851 *more = TRUE;
1852 }
1853
1854dbfn_close(dbm_file);
1855return TRUE;
1856}
1857
059ec3d9
PH
1858/*************************************************
1859* Deliver waiting message down same socket *
1860*************************************************/
1861
57cc2785
JH
1862/* Just the regain-root-privilege exec portion */
1863void
1864transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
1865 const uschar *hostaddress, uschar *id, int socket_fd)
1866{
57cc2785
JH
1867int i = 20;
1868const uschar **argv;
1869
1870/* Set up the calling arguments; use the standard function for the basics,
1871but we have a number of extras that may be added. */
1872
1873argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
1874
8768d548 1875if (f.smtp_authenticated) argv[i++] = US"-MCA";
14de8063
JH
1876if (smtp_peer_options & OPTION_CHUNKING) argv[i++] = US"-MCK";
1877if (smtp_peer_options & OPTION_DSN) argv[i++] = US"-MCD";
1878if (smtp_peer_options & OPTION_PIPE) argv[i++] = US"-MCP";
1879if (smtp_peer_options & OPTION_SIZE) argv[i++] = US"-MCS";
57cc2785 1880#ifdef SUPPORT_TLS
14de8063 1881if (smtp_peer_options & OPTION_TLS)
74f1a423 1882 if (tls_out.active.sock >= 0 || continue_proxy_cipher)
57cc2785
JH
1883 {
1884 argv[i++] = US"-MCt";
1885 argv[i++] = sending_ip_address;
1886 argv[i++] = string_sprintf("%d", sending_port);
74f1a423 1887 argv[i++] = tls_out.active.sock >= 0 ? tls_out.cipher : continue_proxy_cipher;
57cc2785
JH
1888 }
1889 else
1890 argv[i++] = US"-MCT";
1891#endif
1892
1893if (queue_run_pid != (pid_t)0)
1894 {
1895 argv[i++] = US"-MCQ";
1896 argv[i++] = string_sprintf("%d", queue_run_pid);
1897 argv[i++] = string_sprintf("%d", queue_run_pipe);
1898 }
1899
1900argv[i++] = US"-MC";
1901argv[i++] = US transport_name;
1902argv[i++] = US hostname;
1903argv[i++] = US hostaddress;
1904argv[i++] = string_sprintf("%d", continue_sequence + 1);
1905argv[i++] = id;
1906argv[i++] = NULL;
1907
1908/* Arrange for the channel to be on stdin. */
1909
1910if (socket_fd != 0)
1911 {
1912 (void)dup2(socket_fd, 0);
1913 (void)close(socket_fd);
1914 }
1915
1916DEBUG(D_exec) debug_print_argv(argv);
1917exim_nullstd(); /* Ensure std{out,err} exist */
1918execv(CS argv[0], (char *const *)argv);
1919
1920DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
1921_exit(errno); /* Note: must be _exit(), NOT exit() */
1922}
1923
1924
1925
059ec3d9
PH
1926/* Fork a new exim process to deliver the message, and do a re-exec, both to
1927get a clean delivery process, and to regain root privilege in cases where it
1928has been given away.
1929
1930Arguments:
1931 transport_name to pass to the new process
1932 hostname ditto
1933 hostaddress ditto
1934 id the new message to process
1935 socket_fd the connected socket
1936
1937Returns: FALSE if fork fails; TRUE otherwise
1938*/
1939
1940BOOL
55414b25
JH
1941transport_pass_socket(const uschar *transport_name, const uschar *hostname,
1942 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
1943{
1944pid_t pid;
1945int status;
1946
1947DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
1948
1949if ((pid = fork()) == 0)
1950 {
059ec3d9
PH
1951 /* Disconnect entirely from the parent process. If we are running in the
1952 test harness, wait for a bit to allow the previous process time to finish,
1953 write the log, etc., so that the output is always in the same order for
1954 automatic comparison. */
1955
b7d3afcf
JH
1956 if ((pid = fork()) != 0)
1957 {
1958 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (final-pid %d)\n", pid);
1959 _exit(EXIT_SUCCESS);
1960 }
8768d548 1961 if (f.running_in_test_harness) sleep(1);
059ec3d9 1962
57cc2785
JH
1963 transport_do_pass_socket(transport_name, hostname, hostaddress,
1964 id, socket_fd);
059ec3d9
PH
1965 }
1966
1967/* If the process creation succeeded, wait for the first-level child, which
1968immediately exits, leaving the second level process entirely disconnected from
1969this one. */
1970
1971if (pid > 0)
1972 {
1973 int rc;
1974 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
b7d3afcf 1975 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded (inter-pid %d)\n", pid);
059ec3d9
PH
1976 return TRUE;
1977 }
1978else
1979 {
1980 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
1981 strerror(errno));
1982 return FALSE;
1983 }
1984}
1985
1986
1987
1988/*************************************************
1989* Set up direct (non-shell) command *
1990*************************************************/
1991
1992/* This function is called when a command line is to be parsed and executed
1993directly, without the use of /bin/sh. It is called by the pipe transport,
1994the queryprogram router, and also from the main delivery code when setting up a
1995transport filter process. The code for ETRN also makes use of this; in that
1996case, no addresses are passed.
1997
1998Arguments:
1999 argvptr pointer to anchor for argv vector
55414b25 2000 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
2001 expand_arguments true if expansion is to occur
2002 expand_failed error value to set if expansion fails; not relevant if
2003 addr == NULL
2004 addr chain of addresses, or NULL
2005 etext text for use in error messages
2006 errptr where to put error message if addr is NULL;
2007 otherwise it is put in the first address
2008
2009Returns: TRUE if all went well; otherwise an error will be
2010 set in the first address and FALSE returned
2011*/
2012
2013BOOL
55414b25
JH
2014transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2015 BOOL expand_arguments, int expand_failed, address_item *addr,
2016 uschar *etext, uschar **errptr)
059ec3d9
PH
2017{
2018address_item *ad;
55414b25 2019const uschar **argv;
059ec3d9
PH
2020uschar *s, *ss;
2021int address_count = 0;
2022int argcount = 0;
2023int i, max_args;
2024
2025/* Get store in which to build an argument list. Count the number of addresses
2026supplied, and allow for that many arguments, plus an additional 60, which
2027should be enough for anybody. Multiple addresses happen only when the local
2028delivery batch option is set. */
2029
2030for (ad = addr; ad != NULL; ad = ad->next) address_count++;
2031max_args = address_count + 60;
2032*argvptr = argv = store_get((max_args+1)*sizeof(uschar *));
2033
2034/* Split the command up into arguments terminated by white space. Lose
2035trailing space at the start and end. Double-quoted arguments can contain \\ and
2036\" escapes and so can be handled by the standard function; single-quoted
2037arguments are verbatim. Copy each argument into a new string. */
2038
2039s = cmd;
2040while (isspace(*s)) s++;
2041
2042while (*s != 0 && argcount < max_args)
2043 {
2044 if (*s == '\'')
2045 {
2046 ss = s + 1;
2047 while (*ss != 0 && *ss != '\'') ss++;
2048 argv[argcount++] = ss = store_get(ss - s++);
2049 while (*s != 0 && *s != '\'') *ss++ = *s++;
2050 if (*s != 0) s++;
2051 *ss++ = 0;
2052 }
55414b25 2053 else argv[argcount++] = string_copy(string_dequote(CUSS &s));
059ec3d9
PH
2054 while (isspace(*s)) s++;
2055 }
2056
5903c6ff 2057argv[argcount] = US 0;
059ec3d9
PH
2058
2059/* If *s != 0 we have run out of argument slots. */
2060
2061if (*s != 0)
2062 {
2063 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2064 "%s", cmd, etext);
2065 if (addr != NULL)
2066 {
2067 addr->transport_return = FAIL;
2068 addr->message = msg;
2069 }
2070 else *errptr = msg;
2071 return FALSE;
2072 }
2073
2074/* Expand each individual argument if required. Expansion happens for pipes set
2075up in filter files and with directly-supplied commands. It does not happen if
2076the pipe comes from a traditional .forward file. A failing expansion is a big
2077disaster if the command came from Exim's configuration; if it came from a user
2078it is just a normal failure. The expand_failed value is used as the error value
2079to cater for these two cases.
2080
2081An argument consisting just of the text "$pipe_addresses" is treated specially.
2082It is not passed to the general expansion function. Instead, it is replaced by
2083a number of arguments, one for each address. This avoids problems with shell
2084metacharacters and spaces in addresses.
2085
2086If the parent of the top address has an original part of "system-filter", this
2087pipe was set up by the system filter, and we can permit the expansion of
2088$recipients. */
2089
2090DEBUG(D_transport)
2091 {
2092 debug_printf("direct command:\n");
5903c6ff 2093 for (i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2094 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2095 }
2096
2097if (expand_arguments)
2098 {
2099 BOOL allow_dollar_recipients = addr != NULL &&
2100 addr->parent != NULL &&
2101 Ustrcmp(addr->parent->address, "system-filter") == 0;
2102
5903c6ff 2103 for (i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2104 {
2105
2106 /* Handle special fudge for passing an address list */
2107
2108 if (addr != NULL &&
2109 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2110 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2111 {
2112 int additional;
2113
2114 if (argcount + address_count - 1 > max_args)
2115 {
2116 addr->transport_return = FAIL;
2117 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2118 "in %s", cmd, etext);
2119 return FALSE;
2120 }
2121
2122 additional = address_count - 1;
2123 if (additional > 0)
2124 memmove(argv + i + 1 + additional, argv + i + 1,
2125 (argcount - i)*sizeof(uschar *));
2126
09792322
NK
2127 for (ad = addr; ad != NULL; ad = ad->next) {
2128 argv[i++] = ad->address;
2129 argcount++;
2130 }
2131
700d22f3
PP
2132 /* Subtract one since we replace $pipe_addresses */
2133 argcount--;
2134 i--;
09792322
NK
2135 }
2136
2137 /* Handle special case of $address_pipe when af_force_command is set */
2138
2139 else if (addr != NULL && testflag(addr,af_force_command) &&
2140 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2141 Ustrcmp(argv[i], "${address_pipe}") == 0))
2142 {
2143 int address_pipe_i;
2144 int address_pipe_argcount = 0;
2145 int address_pipe_max_args;
2146 uschar **address_pipe_argv;
2147
2148 /* We can never have more then the argv we will be loading into */
2149 address_pipe_max_args = max_args - argcount + 1;
2150
2151 DEBUG(D_transport)
700d22f3 2152 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2153
2154 /* We allocate an additional for (uschar *)0 */
2155 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *));
2156
2157 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2158 s = expand_string(addr->local_part + 1);
2159
700d22f3
PP
2160 if (s == NULL || *s == '\0')
2161 {
2162 addr->transport_return = FAIL;
2163 addr->message = string_sprintf("Expansion of \"%s\" "
2164 "from command \"%s\" in %s failed: %s",
2165 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2166 return FALSE;
700d22f3 2167 }
09792322
NK
2168
2169 while (isspace(*s)) s++; /* strip leading space */
2170
2171 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2172 {
2173 if (*s == '\'')
2174 {
2175 ss = s + 1;
2176 while (*ss != 0 && *ss != '\'') ss++;
2177 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++);
2178 while (*s != 0 && *s != '\'') *ss++ = *s++;
2179 if (*s != 0) s++;
2180 *ss++ = 0;
2181 }
55414b25
JH
2182 else address_pipe_argv[address_pipe_argcount++] =
2183 string_copy(string_dequote(CUSS &s));
09792322
NK
2184 while (isspace(*s)) s++; /* strip space after arg */
2185 }
2186
5903c6ff 2187 address_pipe_argv[address_pipe_argcount] = US 0;
09792322
NK
2188
2189 /* If *s != 0 we have run out of argument slots. */
2190 if (*s != 0)
2191 {
2192 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2193 "\"%s\" in %s", addr->local_part + 1, etext);
2194 if (addr != NULL)
2195 {
2196 addr->transport_return = FAIL;
2197 addr->message = msg;
2198 }
2199 else *errptr = msg;
2200 return FALSE;
2201 }
2202
700d22f3
PP
2203 /* address_pipe_argcount - 1
2204 * because we are replacing $address_pipe in the argument list
2205 * with the first thing it expands to */
09792322
NK
2206 if (argcount + address_pipe_argcount - 1 > max_args)
2207 {
2208 addr->transport_return = FAIL;
2209 addr->message = string_sprintf("Too many arguments to command "
2210 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2211 return FALSE;
2212 }
2213
2214 /* If we are not just able to replace the slot that contained
2215 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2216 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2217 * Visually if address_pipe_argcount == 2:
2218 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2219 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2220 */
2221 if (address_pipe_argcount > 1)
2222 memmove(
4c04137d 2223 /* current position + additional args */
09792322
NK
2224 argv + i + address_pipe_argcount,
2225 /* current position + 1 (for the (uschar *)0 at the end) */
2226 argv + i + 1,
2227 /* -1 for the (uschar *)0 at the end)*/
2228 (argcount - i)*sizeof(uschar *)
2229 );
2230
2231 /* Now we fill in the slots we just moved argv out of
2232 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2233 */
700d22f3 2234 for (address_pipe_i = 0;
5903c6ff 2235 address_pipe_argv[address_pipe_i] != US 0;
700d22f3
PP
2236 address_pipe_i++)
2237 {
2238 argv[i++] = address_pipe_argv[address_pipe_i];
2239 argcount++;
09792322
NK
2240 }
2241
700d22f3
PP
2242 /* Subtract one since we replace $address_pipe */
2243 argcount--;
2244 i--;
059ec3d9
PH
2245 }
2246
2247 /* Handle normal expansion string */
2248
2249 else
2250 {
55414b25 2251 const uschar *expanded_arg;
8768d548 2252 f.enable_dollar_recipients = allow_dollar_recipients;
55414b25 2253 expanded_arg = expand_cstring(argv[i]);
8768d548 2254 f.enable_dollar_recipients = FALSE;
059ec3d9
PH
2255
2256 if (expanded_arg == NULL)
2257 {
2258 uschar *msg = string_sprintf("Expansion of \"%s\" "
2259 "from command \"%s\" in %s failed: %s",
2260 argv[i], cmd, etext, expand_string_message);
2261 if (addr != NULL)
2262 {
2263 addr->transport_return = expand_failed;
2264 addr->message = msg;
2265 }
2266 else *errptr = msg;
2267 return FALSE;
2268 }
2269 argv[i] = expanded_arg;
2270 }
2271 }
2272
2273 DEBUG(D_transport)
2274 {
2275 debug_printf("direct command after expansion:\n");
5903c6ff 2276 for (i = 0; argv[i] != US 0; i++)
059ec3d9
PH
2277 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2278 }
2279 }
2280
2281return TRUE;
2282}
2283
d185889f 2284#endif /*!MACRO_PREDEF*/
511a6c14
JH
2285/* vi: aw ai sw=2
2286*/
059ec3d9 2287/* End of transport.c */