Docs: add warning on non-ASCII results from SpamAssassin. Bug 1863
[exim.git] / src / src / deliver.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
80fea873 5/* Copyright (c) University of Cambridge 1995 - 2016 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* The main code for delivering a message. */
9
10
11#include "exim.h"
dadff1d4 12#include <assert.h>
059ec3d9
PH
13
14
15/* Data block for keeping track of subprocesses for parallel remote
16delivery. */
17
18typedef struct pardata {
19 address_item *addrlist; /* chain of addresses */
20 address_item *addr; /* next address data expected for */
21 pid_t pid; /* subprocess pid */
22 int fd; /* pipe fd for getting result from subprocess */
23 int transport_count; /* returned transport count value */
24 BOOL done; /* no more data needed */
25 uschar *msg; /* error message */
26 uschar *return_path; /* return_path for these addresses */
27} pardata;
28
29/* Values for the process_recipients variable */
30
31enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
32 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
33 RECIP_FAIL_LOOP};
34
35/* Mutually recursive functions for marking addresses done. */
36
37static void child_done(address_item *, uschar *);
38static void address_done(address_item *, uschar *);
39
40/* Table for turning base-62 numbers into binary */
41
42static uschar tab62[] =
43 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
44 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
45 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
46 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
47 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
48 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
49 59,60,61}; /* x-z */
50
51
52/*************************************************
53* Local static variables *
54*************************************************/
55
56/* addr_duplicate is global because it needs to be seen from the Envelope-To
57writing code. */
58
59static address_item *addr_defer = NULL;
60static address_item *addr_failed = NULL;
61static address_item *addr_fallback = NULL;
62static address_item *addr_local = NULL;
63static address_item *addr_new = NULL;
64static address_item *addr_remote = NULL;
65static address_item *addr_route = NULL;
66static address_item *addr_succeed = NULL;
6c1c3d1d
WB
67static address_item *addr_dsntmp = NULL;
68static address_item *addr_senddsn = NULL;
059ec3d9
PH
69
70static FILE *message_log = NULL;
71static BOOL update_spool;
72static BOOL remove_journal;
73static int parcount = 0;
74static pardata *parlist = NULL;
75static int return_count;
76static uschar *frozen_info = US"";
77static uschar *used_return_path = NULL;
78
059ec3d9
PH
79
80
81/*************************************************
82* Make a new address item *
83*************************************************/
84
85/* This function gets the store and initializes with default values. The
86transport_return value defaults to DEFER, so that any unexpected failure to
87deliver does not wipe out the message. The default unique string is set to a
88copy of the address, so that its domain can be lowercased.
89
90Argument:
91 address the RFC822 address string
92 copy force a copy of the address
93
94Returns: a pointer to an initialized address_item
95*/
96
97address_item *
98deliver_make_addr(uschar *address, BOOL copy)
99{
100address_item *addr = store_get(sizeof(address_item));
101*addr = address_defaults;
102if (copy) address = string_copy(address);
103addr->address = address;
104addr->unique = string_copy(address);
105return addr;
106}
107
108
109
110
111/*************************************************
112* Set expansion values for an address *
113*************************************************/
114
115/* Certain expansion variables are valid only when handling an address or
116address list. This function sets them up or clears the values, according to its
117argument.
118
119Arguments:
120 addr the address in question, or NULL to clear values
121Returns: nothing
122*/
123
124void
125deliver_set_expansions(address_item *addr)
126{
d6c82d7b 127if (!addr)
059ec3d9 128 {
55414b25 129 const uschar ***p = address_expansions;
d6c82d7b 130 while (*p) **p++ = NULL;
059ec3d9
PH
131 return;
132 }
133
134/* Exactly what gets set depends on whether there is one or more addresses, and
135what they contain. These first ones are always set, taking their values from
136the first address. */
137
d6c82d7b 138if (!addr->host_list)
059ec3d9
PH
139 {
140 deliver_host = deliver_host_address = US"";
a7538db1 141 deliver_host_port = 0;
059ec3d9
PH
142 }
143else
144 {
145 deliver_host = addr->host_list->name;
146 deliver_host_address = addr->host_list->address;
a7538db1 147 deliver_host_port = addr->host_list->port;
059ec3d9
PH
148 }
149
150deliver_recipients = addr;
d43cbe25
JH
151deliver_address_data = addr->prop.address_data;
152deliver_domain_data = addr->prop.domain_data;
153deliver_localpart_data = addr->prop.localpart_data;
059ec3d9
PH
154
155/* These may be unset for multiple addresses */
156
157deliver_domain = addr->domain;
158self_hostname = addr->self_hostname;
159
8523533c
TK
160#ifdef EXPERIMENTAL_BRIGHTMAIL
161bmi_deliver = 1; /* deliver by default */
162bmi_alt_location = NULL;
163bmi_base64_verdict = NULL;
164bmi_base64_tracker_verdict = NULL;
165#endif
166
059ec3d9
PH
167/* If there's only one address we can set everything. */
168
d6c82d7b 169if (!addr->next)
059ec3d9
PH
170 {
171 address_item *addr_orig;
172
173 deliver_localpart = addr->local_part;
174 deliver_localpart_prefix = addr->prefix;
175 deliver_localpart_suffix = addr->suffix;
176
d6c82d7b 177 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
059ec3d9
PH
178 deliver_domain_orig = addr_orig->domain;
179
180 /* Re-instate any prefix and suffix in the original local part. In all
181 normal cases, the address will have a router associated with it, and we can
182 choose the caseful or caseless version accordingly. However, when a system
183 filter sets up a pipe, file, or autoreply delivery, no router is involved.
184 In this case, though, there won't be any prefix or suffix to worry about. */
185
d6c82d7b
JH
186 deliver_localpart_orig = !addr_orig->router
187 ? addr_orig->local_part
188 : addr_orig->router->caseful_local_part
189 ? addr_orig->cc_local_part
190 : addr_orig->lc_local_part;
059ec3d9
PH
191
192 /* If there's a parent, make its domain and local part available, and if
193 delivering to a pipe or file, or sending an autoreply, get the local
194 part from the parent. For pipes and files, put the pipe or file string
195 into address_pipe and address_file. */
196
d6c82d7b 197 if (addr->parent)
059ec3d9
PH
198 {
199 deliver_domain_parent = addr->parent->domain;
d6c82d7b
JH
200 deliver_localpart_parent = !addr->parent->router
201 ? addr->parent->local_part
202 : addr->parent->router->caseful_local_part
203 ? addr->parent->cc_local_part
204 : addr->parent->lc_local_part;
059ec3d9
PH
205
206 /* File deliveries have their own flag because they need to be picked out
207 as special more often. */
208
209 if (testflag(addr, af_pfr))
210 {
d6c82d7b
JH
211 if (testflag(addr, af_file)) address_file = addr->local_part;
212 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
059ec3d9
PH
213 deliver_localpart = addr->parent->local_part;
214 deliver_localpart_prefix = addr->parent->prefix;
215 deliver_localpart_suffix = addr->parent->suffix;
216 }
217 }
8523533c
TK
218
219#ifdef EXPERIMENTAL_BRIGHTMAIL
220 /* Set expansion variables related to Brightmail AntiSpam */
221 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
222 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
223 /* get message delivery status (0 - don't deliver | 1 - deliver) */
224 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
225 /* if message is to be delivered, get eventual alternate location */
d6c82d7b 226 if (bmi_deliver == 1)
8523533c 227 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
8523533c
TK
228#endif
229
059ec3d9
PH
230 }
231
232/* For multiple addresses, don't set local part, and leave the domain and
f7fd3850
PH
233self_hostname set only if it is the same for all of them. It is possible to
234have multiple pipe and file addresses, but only when all addresses have routed
235to the same pipe or file. */
059ec3d9
PH
236
237else
238 {
239 address_item *addr2;
f7fd3850
PH
240 if (testflag(addr, af_pfr))
241 {
d6c82d7b
JH
242 if (testflag(addr, af_file)) address_file = addr->local_part;
243 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
f7fd3850 244 }
d6c82d7b 245 for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 246 {
d6c82d7b 247 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
059ec3d9 248 deliver_domain = NULL;
d6c82d7b
JH
249 if ( self_hostname
250 && ( !addr2->self_hostname
251 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
252 ) )
059ec3d9 253 self_hostname = NULL;
d6c82d7b 254 if (!deliver_domain && !self_hostname) break;
059ec3d9
PH
255 }
256 }
257}
258
259
260
261
262/*************************************************
263* Open a msglog file *
264*************************************************/
265
266/* This function is used both for normal message logs, and for files in the
267msglog directory that are used to catch output from pipes. Try to create the
268directory if it does not exist. From release 4.21, normal message logs should
269be created when the message is received.
270
271Argument:
272 filename the file name
273 mode the mode required
274 error used for saying what failed
275
276Returns: a file descriptor, or -1 (with errno set)
277*/
278
279static int
280open_msglog_file(uschar *filename, int mode, uschar **error)
281{
282int fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
283
284if (fd < 0 && errno == ENOENT)
285 {
41313d92
JH
286 (void)directory_make(spool_directory,
287 spool_sname(US"msglog", message_subdir),
288 MSGLOG_DIRECTORY_MODE, TRUE);
059ec3d9
PH
289 fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
290 }
291
292/* Set the close-on-exec flag and change the owner to the exim uid/gid (this
293function is called as root). Double check the mode, because the group setting
294doesn't always get set automatically. */
295
296if (fd >= 0)
297 {
ff790e47 298 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
059ec3d9
PH
299 if (fchown(fd, exim_uid, exim_gid) < 0)
300 {
301 *error = US"chown";
302 return -1;
303 }
304 if (fchmod(fd, mode) < 0)
305 {
306 *error = US"chmod";
307 return -1;
308 }
309 }
310else *error = US"create";
311
312return fd;
313}
314
315
316
317
318/*************************************************
319* Write to msglog if required *
320*************************************************/
321
322/* Write to the message log, if configured. This function may also be called
323from transports.
324
325Arguments:
326 format a string format
327
328Returns: nothing
329*/
330
331void
332deliver_msglog(const char *format, ...)
333{
334va_list ap;
335if (!message_logs) return;
336va_start(ap, format);
337vfprintf(message_log, format, ap);
338fflush(message_log);
339va_end(ap);
340}
341
342
343
344
345/*************************************************
346* Replicate status for batch *
347*************************************************/
348
349/* When a transport handles a batch of addresses, it may treat them
350individually, or it may just put the status in the first one, and return FALSE,
351requesting that the status be copied to all the others externally. This is the
352replication function. As well as the status, it copies the transport pointer,
353which may have changed if appendfile passed the addresses on to a different
354transport.
355
356Argument: pointer to the first address in a chain
357Returns: nothing
358*/
359
360static void
361replicate_status(address_item *addr)
362{
363address_item *addr2;
d6c82d7b 364for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 365 {
d6c82d7b 366 addr2->transport = addr->transport;
059ec3d9 367 addr2->transport_return = addr->transport_return;
d6c82d7b
JH
368 addr2->basic_errno = addr->basic_errno;
369 addr2->more_errno = addr->more_errno;
370 addr2->special_action = addr->special_action;
371 addr2->message = addr->message;
372 addr2->user_message = addr->user_message;
059ec3d9
PH
373 }
374}
375
376
377
378/*************************************************
379* Compare lists of hosts *
380*************************************************/
381
382/* This function is given two pointers to chains of host items, and it yields
383TRUE if the lists refer to the same hosts in the same order, except that
384
385(1) Multiple hosts with the same non-negative MX values are permitted to appear
386 in different orders. Round-robinning nameservers can cause this to happen.
387
388(2) Multiple hosts with the same negative MX values less than MX_NONE are also
389 permitted to appear in different orders. This is caused by randomizing
390 hosts lists.
391
392This enables Exim to use a single SMTP transaction for sending to two entirely
393different domains that happen to end up pointing at the same hosts.
394
395Arguments:
396 one points to the first host list
397 two points to the second host list
398
399Returns: TRUE if the lists refer to the same host set
400*/
401
402static BOOL
403same_hosts(host_item *one, host_item *two)
404{
d6c82d7b 405while (one && two)
059ec3d9
PH
406 {
407 if (Ustrcmp(one->name, two->name) != 0)
408 {
409 int mx = one->mx;
410 host_item *end_one = one;
411 host_item *end_two = two;
412
413 /* Batch up only if there was no MX and the list was not randomized */
414
415 if (mx == MX_NONE) return FALSE;
416
417 /* Find the ends of the shortest sequence of identical MX values */
418
d6c82d7b
JH
419 while ( end_one->next && end_one->next->mx == mx
420 && end_two->next && end_two->next->mx == mx)
059ec3d9
PH
421 {
422 end_one = end_one->next;
423 end_two = end_two->next;
424 }
425
426 /* If there aren't any duplicates, there's no match. */
427
428 if (end_one == one) return FALSE;
429
430 /* For each host in the 'one' sequence, check that it appears in the 'two'
431 sequence, returning FALSE if not. */
432
433 for (;;)
434 {
435 host_item *hi;
436 for (hi = two; hi != end_two->next; hi = hi->next)
437 if (Ustrcmp(one->name, hi->name) == 0) break;
438 if (hi == end_two->next) return FALSE;
439 if (one == end_one) break;
440 one = one->next;
441 }
442
443 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
444 Ensure both are pointing at the last host, and carry on as for equality. */
445
446 two = end_two;
447 }
448
449 /* Hosts matched */
450
451 one = one->next;
452 two = two->next;
453 }
454
455/* True if both are NULL */
456
457return (one == two);
458}
459
460
461
462/*************************************************
463* Compare header lines *
464*************************************************/
465
466/* This function is given two pointers to chains of header items, and it yields
467TRUE if they are the same header texts in the same order.
468
469Arguments:
470 one points to the first header list
471 two points to the second header list
472
473Returns: TRUE if the lists refer to the same header set
474*/
475
476static BOOL
477same_headers(header_line *one, header_line *two)
478{
d6c82d7b 479for (;; one = one->next, two = two->next)
059ec3d9
PH
480 {
481 if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 482 if (!one || !two) return FALSE;
059ec3d9 483 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
059ec3d9
PH
484 }
485}
486
487
488
489/*************************************************
490* Compare string settings *
491*************************************************/
492
493/* This function is given two pointers to strings, and it returns
494TRUE if they are the same pointer, or if the two strings are the same.
495
496Arguments:
497 one points to the first string
498 two points to the second string
499
500Returns: TRUE or FALSE
501*/
502
503static BOOL
504same_strings(uschar *one, uschar *two)
505{
506if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 507if (!one || !two) return FALSE;
059ec3d9
PH
508return (Ustrcmp(one, two) == 0);
509}
510
511
512
513/*************************************************
514* Compare uid/gid for addresses *
515*************************************************/
516
517/* This function is given a transport and two addresses. It yields TRUE if the
518uid/gid/initgroups settings for the two addresses are going to be the same when
519they are delivered.
520
521Arguments:
522 tp the transort
523 addr1 the first address
524 addr2 the second address
525
526Returns: TRUE or FALSE
527*/
528
529static BOOL
530same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
531{
d6c82d7b
JH
532if ( !tp->uid_set && !tp->expand_uid
533 && !tp->deliver_as_creator
534 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
535 || ( testflag(addr1, af_uid_set)
536 && ( addr1->uid != addr2->uid
537 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
538 ) ) ) )
539 return FALSE;
059ec3d9 540
d6c82d7b
JH
541if ( !tp->gid_set && !tp->expand_gid
542 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
543 || ( testflag(addr1, af_gid_set)
544 && addr1->gid != addr2->gid
545 ) ) )
546 return FALSE;
059ec3d9
PH
547
548return TRUE;
549}
550
551
552
553
554/*************************************************
555* Record that an address is complete *
556*************************************************/
557
558/* This function records that an address is complete. This is straightforward
559for most addresses, where the unique address is just the full address with the
560domain lower cased. For homonyms (addresses that are the same as one of their
561ancestors) their are complications. Their unique addresses have \x\ prepended
562(where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
563cousins.
564
565Exim used to record the unique addresses of homonyms as "complete". This,
566however, fails when the pattern of redirection varies over time (e.g. if taking
567unseen copies at only some times of day) because the prepended numbers may vary
568from one delivery run to the next. This problem is solved by never recording
569prepended unique addresses as complete. Instead, when a homonymic address has
570actually been delivered via a transport, we record its basic unique address
571followed by the name of the transport. This is checked in subsequent delivery
572runs whenever an address is routed to a transport.
573
574If the completed address is a top-level one (has no parent, which means it
575cannot be homonymic) we also add the original address to the non-recipients
576tree, so that it gets recorded in the spool file and therefore appears as
577"done" in any spool listings. The original address may differ from the unique
578address in the case of the domain.
579
580Finally, this function scans the list of duplicates, marks as done any that
581match this address, and calls child_done() for their ancestors.
582
583Arguments:
584 addr address item that has been completed
585 now current time as a string
586
587Returns: nothing
588*/
589
590static void
591address_done(address_item *addr, uschar *now)
592{
593address_item *dup;
594
595update_spool = TRUE; /* Ensure spool gets updated */
596
597/* Top-level address */
598
d6c82d7b 599if (!addr->parent)
059ec3d9
PH
600 {
601 tree_add_nonrecipient(addr->unique);
602 tree_add_nonrecipient(addr->address);
603 }
604
605/* Homonymous child address */
606
607else if (testflag(addr, af_homonym))
608 {
d6c82d7b 609 if (addr->transport)
059ec3d9
PH
610 tree_add_nonrecipient(
611 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
059ec3d9
PH
612 }
613
614/* Non-homonymous child address */
615
616else tree_add_nonrecipient(addr->unique);
617
618/* Check the list of duplicate addresses and ensure they are now marked
619done as well. */
620
d6c82d7b 621for (dup = addr_duplicate; dup; dup = dup->next)
059ec3d9
PH
622 if (Ustrcmp(addr->unique, dup->unique) == 0)
623 {
57730b52 624 tree_add_nonrecipient(dup->unique);
059ec3d9
PH
625 child_done(dup, now);
626 }
059ec3d9
PH
627}
628
629
630
631
632/*************************************************
633* Decrease counts in parents and mark done *
634*************************************************/
635
636/* This function is called when an address is complete. If there is a parent
637address, its count of children is decremented. If there are still other
638children outstanding, the function exits. Otherwise, if the count has become
639zero, address_done() is called to mark the parent and its duplicates complete.
640Then loop for any earlier ancestors.
641
642Arguments:
643 addr points to the completed address item
644 now the current time as a string, for writing to the message log
645
646Returns: nothing
647*/
648
649static void
650child_done(address_item *addr, uschar *now)
651{
652address_item *aa;
d6c82d7b 653while (addr->parent)
059ec3d9
PH
654 {
655 addr = addr->parent;
656 if ((addr->child_count -= 1) > 0) return; /* Incomplete parent */
657 address_done(addr, now);
658
659 /* Log the completion of all descendents only when there is no ancestor with
660 the same original address. */
661
d6c82d7b 662 for (aa = addr->parent; aa; aa = aa->parent)
059ec3d9 663 if (Ustrcmp(aa->address, addr->address) == 0) break;
d6c82d7b 664 if (aa) continue;
059ec3d9
PH
665
666 deliver_msglog("%s %s: children all complete\n", now, addr->address);
667 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
668 }
669}
670
671
672
ac881e27
TF
673/*************************************************
674* Delivery logging support functions *
675*************************************************/
676
677/* The LOGGING() checks in d_log_interface() are complicated for backwards
678compatibility. When outgoing interface logging was originally added, it was
679conditional on just incoming_interface (which is off by default). The
680outgoing_interface option is on by default to preserve this behaviour, but
681you can enable incoming_interface and disable outgoing_interface to get I=
682fields on incoming lines only.
683
684Arguments:
685 s The log line buffer
686 sizep Pointer to the buffer size
687 ptrp Pointer to current index into buffer
688 addr The address to be logged
689
690Returns: New value for s
691*/
059ec3d9 692
67d81c10 693static uschar *
ac881e27 694d_log_interface(uschar *s, int *sizep, int *ptrp)
67d81c10 695{
ac881e27 696if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
d6c82d7b 697 && sending_ip_address)
ac881e27
TF
698 {
699 s = string_append(s, sizep, ptrp, 2, US" I=[", sending_ip_address);
d6c82d7b
JH
700 s = LOGGING(outgoing_port)
701 ? string_append(s, sizep, ptrp, 2, US"]:",
702 string_sprintf("%d", sending_port))
c2f669a4 703 : string_catn(s, sizep, ptrp, US"]", 1);
ac881e27
TF
704 }
705return s;
67d81c10
JH
706}
707
ac881e27
TF
708
709
710static uschar *
fc16abb4 711d_hostlog(uschar * s, int * sp, int * pp, address_item * addr)
ac881e27 712{
fc16abb4
JH
713host_item * h = addr->host_used;
714
715s = string_append(s, sp, pp, 2, US" H=", h->name);
716
717if (LOGGING(dnssec) && h->dnssec == DS_YES)
718 s = string_cat(s, sp, pp, US" DS");
719
720s = string_append(s, sp, pp, 3, US" [", h->address, US"]");
721
ac881e27 722if (LOGGING(outgoing_port))
fc16abb4 723 s = string_append(s, sp, pp, 2, US":", string_sprintf("%d", h->port));
e6d2a989
JH
724
725#ifdef SUPPORT_SOCKS
726if (LOGGING(proxy) && proxy_local_address)
727 {
fc16abb4 728 s = string_append(s, sp, pp, 3, US" PRX=[", proxy_local_address, US"]");
e6d2a989 729 if (LOGGING(outgoing_port))
fc16abb4 730 s = string_append(s, sp, pp, 2, US":", string_sprintf("%d",
e6d2a989
JH
731 proxy_local_port));
732 }
733#endif
734
fc16abb4 735return d_log_interface(s, sp, pp);
ac881e27
TF
736}
737
738
739
e6d2a989
JH
740
741
67d81c10
JH
742#ifdef SUPPORT_TLS
743static uschar *
744d_tlslog(uschar * s, int * sizep, int * ptrp, address_item * addr)
745{
d6c82d7b 746if (LOGGING(tls_cipher) && addr->cipher)
ac881e27 747 s = string_append(s, sizep, ptrp, 2, US" X=", addr->cipher);
d6c82d7b 748if (LOGGING(tls_certificate_verified) && addr->cipher)
ac881e27
TF
749 s = string_append(s, sizep, ptrp, 2, US" CV=",
750 testflag(addr, af_cert_verified)
751 ?
53a7196b 752#ifdef EXPERIMENTAL_DANE
ac881e27
TF
753 testflag(addr, af_dane_verified)
754 ? "dane"
755 :
53a7196b 756#endif
ac881e27
TF
757 "yes"
758 : "no");
d6c82d7b 759if (LOGGING(tls_peerdn) && addr->peerdn)
ac881e27
TF
760 s = string_append(s, sizep, ptrp, 3, US" DN=\"",
761 string_printing(addr->peerdn), US"\"");
762return s;
67d81c10
JH
763}
764#endif
765
a7538db1 766
14a465c3
JH
767
768
0cbf2b82 769#ifndef DISABLE_EVENT
b30275b8 770uschar *
55414b25 771event_raise(uschar * action, const uschar * event, uschar * ev_data)
a7538db1
JH
772{
773uschar * s;
774if (action)
775 {
776 DEBUG(D_deliver)
774ef2d7 777 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
a7538db1
JH
778 event,
779 action, deliver_host_address);
780
774ef2d7
JH
781 event_name = event;
782 event_data = ev_data;
a7538db1
JH
783
784 if (!(s = expand_string(action)) && *expand_string_message)
785 log_write(0, LOG_MAIN|LOG_PANIC,
774ef2d7 786 "failed to expand event_action %s in %s: %s\n",
a7538db1
JH
787 event, transport_name, expand_string_message);
788
774ef2d7 789 event_name = event_data = NULL;
a7538db1
JH
790
791 /* If the expansion returns anything but an empty string, flag for
792 the caller to modify his normal processing
793 */
794 if (s && *s)
795 {
796 DEBUG(D_deliver)
774ef2d7 797 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
b30275b8 798 return s;
a7538db1
JH
799 }
800 }
b30275b8 801return NULL;
a7538db1 802}
14a465c3 803
5ef5dd52 804void
55414b25 805msg_event_raise(const uschar * event, const address_item * addr)
14a465c3 806{
55414b25 807const uschar * save_domain = deliver_domain;
14a465c3 808uschar * save_local = deliver_localpart;
55414b25 809const uschar * save_host = deliver_host;
5ef5dd52
JB
810const uschar * save_address = deliver_host_address;
811const int save_port = deliver_host_port;
14a465c3
JH
812
813if (!addr->transport)
814 return;
815
816router_name = addr->router ? addr->router->name : NULL;
817transport_name = addr->transport->name;
818deliver_domain = addr->domain;
819deliver_localpart = addr->local_part;
c562fd30 820deliver_host = addr->host_used ? addr->host_used->name : NULL;
14a465c3 821
774ef2d7 822(void) event_raise(addr->transport->event_action, event,
5ef5dd52
JB
823 addr->host_used
824 || Ustrcmp(addr->transport->driver_name, "smtp") == 0
825 || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
826 ? addr->message : NULL);
14a465c3 827
5ef5dd52
JB
828deliver_host_port = save_port;
829deliver_host_address = save_address;
c562fd30 830deliver_host = save_host;
14a465c3
JH
831deliver_localpart = save_local;
832deliver_domain = save_domain;
833router_name = transport_name = NULL;
834}
0cbf2b82 835#endif /*DISABLE_EVENT*/
14a465c3
JH
836
837
a7538db1 838
817d9f57 839/* If msg is NULL this is a delivery log and logchar is used. Otherwise
67d81c10 840this is a nonstandard call; no two-character delivery flag is written
817d9f57
JH
841but sender-host and sender are prefixed and "msg" is inserted in the log line.
842
843Arguments:
844 flags passed to log_write()
845*/
e4bdf652 846void
817d9f57 847delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
e4bdf652
JH
848{
849uschar *log_address;
850int size = 256; /* Used for a temporary, */
851int ptr = 0; /* expanding buffer, for */
852uschar *s; /* building log lines; */
853void *reset_point; /* released afterwards. */
854
e4bdf652
JH
855/* Log the delivery on the main log. We use an extensible string to build up
856the log line, and reset the store afterwards. Remote deliveries should always
857have a pointer to the host item that succeeded; local deliveries can have a
858pointer to a single host item in their host list, for use by the transport. */
859
0cbf2b82 860#ifndef DISABLE_EVENT
a7538db1 861 /* presume no successful remote delivery */
783b385f 862 lookup_dnssec_authenticated = NULL;
d68218c7
JH
863#endif
864
e4bdf652 865s = reset_point = store_get(size);
e4bdf652 866
6c6d6e48 867log_address = string_log_address(addr, LOGGING(all_parents), TRUE);
817d9f57
JH
868if (msg)
869 s = string_append(s, &size, &ptr, 3, host_and_ident(TRUE), US" ", log_address);
870else
871 {
872 s[ptr++] = logchar;
873 s = string_append(s, &size, &ptr, 2, US"> ", log_address);
874 }
e4bdf652 875
6c6d6e48 876if (LOGGING(sender_on_delivery) || msg)
3c8b3577 877 s = string_append(s, &size, &ptr, 3, US" F=<",
8c5d388a 878#ifdef SUPPORT_I18N
3c8b3577
JH
879 testflag(addr, af_utf8_downcvt)
880 ? string_address_utf8_to_alabel(sender_address, NULL)
881 :
882#endif
883 sender_address,
884 US">");
e4bdf652 885
afa6d3ad
JH
886if (*queue_name)
887 s = string_append(s, &size, &ptr, 2, US" Q=", queue_name);
888
e4bdf652 889#ifdef EXPERIMENTAL_SRS
d43cbe25
JH
890if(addr->prop.srs_sender)
891 s = string_append(s, &size, &ptr, 3, US" SRS=<", addr->prop.srs_sender, US">");
e4bdf652
JH
892#endif
893
894/* You might think that the return path must always be set for a successful
895delivery; indeed, I did for some time, until this statement crashed. The case
896when it is not set is for a delivery to /dev/null which is optimised by not
897being run at all. */
898
d6c82d7b 899if (used_return_path && LOGGING(return_path_on_delivery))
e4bdf652
JH
900 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
901
817d9f57
JH
902if (msg)
903 s = string_append(s, &size, &ptr, 2, US" ", msg);
e4bdf652 904
817d9f57 905/* For a delivery from a system filter, there may not be a router */
d6c82d7b 906if (addr->router)
e4bdf652
JH
907 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
908
909s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
910
6c6d6e48 911if (LOGGING(delivery_size))
e4bdf652
JH
912 s = string_append(s, &size, &ptr, 2, US" S=",
913 string_sprintf("%d", transport_count));
914
915/* Local delivery */
916
917if (addr->transport->info->local)
918 {
a7538db1 919 if (addr->host_list)
e4bdf652 920 s = string_append(s, &size, &ptr, 2, US" H=", addr->host_list->name);
ac881e27 921 s = d_log_interface(s, &size, &ptr);
d6c82d7b 922 if (addr->shadow_message)
c2f669a4 923 s = string_cat(s, &size, &ptr, addr->shadow_message);
e4bdf652
JH
924 }
925
926/* Remote delivery */
927
928else
929 {
783b385f 930 if (addr->host_used)
e4bdf652 931 {
67d81c10 932 s = d_hostlog(s, &size, &ptr, addr);
e4bdf652 933 if (continue_sequence > 1)
c2f669a4 934 s = string_catn(s, &size, &ptr, US"*", 1);
d68218c7 935
0cbf2b82 936#ifndef DISABLE_EVENT
a7538db1
JH
937 deliver_host_address = addr->host_used->address;
938 deliver_host_port = addr->host_used->port;
029f4192 939 deliver_host = addr->host_used->name;
783b385f
JH
940
941 /* DNS lookup status */
942 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
943 : addr->host_used->dnssec==DS_NO ? US"no"
944 : NULL;
a7538db1 945#endif
e4bdf652
JH
946 }
947
a7538db1 948#ifdef SUPPORT_TLS
67d81c10 949 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 950#endif
e4bdf652 951
018c60d7 952 if (addr->authenticator)
6f123593 953 {
018c60d7
JH
954 s = string_append(s, &size, &ptr, 2, US" A=", addr->authenticator);
955 if (addr->auth_id)
c8e2fc1e 956 {
018c60d7 957 s = string_append(s, &size, &ptr, 2, US":", addr->auth_id);
6c6d6e48 958 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
018c60d7 959 s = string_append(s, &size, &ptr, 2, US":", addr->auth_sndr);
c8e2fc1e 960 }
6f123593
JH
961 }
962
a7538db1 963#ifndef DISABLE_PRDR
fd98a5c6
JH
964 if (addr->flags & af_prdr_used)
965 s = string_append(s, &size, &ptr, 1, US" PRDR");
a7538db1 966#endif
76f44207
WB
967 }
968
969/* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
fd98a5c6 970
d6c82d7b
JH
971if ( LOGGING(smtp_confirmation)
972 && addr->message
973 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
974 )
76f44207 975 {
b980ed83 976 unsigned i;
1c5b1499 977 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
76f44207
WB
978 uschar *p = big_buffer;
979 uschar *ss = addr->message;
980 *p++ = '\"';
b980ed83 981 for (i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
e4bdf652 982 {
27520a68 983 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
76f44207 984 *p++ = ss[i];
e4bdf652 985 }
76f44207
WB
986 *p++ = '\"';
987 *p = 0;
988 s = string_append(s, &size, &ptr, 2, US" C=", big_buffer);
e4bdf652
JH
989 }
990
991/* Time on queue and actual time taken to deliver */
992
6c6d6e48 993if (LOGGING(queue_time))
e4bdf652 994 s = string_append(s, &size, &ptr, 2, US" QT=",
19050083 995 readconf_printtime( (int) ((long)time(NULL) - (long)received_time)) );
e4bdf652 996
6c6d6e48 997if (LOGGING(deliver_time))
e4bdf652
JH
998 s = string_append(s, &size, &ptr, 2, US" DT=",
999 readconf_printtime(addr->more_errno));
e4bdf652
JH
1000
1001/* string_cat() always leaves room for the terminator. Release the
1002store we used to build the line after writing it. */
1003
1004s[ptr] = 0;
817d9f57 1005log_write(0, flags, "%s", s);
d68218c7 1006
0cbf2b82 1007#ifndef DISABLE_EVENT
774ef2d7 1008if (!msg) msg_event_raise(US"msg:delivery", addr);
d68218c7 1009#endif
14a465c3 1010
e4bdf652
JH
1011store_reset(reset_point);
1012return;
1013}
1014
1015
1016
059ec3d9
PH
1017/*************************************************
1018* Actions at the end of handling an address *
1019*************************************************/
1020
1021/* This is a function for processing a single address when all that can be done
1022with it has been done.
1023
1024Arguments:
1025 addr points to the address block
1026 result the result of the delivery attempt
1027 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1028 driver_type indicates which type of driver (transport, or router) was last
1029 to process the address
1030 logchar '=' or '-' for use when logging deliveries with => or ->
1031
1032Returns: nothing
1033*/
1034
1035static void
1036post_process_one(address_item *addr, int result, int logflags, int driver_type,
1037 int logchar)
1038{
1039uschar *now = tod_stamp(tod_log);
1040uschar *driver_kind = NULL;
1041uschar *driver_name = NULL;
1042uschar *log_address;
1043
1044int size = 256; /* Used for a temporary, */
1045int ptr = 0; /* expanding buffer, for */
1046uschar *s; /* building log lines; */
1047void *reset_point; /* released afterwards. */
1048
059ec3d9
PH
1049DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1050
1051/* Set up driver kind and name for logging. Disable logging if the router or
1052transport has disabled it. */
1053
1054if (driver_type == DTYPE_TRANSPORT)
1055 {
d6c82d7b 1056 if (addr->transport)
059ec3d9
PH
1057 {
1058 driver_name = addr->transport->name;
1059 driver_kind = US" transport";
1060 disable_logging = addr->transport->disable_logging;
1061 }
1062 else driver_kind = US"transporting";
1063 }
1064else if (driver_type == DTYPE_ROUTER)
1065 {
d6c82d7b 1066 if (addr->router)
059ec3d9
PH
1067 {
1068 driver_name = addr->router->name;
1069 driver_kind = US" router";
1070 disable_logging = addr->router->disable_logging;
1071 }
1072 else driver_kind = US"routing";
1073 }
1074
1075/* If there's an error message set, ensure that it contains only printing
1076characters - it should, but occasionally things slip in and this at least
49c2d5ea
PH
1077stops the log format from getting wrecked. We also scan the message for an LDAP
1078expansion item that has a password setting, and flatten the password. This is a
1079fudge, but I don't know a cleaner way of doing this. (If the item is badly
1080malformed, it won't ever have gone near LDAP.) */
059ec3d9 1081
d6c82d7b 1082if (addr->message)
49c2d5ea 1083 {
55414b25 1084 const uschar * s = string_printing(addr->message);
f42deca9
JH
1085
1086 /* deconst cast ok as string_printing known to have alloc'n'copied */
1087 addr->message = expand_hide_passwords(US s);
49c2d5ea 1088 }
059ec3d9
PH
1089
1090/* If we used a transport that has one of the "return_output" options set, and
1091if it did in fact generate some output, then for return_output we treat the
1092message as failed if it was not already set that way, so that the output gets
1093returned to the sender, provided there is a sender to send it to. For
1094return_fail_output, do this only if the delivery failed. Otherwise we just
1095unlink the file, and remove the name so that if the delivery failed, we don't
1096try to send back an empty or unwanted file. The log_output options operate only
1097on a non-empty file.
1098
1099In any case, we close the message file, because we cannot afford to leave a
1100file-descriptor for one address while processing (maybe very many) others. */
1101
d6c82d7b 1102if (addr->return_file >= 0 && addr->return_filename)
059ec3d9
PH
1103 {
1104 BOOL return_output = FALSE;
1105 struct stat statbuf;
54fc8428 1106 (void)EXIMfsync(addr->return_file);
059ec3d9
PH
1107
1108 /* If there is no output, do nothing. */
1109
1110 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1111 {
1112 transport_instance *tb = addr->transport;
1113
1114 /* Handle logging options */
1115
d6c82d7b
JH
1116 if ( tb->log_output
1117 || result == FAIL && tb->log_fail_output
1118 || result == DEFER && tb->log_defer_output
1119 )
059ec3d9
PH
1120 {
1121 uschar *s;
1122 FILE *f = Ufopen(addr->return_filename, "rb");
d6c82d7b 1123 if (!f)
059ec3d9
PH
1124 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1125 "from %s transport: %s", addr->return_filename, tb->name,
1126 strerror(errno));
1127 else
d6c82d7b 1128 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
059ec3d9
PH
1129 {
1130 uschar *p = big_buffer + Ustrlen(big_buffer);
55414b25 1131 const uschar * sp;
059ec3d9
PH
1132 while (p > big_buffer && isspace(p[-1])) p--;
1133 *p = 0;
55414b25 1134 sp = string_printing(big_buffer);
059ec3d9 1135 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
55414b25 1136 addr->address, tb->name, sp);
059ec3d9 1137 }
f1e894f3 1138 (void)fclose(f);
059ec3d9
PH
1139 }
1140
1141 /* Handle returning options, but only if there is an address to return
1142 the text to. */
1143
d6c82d7b 1144 if (sender_address[0] != 0 || addr->prop.errors_address)
059ec3d9
PH
1145 if (tb->return_output)
1146 {
1147 addr->transport_return = result = FAIL;
d6c82d7b 1148 if (addr->basic_errno == 0 && !addr->message)
059ec3d9
PH
1149 addr->message = US"return message generated";
1150 return_output = TRUE;
1151 }
1152 else
1153 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
059ec3d9
PH
1154 }
1155
1156 /* Get rid of the file unless it might be returned, but close it in
1157 all cases. */
1158
1159 if (!return_output)
1160 {
1161 Uunlink(addr->return_filename);
1162 addr->return_filename = NULL;
1163 addr->return_file = -1;
1164 }
1165
f1e894f3 1166 (void)close(addr->return_file);
059ec3d9
PH
1167 }
1168
9d1c15ef 1169/* The success case happens only after delivery by a transport. */
059ec3d9
PH
1170
1171if (result == OK)
1172 {
1173 addr->next = addr_succeed;
1174 addr_succeed = addr;
1175
1176 /* Call address_done() to ensure that we don't deliver to this address again,
1177 and write appropriate things to the message log. If it is a child address, we
1178 call child_done() to scan the ancestors and mark them complete if this is the
1179 last child to complete. */
1180
1181 address_done(addr, now);
1182 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1183
d6c82d7b 1184 if (!addr->parent)
059ec3d9
PH
1185 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1186 driver_name, driver_kind);
059ec3d9
PH
1187 else
1188 {
1189 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1190 addr->parent->address, driver_name, driver_kind);
1191 child_done(addr, now);
1192 }
1193
774ef2d7 1194 /* Certificates for logging (via events) */
a7538db1 1195#ifdef SUPPORT_TLS
9d1c15ef
JH
1196 tls_out.ourcert = addr->ourcert;
1197 addr->ourcert = NULL;
1198 tls_out.peercert = addr->peercert;
1199 addr->peercert = NULL;
018058b2 1200
018058b2
JH
1201 tls_out.cipher = addr->cipher;
1202 tls_out.peerdn = addr->peerdn;
1203 tls_out.ocsp = addr->ocsp;
83b27293
JH
1204# ifdef EXPERIMENTAL_DANE
1205 tls_out.dane_verified = testflag(addr, af_dane_verified);
1206# endif
a7538db1 1207#endif
9d1c15ef 1208
817d9f57 1209 delivery_log(LOG_MAIN, addr, logchar, NULL);
9d1c15ef 1210
a7538db1 1211#ifdef SUPPORT_TLS
790fbb71
JH
1212 tls_free_cert(&tls_out.ourcert);
1213 tls_free_cert(&tls_out.peercert);
018058b2
JH
1214 tls_out.cipher = NULL;
1215 tls_out.peerdn = NULL;
1216 tls_out.ocsp = OCSP_NOT_REQ;
83b27293
JH
1217# ifdef EXPERIMENTAL_DANE
1218 tls_out.dane_verified = FALSE;
1219# endif
a7538db1 1220#endif
059ec3d9
PH
1221 }
1222
1223
1224/* Soft failure, or local delivery process failed; freezing may be
1225requested. */
1226
1227else if (result == DEFER || result == PANIC)
1228 {
1229 if (result == PANIC) logflags |= LOG_PANIC;
1230
1231 /* This puts them on the chain in reverse order. Do not change this, because
1232 the code for handling retries assumes that the one with the retry
1233 information is last. */
1234
1235 addr->next = addr_defer;
1236 addr_defer = addr;
1237
1238 /* The only currently implemented special action is to freeze the
1239 message. Logging of this is done later, just before the -H file is
1240 updated. */
1241
1242 if (addr->special_action == SPECIAL_FREEZE)
1243 {
1244 deliver_freeze = TRUE;
1245 deliver_frozen_at = time(NULL);
1246 update_spool = TRUE;
1247 }
1248
1249 /* If doing a 2-stage queue run, we skip writing to either the message
1250 log or the main log for SMTP defers. */
1251
1252 if (!queue_2stage || addr->basic_errno != 0)
1253 {
1254 uschar ss[32];
1255
1256 /* For errors of the type "retry time not reached" (also remotes skipped
1257 on queue run), logging is controlled by L_retry_defer. Note that this kind
1258 of error number is negative, and all the retry ones are less than any
1259 others. */
1260
d6c82d7b
JH
1261 unsigned int use_log_selector = addr->basic_errno <= ERRNO_RETRY_BASE
1262 ? L_retry_defer : 0;
059ec3d9
PH
1263
1264 /* Build up the line that is used for both the message log and the main
1265 log. */
1266
1267 s = reset_point = store_get(size);
e4bdf652
JH
1268
1269 /* Create the address string for logging. Must not do this earlier, because
1270 an OK result may be changed to FAIL when a pipe returns text. */
1271
6c6d6e48 1272 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1273
c2f669a4 1274 s = string_cat(s, &size, &ptr, log_address);
059ec3d9 1275
afa6d3ad
JH
1276 if (*queue_name)
1277 s = string_append(s, &size, &ptr, 2, US" Q=", queue_name);
1278
059ec3d9
PH
1279 /* Either driver_name contains something and driver_kind contains
1280 " router" or " transport" (note the leading space), or driver_name is
1281 a null string and driver_kind contains "routing" without the leading
1282 space, if all routing has been deferred. When a domain has been held,
1283 so nothing has been done at all, both variables contain null strings. */
1284
d6c82d7b 1285 if (driver_name)
059ec3d9 1286 {
d6c82d7b 1287 if (driver_kind[1] == 't' && addr->router)
059ec3d9
PH
1288 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
1289 Ustrcpy(ss, " ?=");
1290 ss[1] = toupper(driver_kind[1]);
1291 s = string_append(s, &size, &ptr, 2, ss, driver_name);
1292 }
d6c82d7b
JH
1293 else if (driver_kind)
1294 s = string_append(s, &size, &ptr, 2, US" ", driver_kind);
059ec3d9
PH
1295
1296 sprintf(CS ss, " defer (%d)", addr->basic_errno);
c2f669a4 1297 s = string_cat(s, &size, &ptr, ss);
059ec3d9
PH
1298
1299 if (addr->basic_errno > 0)
1300 s = string_append(s, &size, &ptr, 2, US": ",
1301 US strerror(addr->basic_errno));
1302
c562fd30 1303 if (addr->host_used)
1a47b633 1304 {
c562fd30
JH
1305 s = string_append(s, &size, &ptr, 5,
1306 US" H=", addr->host_used->name,
1307 US" [", addr->host_used->address, US"]");
1a47b633
JH
1308 if (LOGGING(outgoing_port))
1309 {
1310 int port = addr->host_used->port;
1311 s = string_append(s, &size, &ptr, 2,
1312 US":", port == PORT_NONE ? US"25" : string_sprintf("%d", port));
1313 }
1314 }
c562fd30 1315
d6c82d7b 1316 if (addr->message)
059ec3d9
PH
1317 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1318
1319 s[ptr] = 0;
1320
1321 /* Log the deferment in the message log, but don't clutter it
1322 up with retry-time defers after the first delivery attempt. */
1323
1324 if (deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1325 deliver_msglog("%s %s\n", now, s);
1326
1327 /* Write the main log and reset the store */
1328
1329 log_write(use_log_selector, logflags, "== %s", s);
1330 store_reset(reset_point);
1331 }
1332 }
1333
1334
1335/* Hard failure. If there is an address to which an error message can be sent,
1336put this address on the failed list. If not, put it on the deferred list and
1337freeze the mail message for human attention. The latter action can also be
1338explicitly requested by a router or transport. */
1339
1340else
1341 {
1342 /* If this is a delivery error, or a message for which no replies are
1343 wanted, and the message's age is greater than ignore_bounce_errors_after,
1344 force the af_ignore_error flag. This will cause the address to be discarded
1345 later (with a log entry). */
1346
1347 if (sender_address[0] == 0 && message_age >= ignore_bounce_errors_after)
1348 setflag(addr, af_ignore_error);
1349
1350 /* Freeze the message if requested, or if this is a bounce message (or other
652e1b65
PH
1351 message with null sender) and this address does not have its own errors
1352 address. However, don't freeze if errors are being ignored. The actual code
1353 to ignore occurs later, instead of sending a message. Logging of freezing
1354 occurs later, just before writing the -H file. */
059ec3d9 1355
d6c82d7b
JH
1356 if ( !testflag(addr, af_ignore_error)
1357 && ( addr->special_action == SPECIAL_FREEZE
1358 || (sender_address[0] == 0 && !addr->prop.errors_address)
1359 ) )
059ec3d9 1360 {
d6c82d7b
JH
1361 frozen_info = addr->special_action == SPECIAL_FREEZE
1362 ? US""
1363 : sender_local && !local_error_message
1364 ? US" (message created with -f <>)"
1365 : US" (delivery error message)";
059ec3d9
PH
1366 deliver_freeze = TRUE;
1367 deliver_frozen_at = time(NULL);
1368 update_spool = TRUE;
1369
1370 /* The address is put on the defer rather than the failed queue, because
1371 the message is being retained. */
1372
1373 addr->next = addr_defer;
1374 addr_defer = addr;
1375 }
1376
1377 /* Don't put the address on the nonrecipients tree yet; wait until an
1378 error message has been successfully sent. */
1379
1380 else
1381 {
1382 addr->next = addr_failed;
1383 addr_failed = addr;
1384 }
1385
1386 /* Build up the log line for the message and main logs */
1387
1388 s = reset_point = store_get(size);
e4bdf652
JH
1389
1390 /* Create the address string for logging. Must not do this earlier, because
1391 an OK result may be changed to FAIL when a pipe returns text. */
1392
6c6d6e48 1393 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1394
c2f669a4 1395 s = string_cat(s, &size, &ptr, log_address);
059ec3d9 1396
6c6d6e48 1397 if (LOGGING(sender_on_delivery))
059ec3d9
PH
1398 s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
1399
afa6d3ad
JH
1400 if (*queue_name)
1401 s = string_append(s, &size, &ptr, 2, US" Q=", queue_name);
1402
059ec3d9
PH
1403 /* Return path may not be set if no delivery actually happened */
1404
d6c82d7b 1405 if (used_return_path && LOGGING(return_path_on_delivery))
059ec3d9 1406 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
059ec3d9 1407
d6c82d7b 1408 if (addr->router)
059ec3d9 1409 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
d6c82d7b 1410 if (addr->transport)
059ec3d9
PH
1411 s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
1412
d6c82d7b 1413 if (addr->host_used)
67d81c10
JH
1414 s = d_hostlog(s, &size, &ptr, addr);
1415
a7538db1 1416#ifdef SUPPORT_TLS
67d81c10 1417 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 1418#endif
059ec3d9
PH
1419
1420 if (addr->basic_errno > 0)
1421 s = string_append(s, &size, &ptr, 2, US": ",
1422 US strerror(addr->basic_errno));
1423
d6c82d7b 1424 if (addr->message)
059ec3d9
PH
1425 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1426
1427 s[ptr] = 0;
1428
1429 /* Do the logging. For the message log, "routing failed" for those cases,
1430 just to make it clearer. */
1431
d6c82d7b 1432 if (driver_name)
059ec3d9 1433 deliver_msglog("%s %s\n", now, s);
d6c82d7b
JH
1434 else
1435 deliver_msglog("%s %s failed for %s\n", now, driver_kind, s);
059ec3d9
PH
1436
1437 log_write(0, LOG_MAIN, "** %s", s);
14a465c3 1438
0cbf2b82 1439#ifndef DISABLE_EVENT
774ef2d7 1440 msg_event_raise(US"msg:fail:delivery", addr);
14a465c3
JH
1441#endif
1442
059ec3d9
PH
1443 store_reset(reset_point);
1444 }
1445
1446/* Ensure logging is turned on again in all cases */
1447
1448disable_logging = FALSE;
1449}
1450
1451
1452
1453
1454/*************************************************
1455* Address-independent error *
1456*************************************************/
1457
1458/* This function is called when there's an error that is not dependent on a
1459particular address, such as an expansion string failure. It puts the error into
1460all the addresses in a batch, logs the incident on the main and panic logs, and
1461clears the expansions. It is mostly called from local_deliver(), but can be
1462called for a remote delivery via findugid().
1463
1464Arguments:
1465 logit TRUE if (MAIN+PANIC) logging required
1466 addr the first of the chain of addresses
1467 code the error code
1468 format format string for error message, or NULL if already set in addr
1469 ... arguments for the format
1470
1471Returns: nothing
1472*/
1473
1474static void
1475common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1476{
1477address_item *addr2;
1478addr->basic_errno = code;
1479
d6c82d7b 1480if (format)
059ec3d9
PH
1481 {
1482 va_list ap;
1483 uschar buffer[512];
1484 va_start(ap, format);
1485 if (!string_vformat(buffer, sizeof(buffer), CS format, ap))
1486 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
81f91683 1487 "common_error expansion was longer than " SIZE_T_FMT, sizeof(buffer));
059ec3d9
PH
1488 va_end(ap);
1489 addr->message = string_copy(buffer);
1490 }
1491
d6c82d7b 1492for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9
PH
1493 {
1494 addr2->basic_errno = code;
1495 addr2->message = addr->message;
1496 }
1497
1498if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1499deliver_set_expansions(NULL);
1500}
1501
1502
1503
1504
1505/*************************************************
1506* Check a "never users" list *
1507*************************************************/
1508
1509/* This function is called to check whether a uid is on one of the two "never
1510users" lists.
1511
1512Arguments:
1513 uid the uid to be checked
1514 nusers the list to be scanned; the first item in the list is the count
1515
1516Returns: TRUE if the uid is on the list
1517*/
1518
1519static BOOL
1520check_never_users(uid_t uid, uid_t *nusers)
1521{
1522int i;
d6c82d7b 1523if (!nusers) return FALSE;
059ec3d9
PH
1524for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1525return FALSE;
1526}
1527
1528
1529
1530/*************************************************
1531* Find uid and gid for a transport *
1532*************************************************/
1533
1534/* This function is called for both local and remote deliveries, to find the
1535uid/gid under which to run the delivery. The values are taken preferentially
1536from the transport (either explicit or deliver_as_creator), then from the
1537address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1538the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1539panic error is logged, and the function fails (which normally leads to delivery
1540deferral).
1541
1542Arguments:
1543 addr the address (possibly a chain)
1544 tp the transport
1545 uidp pointer to uid field
1546 gidp pointer to gid field
1547 igfp pointer to the use_initgroups field
1548
1549Returns: FALSE if failed - error has been set in address(es)
1550*/
1551
1552static BOOL
1553findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1554 BOOL *igfp)
1555{
d6c82d7b 1556uschar *nuname;
059ec3d9
PH
1557BOOL gid_set = FALSE;
1558
1559/* Default initgroups flag comes from the transport */
1560
1561*igfp = tp->initgroups;
1562
1563/* First see if there's a gid on the transport, either fixed or expandable.
1564The expanding function always logs failure itself. */
1565
1566if (tp->gid_set)
1567 {
1568 *gidp = tp->gid;
1569 gid_set = TRUE;
1570 }
d6c82d7b 1571else if (tp->expand_gid)
059ec3d9 1572 {
d6c82d7b
JH
1573 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1574 &(addr->message)))
059ec3d9
PH
1575 {
1576 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1577 return FALSE;
1578 }
d6c82d7b 1579 gid_set = TRUE;
059ec3d9
PH
1580 }
1581
911f6fde
PH
1582/* If the transport did not set a group, see if the router did. */
1583
1584if (!gid_set && testflag(addr, af_gid_set))
1585 {
1586 *gidp = addr->gid;
1587 gid_set = TRUE;
1588 }
1589
059ec3d9
PH
1590/* Pick up a uid from the transport if one is set. */
1591
1592if (tp->uid_set) *uidp = tp->uid;
1593
1594/* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1595it does not provide a passwd value from which a gid can be taken. */
1596
d6c82d7b 1597else if (tp->expand_uid)
059ec3d9
PH
1598 {
1599 struct passwd *pw;
1600 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1601 uidp, &(addr->message)))
1602 {
1603 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1604 return FALSE;
1605 }
d6c82d7b 1606 if (!gid_set && pw)
059ec3d9
PH
1607 {
1608 *gidp = pw->pw_gid;
1609 gid_set = TRUE;
1610 }
1611 }
1612
1613/* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1614
1615else if (tp->deliver_as_creator)
1616 {
1617 *uidp = originator_uid;
1618 if (!gid_set)
1619 {
1620 *gidp = originator_gid;
1621 gid_set = TRUE;
1622 }
1623 }
1624
911f6fde
PH
1625/* Otherwise see if the address specifies the uid and if so, take it and its
1626initgroups flag. */
059ec3d9
PH
1627
1628else if (testflag(addr, af_uid_set))
1629 {
1630 *uidp = addr->uid;
1631 *igfp = testflag(addr, af_initgroups);
059ec3d9
PH
1632 }
1633
1634/* Nothing has specified the uid - default to the Exim user, and group if the
1635gid is not set. */
1636
1637else
1638 {
1639 *uidp = exim_uid;
1640 if (!gid_set)
1641 {
1642 *gidp = exim_gid;
1643 gid_set = TRUE;
1644 }
1645 }
1646
911f6fde
PH
1647/* If no gid is set, it is a disaster. We default to the Exim gid only if
1648defaulting to the Exim uid. In other words, if the configuration has specified
1649a uid, it must also provide a gid. */
059ec3d9
PH
1650
1651if (!gid_set)
1652 {
1653 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1654 "%s transport", tp->name);
1655 return FALSE;
1656 }
1657
1658/* Check that the uid is not on the lists of banned uids that may not be used
1659for delivery processes. */
1660
d6c82d7b
JH
1661nuname = check_never_users(*uidp, never_users)
1662 ? US"never_users"
1663 : check_never_users(*uidp, fixed_never_users)
1664 ? US"fixed_never_users"
1665 : NULL;
1666if (nuname)
059ec3d9
PH
1667 {
1668 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1669 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1670 return FALSE;
1671 }
1672
1673/* All is well */
1674
1675return TRUE;
1676}
1677
1678
1679
1680
1681/*************************************************
1682* Check the size of a message for a transport *
1683*************************************************/
1684
1685/* Checks that the message isn't too big for the selected transport.
1686This is called only when it is known that the limit is set.
1687
1688Arguments:
1689 tp the transport
1690 addr the (first) address being delivered
1691
1692Returns: OK
1693 DEFER expansion failed or did not yield an integer
1694 FAIL message too big
1695*/
1696
1697int
1698check_message_size(transport_instance *tp, address_item *addr)
1699{
1700int rc = OK;
1701int size_limit;
1702
1703deliver_set_expansions(addr);
d45b1de8 1704size_limit = expand_string_integer(tp->message_size_limit, TRUE);
059ec3d9
PH
1705deliver_set_expansions(NULL);
1706
d6c82d7b 1707if (expand_string_message)
059ec3d9
PH
1708 {
1709 rc = DEFER;
d6c82d7b
JH
1710 addr->message = size_limit == -1
1711 ? string_sprintf("failed to expand message_size_limit "
1712 "in %s transport: %s", tp->name, expand_string_message)
1713 : string_sprintf("invalid message_size_limit "
059ec3d9
PH
1714 "in %s transport: %s", tp->name, expand_string_message);
1715 }
1716else if (size_limit > 0 && message_size > size_limit)
1717 {
1718 rc = FAIL;
1719 addr->message =
1720 string_sprintf("message is too big (transport limit = %d)",
1721 size_limit);
1722 }
1723
1724return rc;
1725}
1726
1727
1728
1729/*************************************************
1730* Transport-time check for a previous delivery *
1731*************************************************/
1732
1733/* Check that this base address hasn't previously been delivered to its routed
c2c19e9d
PH
1734transport. If it has been delivered, mark it done. The check is necessary at
1735delivery time in order to handle homonymic addresses correctly in cases where
1736the pattern of redirection changes between delivery attempts (so the unique
1737fields change). Non-homonymic previous delivery is detected earlier, at routing
1738time (which saves unnecessary routing).
1739
1740Arguments:
1741 addr the address item
1742 testing TRUE if testing wanted only, without side effects
059ec3d9 1743
059ec3d9
PH
1744Returns: TRUE if previously delivered by the transport
1745*/
1746
1747static BOOL
c2c19e9d 1748previously_transported(address_item *addr, BOOL testing)
059ec3d9
PH
1749{
1750(void)string_format(big_buffer, big_buffer_size, "%s/%s",
1751 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
1752
1753if (tree_search(tree_nonrecipients, big_buffer) != 0)
1754 {
1755 DEBUG(D_deliver|D_route|D_transport)
1756 debug_printf("%s was previously delivered (%s transport): discarded\n",
1757 addr->address, addr->transport->name);
c2c19e9d 1758 if (!testing) child_done(addr, tod_stamp(tod_log));
059ec3d9
PH
1759 return TRUE;
1760 }
1761
1762return FALSE;
1763}
1764
1765
1766
064a94c9
PH
1767/******************************************************
1768* Check for a given header in a header string *
1769******************************************************/
1770
1771/* This function is used when generating quota warnings. The configuration may
1772specify any header lines it likes in quota_warn_message. If certain of them are
1773missing, defaults are inserted, so we need to be able to test for the presence
1774of a given header.
1775
1776Arguments:
1777 hdr the required header name
1778 hstring the header string
1779
1780Returns: TRUE the header is in the string
1781 FALSE the header is not in the string
1782*/
1783
1784static BOOL
1785contains_header(uschar *hdr, uschar *hstring)
1786{
1787int len = Ustrlen(hdr);
1788uschar *p = hstring;
1789while (*p != 0)
1790 {
1791 if (strncmpic(p, hdr, len) == 0)
1792 {
1793 p += len;
1794 while (*p == ' ' || *p == '\t') p++;
1795 if (*p == ':') return TRUE;
1796 }
1797 while (*p != 0 && *p != '\n') p++;
1798 if (*p == '\n') p++;
1799 }
1800return FALSE;
1801}
1802
1803
1804
059ec3d9
PH
1805
1806/*************************************************
1807* Perform a local delivery *
1808*************************************************/
1809
1810/* Each local delivery is performed in a separate process which sets its
1811uid and gid as specified. This is a safer way than simply changing and
1812restoring using seteuid(); there is a body of opinion that seteuid() cannot be
1813used safely. From release 4, Exim no longer makes any use of it. Besides, not
1814all systems have seteuid().
1815
1816If the uid/gid are specified in the transport_instance, they are used; the
1817transport initialization must ensure that either both or neither are set.
1818Otherwise, the values associated with the address are used. If neither are set,
1819it is a configuration error.
1820
1821The transport or the address may specify a home directory (transport over-
1822rides), and if they do, this is set as $home. If neither have set a working
1823directory, this value is used for that as well. Otherwise $home is left unset
1824and the cwd is set to "/" - a directory that should be accessible to all users.
1825
1826Using a separate process makes it more complicated to get error information
1827back. We use a pipe to pass the return code and also an error code and error
1828text string back to the parent process.
1829
1830Arguments:
1831 addr points to an address block for this delivery; for "normal" local
1832 deliveries this is the only address to be delivered, but for
1833 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
1834 a number of addresses can be handled simultaneously, and in this
1835 case addr will point to a chain of addresses with the same
1836 characteristics.
1837
1838 shadowing TRUE if running a shadow transport; this causes output from pipes
1839 to be ignored.
1840
1841Returns: nothing
1842*/
1843
1844static void
1845deliver_local(address_item *addr, BOOL shadowing)
1846{
1847BOOL use_initgroups;
1848uid_t uid;
1849gid_t gid;
1850int status, len, rc;
1851int pfd[2];
1852pid_t pid;
1853uschar *working_directory;
1854address_item *addr2;
1855transport_instance *tp = addr->transport;
1856
1857/* Set up the return path from the errors or sender address. If the transport
1858has its own return path setting, expand it and replace the existing value. */
1859
d6c82d7b 1860if(addr->prop.errors_address)
d43cbe25 1861 return_path = addr->prop.errors_address;
384152a6 1862#ifdef EXPERIMENTAL_SRS
d6c82d7b 1863else if (addr->prop.srs_sender)
d43cbe25 1864 return_path = addr->prop.srs_sender;
384152a6
TK
1865#endif
1866else
1867 return_path = sender_address;
059ec3d9 1868
d6c82d7b 1869if (tp->return_path)
059ec3d9
PH
1870 {
1871 uschar *new_return_path = expand_string(tp->return_path);
d6c82d7b 1872 if (!new_return_path)
059ec3d9
PH
1873 {
1874 if (!expand_string_forcedfail)
1875 {
1876 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
1877 US"Failed to expand return path \"%s\" in %s transport: %s",
1878 tp->return_path, tp->name, expand_string_message);
1879 return;
1880 }
1881 }
1882 else return_path = new_return_path;
1883 }
1884
1885/* For local deliveries, one at a time, the value used for logging can just be
1886set directly, once and for all. */
1887
1888used_return_path = return_path;
1889
1890/* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
1891gets put into the address(es), and the expansions are unset, so we can just
1892return. */
1893
1894if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
1895
5418e93b
PH
1896/* See if either the transport or the address specifies a home directory. A
1897home directory set in the address may already be expanded; a flag is set to
1898indicate that. In other cases we must expand it. */
059ec3d9 1899
d6c82d7b
JH
1900if ( (deliver_home = tp->home_dir) /* Set in transport, or */
1901 || ( (deliver_home = addr->home_dir) /* Set in address and */
1902 && !testflag(addr, af_home_expanded) /* not expanded */
1903 ) )
059ec3d9
PH
1904 {
1905 uschar *rawhome = deliver_home;
1906 deliver_home = NULL; /* in case it contains $home */
d6c82d7b 1907 if (!(deliver_home = expand_string(rawhome)))
059ec3d9
PH
1908 {
1909 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
1910 "to expand for %s transport: %s", rawhome, tp->name,
1911 expand_string_message);
1912 return;
1913 }
1914 if (*deliver_home != '/')
1915 {
1916 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
1917 "is not absolute for %s transport", deliver_home, tp->name);
1918 return;
1919 }
1920 }
1921
5418e93b
PH
1922/* See if either the transport or the address specifies a current directory,
1923and if so, expand it. If nothing is set, use the home directory, unless it is
1924also unset in which case use "/", which is assumed to be a directory to which
1925all users have access. It is necessary to be in a visible directory for some
1926operating systems when running pipes, as some commands (e.g. "rm" under Solaris
19272.5) require this. */
1928
d6c82d7b
JH
1929working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
1930if (working_directory)
059ec3d9
PH
1931 {
1932 uschar *raw = working_directory;
d6c82d7b 1933 if (!(working_directory = expand_string(raw)))
059ec3d9
PH
1934 {
1935 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
1936 "failed to expand for %s transport: %s", raw, tp->name,
1937 expand_string_message);
1938 return;
1939 }
1940 if (*working_directory != '/')
1941 {
1942 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
1943 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
1944 return;
1945 }
1946 }
d6c82d7b 1947else working_directory = deliver_home ? deliver_home : US"/";
059ec3d9
PH
1948
1949/* If one of the return_output flags is set on the transport, create and open a
1950file in the message log directory for the transport to write its output onto.
1951This is mainly used by pipe transports. The file needs to be unique to the
1952address. This feature is not available for shadow transports. */
1953
d6c82d7b
JH
1954if ( !shadowing
1955 && ( tp->return_output || tp->return_fail_output
ec0eb1a3 1956 || tp->log_output || tp->log_fail_output || tp->log_defer_output
d6c82d7b 1957 ) )
059ec3d9 1958 {
41313d92
JH
1959 uschar * error;
1960
059ec3d9 1961 addr->return_filename =
41313d92
JH
1962 spool_fname(US"msglog", message_subdir, message_id,
1963 string_sprintf("-%d-%d", getpid(), return_count++));
1964
1965 if ((addr->return_file = open_msglog_file(addr->return_filename, 0400, &error)) < 0)
059ec3d9
PH
1966 {
1967 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
1968 "to return message: %s", error, tp->name, strerror(errno));
1969 return;
1970 }
1971 }
1972
1973/* Create the pipe for inter-process communication. */
1974
1975if (pipe(pfd) != 0)
1976 {
1977 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
1978 strerror(errno));
1979 return;
1980 }
1981
1982/* Now fork the process to do the real work in the subprocess, but first
1983ensure that all cached resources are freed so that the subprocess starts with
1984a clean slate and doesn't interfere with the parent process. */
1985
1986search_tidyup();
1987
1988if ((pid = fork()) == 0)
1989 {
1990 BOOL replicate = TRUE;
1991
1992 /* Prevent core dumps, as we don't want them in users' home directories.
1993 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
1994 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
1995 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
a29e5231
PP
1996 complain if the error is "not supported".
1997
1998 There are two scenarios where changing the max limit has an effect. In one,
1999 the user is using a .forward and invoking a command of their choice via pipe;
2000 for these, we do need the max limit to be 0 unless the admin chooses to
2001 permit an increased limit. In the other, the command is invoked directly by
2002 the transport and is under administrator control, thus being able to raise
2003 the limit aids in debugging. So there's no general always-right answer.
2004
2005 Thus we inhibit core-dumps completely but let individual transports, while
2006 still root, re-raise the limits back up to aid debugging. We make the
2007 default be no core-dumps -- few enough people can use core dumps in
2008 diagnosis that it's reasonable to make them something that has to be explicitly requested.
2009 */
059ec3d9 2010
a7538db1 2011#ifdef RLIMIT_CORE
059ec3d9
PH
2012 struct rlimit rl;
2013 rl.rlim_cur = 0;
2014 rl.rlim_max = 0;
2015 if (setrlimit(RLIMIT_CORE, &rl) < 0)
2016 {
a7538db1 2017# ifdef SETRLIMIT_NOT_SUPPORTED
059ec3d9 2018 if (errno != ENOSYS && errno != ENOTSUP)
a7538db1 2019# endif
059ec3d9
PH
2020 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
2021 strerror(errno));
2022 }
a7538db1 2023#endif
059ec3d9
PH
2024
2025 /* Reset the random number generator, so different processes don't all
2026 have the same sequence. */
2027
2028 random_seed = 0;
2029
2030 /* If the transport has a setup entry, call this first, while still
2031 privileged. (Appendfile uses this to expand quota, for example, while
2032 able to read private files.) */
2033
d6c82d7b 2034 if (addr->transport->setup)
929ba01c 2035 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
059ec3d9
PH
2036 &(addr->message)))
2037 {
2038 case DEFER:
d6c82d7b
JH
2039 addr->transport_return = DEFER;
2040 goto PASS_BACK;
059ec3d9
PH
2041
2042 case FAIL:
d6c82d7b
JH
2043 addr->transport_return = PANIC;
2044 goto PASS_BACK;
059ec3d9 2045 }
059ec3d9
PH
2046
2047 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2048 when the process becomes unprivileged, it won't be able to write to the
2049 process log. SIGHUP is ignored throughout exim, except when it is being
2050 run as a daemon. */
2051
2052 signal(SIGINT, SIG_IGN);
2053 signal(SIGTERM, SIG_IGN);
2054 signal(SIGUSR1, SIG_IGN);
2055
2056 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2057 half - for transports that exec things (e.g. pipe). Then set the required
2058 gid/uid. */
2059
f1e894f3 2060 (void)close(pfd[pipe_read]);
ff790e47 2061 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
059ec3d9
PH
2062 FD_CLOEXEC);
2063 exim_setugid(uid, gid, use_initgroups,
2064 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2065 addr->address, addr->transport->name));
2066
2067 DEBUG(D_deliver)
2068 {
2069 address_item *batched;
2070 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
d6c82d7b 2071 for (batched = addr->next; batched; batched = batched->next)
059ec3d9
PH
2072 debug_printf("additional batched address: %s\n", batched->address);
2073 }
2074
2075 /* Set an appropriate working directory. */
2076
2077 if (Uchdir(working_directory) < 0)
2078 {
2079 addr->transport_return = DEFER;
2080 addr->basic_errno = errno;
2081 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2082 }
2083
2084 /* If successful, call the transport */
2085
2086 else
2087 {
2088 BOOL ok = TRUE;
2089 set_process_info("delivering %s to %s using %s", message_id,
2090 addr->local_part, addr->transport->name);
2091
2a47f028
JH
2092 /* Setting this global in the subprocess means we need never clear it */
2093 transport_name = addr->transport->name;
2094
059ec3d9
PH
2095 /* If a transport filter has been specified, set up its argument list.
2096 Any errors will get put into the address, and FALSE yielded. */
2097
d6c82d7b 2098 if (addr->transport->filter_command)
059ec3d9
PH
2099 {
2100 ok = transport_set_up_command(&transport_filter_argv,
2101 addr->transport->filter_command,
2102 TRUE, PANIC, addr, US"transport filter", NULL);
2103 transport_filter_timeout = addr->transport->filter_timeout;
2104 }
2105 else transport_filter_argv = NULL;
2106
2107 if (ok)
2108 {
2109 debug_print_string(addr->transport->debug_string);
2110 replicate = !(addr->transport->info->code)(addr->transport, addr);
2111 }
2112 }
2113
2114 /* Pass the results back down the pipe. If necessary, first replicate the
2115 status in the top address to the others in the batch. The label is the
2116 subject of a goto when a call to the transport's setup function fails. We
2117 pass the pointer to the transport back in case it got changed as a result of
2118 file_format in appendfile. */
2119
2120 PASS_BACK:
2121
2122 if (replicate) replicate_status(addr);
d6c82d7b 2123 for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9
PH
2124 {
2125 int i;
2126 int local_part_length = Ustrlen(addr2->local_part);
2127 uschar *s;
1ac6b2e7 2128 int ret;
059ec3d9 2129
d6c82d7b
JH
2130 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2131 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2132 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2133 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2134 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
2135 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2136 || (ret = write(pfd[pipe_write], &addr2->transport,
1ac6b2e7 2137 sizeof(transport_instance *))) != sizeof(transport_instance *)
059ec3d9
PH
2138
2139 /* For a file delivery, pass back the local part, in case the original
2140 was only part of the final delivery path. This gives more complete
2141 logging. */
2142
1ac6b2e7 2143 || (testflag(addr2, af_file)
d6c82d7b 2144 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
1ac6b2e7
JH
2145 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2146 )
2147 )
2148 )
2149 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2150 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2151
2152 /* Now any messages */
2153
2154 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2155 {
d6c82d7b
JH
2156 int message_length = s ? Ustrlen(s) + 1 : 0;
2157 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2158 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
1ac6b2e7
JH
2159 )
2160 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2161 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2162 }
2163 }
2164
2165 /* OK, this process is now done. Free any cached resources that it opened,
2166 and close the pipe we were writing down before exiting. */
2167
f1e894f3 2168 (void)close(pfd[pipe_write]);
059ec3d9
PH
2169 search_tidyup();
2170 exit(EXIT_SUCCESS);
2171 }
2172
2173/* Back in the main process: panic if the fork did not succeed. This seems
2174better than returning an error - if forking is failing it is probably best
2175not to try other deliveries for this message. */
2176
2177if (pid < 0)
2178 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2179 addr->address);
2180
2181/* Read the pipe to get the delivery status codes and error messages. Our copy
2182of the writing end must be closed first, as otherwise read() won't return zero
2183on an empty pipe. We check that a status exists for each address before
2184overwriting the address structure. If data is missing, the default DEFER status
2185will remain. Afterwards, close the reading end. */
2186
f1e894f3 2187(void)close(pfd[pipe_write]);
059ec3d9 2188
d6c82d7b 2189for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9 2190 {
d6c82d7b 2191 len = read(pfd[pipe_read], &status, sizeof(int));
059ec3d9
PH
2192 if (len > 0)
2193 {
2194 int i;
2195 uschar **sptr;
2196
2197 addr2->transport_return = status;
d6c82d7b 2198 len = read(pfd[pipe_read], &transport_count,
059ec3d9 2199 sizeof(transport_count));
d6c82d7b
JH
2200 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2201 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2202 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
2203 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2204 len = read(pfd[pipe_read], &addr2->transport,
059ec3d9
PH
2205 sizeof(transport_instance *));
2206
2207 if (testflag(addr2, af_file))
2208 {
2209 int local_part_length;
d6c82d7b
JH
2210 len = read(pfd[pipe_read], &local_part_length, sizeof(int));
2211 len = read(pfd[pipe_read], big_buffer, local_part_length);
059ec3d9
PH
2212 big_buffer[local_part_length] = 0;
2213 addr2->local_part = string_copy(big_buffer);
2214 }
2215
d6c82d7b 2216 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
059ec3d9
PH
2217 {
2218 int message_length;
d6c82d7b 2219 len = read(pfd[pipe_read], &message_length, sizeof(int));
059ec3d9
PH
2220 if (message_length > 0)
2221 {
d6c82d7b 2222 len = read(pfd[pipe_read], big_buffer, message_length);
dc8091e7 2223 big_buffer[big_buffer_size-1] = '\0'; /* guard byte */
059ec3d9
PH
2224 if (len > 0) *sptr = string_copy(big_buffer);
2225 }
2226 }
2227 }
2228
2229 else
2230 {
2231 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2232 "from delivery subprocess", addr2->unique);
2233 break;
2234 }
2235 }
2236
f1e894f3 2237(void)close(pfd[pipe_read]);
059ec3d9
PH
2238
2239/* Unless shadowing, write all successful addresses immediately to the journal
2240file, to ensure they are recorded asap. For homonymic addresses, use the base
2241address plus the transport name. Failure to write the journal is panic-worthy,
2242but don't stop, as it may prove possible subsequently to update the spool file
2243in order to record the delivery. */
2244
2245if (!shadowing)
2246 {
d6c82d7b
JH
2247 for (addr2 = addr; addr2; addr2 = addr2->next)
2248 if (addr2->transport_return == OK)
2249 {
2250 if (testflag(addr2, af_homonym))
2251 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2252 else
2253 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
059ec3d9 2254
d6c82d7b
JH
2255 /* In the test harness, wait just a bit to let the subprocess finish off
2256 any debug output etc first. */
059ec3d9 2257
d6c82d7b 2258 if (running_in_test_harness) millisleep(300);
059ec3d9 2259
d6c82d7b
JH
2260 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2261 len = Ustrlen(big_buffer);
2262 if (write(journal_fd, big_buffer, len) != len)
2263 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2264 big_buffer, strerror(errno));
2265 }
059ec3d9
PH
2266
2267 /* Ensure the journal file is pushed out to disk. */
2268
54fc8428 2269 if (EXIMfsync(journal_fd) < 0)
059ec3d9
PH
2270 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2271 strerror(errno));
2272 }
2273
2274/* Wait for the process to finish. If it terminates with a non-zero code,
2275freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2276status values of all the addresses as they are. Take care to handle the case
2277when the subprocess doesn't seem to exist. This has been seen on one system
2278when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2279happens, wait() doesn't recognize the termination of child processes. Exim now
2280resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2281
2282while ((rc = wait(&status)) != pid)
059ec3d9
PH
2283 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2284 {
2285 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2286 addr->transport->driver_name);
2287 status = 0;
2288 break;
2289 }
059ec3d9
PH
2290
2291if ((status & 0xffff) != 0)
2292 {
2293 int msb = (status >> 8) & 255;
2294 int lsb = status & 255;
2295 int code = (msb == 0)? (lsb & 0x7f) : msb;
2296 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2297 addr->special_action = SPECIAL_FREEZE;
2298 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2299 "status 0x%04x: %s %d",
2300 addr->transport->driver_name,
2301 status,
d6c82d7b 2302 msb == 0 ? "terminated by signal" : "exit code",
059ec3d9
PH
2303 code);
2304 }
2305
2306/* If SPECIAL_WARN is set in the top address, send a warning message. */
2307
d6c82d7b 2308if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
059ec3d9
PH
2309 {
2310 int fd;
2311 uschar *warn_message;
d6c82d7b 2312 pid_t pid;
059ec3d9
PH
2313
2314 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2315
d6c82d7b 2316 if (!(warn_message = expand_string(addr->transport->warn_message)))
059ec3d9
PH
2317 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2318 "message for %s transport): %s", addr->transport->warn_message,
2319 addr->transport->name, expand_string_message);
d6c82d7b
JH
2320
2321 else if ((pid = child_open_exim(&fd)) > 0)
059ec3d9 2322 {
d6c82d7b
JH
2323 FILE *f = fdopen(fd, "wb");
2324 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2325 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2326 fprintf(f, "Auto-Submitted: auto-replied\n");
2327 if (!contains_header(US"From", warn_message))
2328 moan_write_from(f);
2329 fprintf(f, "%s", CS warn_message);
059ec3d9 2330
d6c82d7b 2331 /* Close and wait for child process to complete, without a timeout. */
059ec3d9 2332
d6c82d7b
JH
2333 (void)fclose(f);
2334 (void)child_close(pid, 0);
059ec3d9
PH
2335 }
2336
2337 addr->special_action = SPECIAL_NONE;
2338 }
2339}
2340
2341
2342
fa41615d 2343
3070ceee
JH
2344/* Check transport for the given concurrency limit. Return TRUE if over
2345the limit (or an expansion failure), else FALSE and if there was a limit,
2346the key for the hints database used for the concurrency count. */
fa41615d 2347
3070ceee
JH
2348static BOOL
2349tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key)
fa41615d 2350{
3070ceee
JH
2351unsigned max_parallel;
2352
2353if (!tp->max_parallel) return FALSE;
2354
2355max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE);
2356if (expand_string_message)
2357 {
2358 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2359 "in %s transport (%s): %s", tp->name, addr->address,
2360 expand_string_message);
2361 return TRUE;
2362 }
2363
2364if (max_parallel > 0)
2365 {
2366 uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name);
2367 if (!enq_start(serialize_key, max_parallel))
2368 {
2369 address_item * next;
2370 DEBUG(D_transport)
2371 debug_printf("skipping tpt %s because concurrency limit %u reached\n",
2372 tp->name, max_parallel);
2373 do
2374 {
2375 next = addr->next;
2376 addr->message = US"concurrency limit reached for transport";
2377 addr->basic_errno = ERRNO_TRETRY;
2378 post_process_one(addr, DEFER, LOG_MAIN, DTYPE_TRANSPORT, 0);
2379 } while ((addr = next));
2380 return TRUE;
2381 }
2382 *key = serialize_key;
2383 }
2384return FALSE;
fa41615d
JH
2385}
2386
2387
2388
059ec3d9
PH
2389/*************************************************
2390* Do local deliveries *
2391*************************************************/
2392
2393/* This function processes the list of addresses in addr_local. True local
2394deliveries are always done one address at a time. However, local deliveries can
2395be batched up in some cases. Typically this is when writing batched SMTP output
2396files for use by some external transport mechanism, or when running local
2397deliveries over LMTP.
2398
2399Arguments: None
2400Returns: Nothing
2401*/
2402
2403static void
2404do_local_deliveries(void)
2405{
2406open_db dbblock;
2407open_db *dbm_file = NULL;
2408time_t now = time(NULL);
2409
2410/* Loop until we have exhausted the supply of local deliveries */
2411
d6c82d7b 2412while (addr_local)
059ec3d9
PH
2413 {
2414 time_t delivery_start;
2415 int deliver_time;
2416 address_item *addr2, *addr3, *nextaddr;
2417 int logflags = LOG_MAIN;
2418 int logchar = dont_deliver? '*' : '=';
2419 transport_instance *tp;
fa41615d 2420 uschar * serialize_key = NULL;
059ec3d9
PH
2421
2422 /* Pick the first undelivered address off the chain */
2423
2424 address_item *addr = addr_local;
2425 addr_local = addr->next;
2426 addr->next = NULL;
2427
2428 DEBUG(D_deliver|D_transport)
2429 debug_printf("--------> %s <--------\n", addr->address);
2430
2431 /* An internal disaster if there is no transport. Should not occur! */
2432
d6c82d7b 2433 if (!(tp = addr->transport))
059ec3d9
PH
2434 {
2435 logflags |= LOG_PANIC;
2436 disable_logging = FALSE; /* Jic */
d6c82d7b
JH
2437 addr->message = addr->router
2438 ? string_sprintf("No transport set by %s router", addr->router->name)
2439 : string_sprintf("No transport set by system filter");
059ec3d9
PH
2440 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
2441 continue;
2442 }
2443
2444 /* Check that this base address hasn't previously been delivered to this
2445 transport. The check is necessary at this point to handle homonymic addresses
2446 correctly in cases where the pattern of redirection changes between delivery
2447 attempts. Non-homonymic previous delivery is detected earlier, at routing
2448 time. */
2449
c2c19e9d 2450 if (previously_transported(addr, FALSE)) continue;
059ec3d9
PH
2451
2452 /* There are weird cases where logging is disabled */
2453
2454 disable_logging = tp->disable_logging;
2455
f7fd3850
PH
2456 /* Check for batched addresses and possible amalgamation. Skip all the work
2457 if either batch_max <= 1 or there aren't any other addresses for local
2458 delivery. */
059ec3d9 2459
d6c82d7b 2460 if (tp->batch_max > 1 && addr_local)
059ec3d9
PH
2461 {
2462 int batch_count = 1;
2463 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
d6c82d7b
JH
2464 BOOL uses_lp = ( testflag(addr, af_pfr)
2465 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2466 )
2467 || readconf_depends((driver_instance *)tp, US"local_part");
059ec3d9
PH
2468 uschar *batch_id = NULL;
2469 address_item **anchor = &addr_local;
2470 address_item *last = addr;
2471 address_item *next;
2472
2473 /* Expand the batch_id string for comparison with other addresses.
2474 Expansion failure suppresses batching. */
2475
d6c82d7b 2476 if (tp->batch_id)
059ec3d9
PH
2477 {
2478 deliver_set_expansions(addr);
2479 batch_id = expand_string(tp->batch_id);
2480 deliver_set_expansions(NULL);
d6c82d7b 2481 if (!batch_id)
059ec3d9
PH
2482 {
2483 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2484 "in %s transport (%s): %s", tp->name, addr->address,
2485 expand_string_message);
2486 batch_count = tp->batch_max;
2487 }
2488 }
2489
2490 /* Until we reach the batch_max limit, pick off addresses which have the
2491 same characteristics. These are:
2492
2493 same transport
7816e254 2494 not previously delivered (see comment about 50 lines above)
059ec3d9 2495 same local part if the transport's configuration contains $local_part
f7fd3850 2496 or if this is a file or pipe delivery from a redirection
059ec3d9
PH
2497 same domain if the transport's configuration contains $domain
2498 same errors address
2499 same additional headers
2500 same headers to be removed
2501 same uid/gid for running the transport
2502 same first host if a host list is set
2503 */
2504
d6c82d7b 2505 while ((next = *anchor) && batch_count < tp->batch_max)
059ec3d9
PH
2506 {
2507 BOOL ok =
d6c82d7b
JH
2508 tp == next->transport
2509 && !previously_transported(next, TRUE)
2510 && (addr->flags & (af_pfr|af_file)) == (next->flags & (af_pfr|af_file))
2511 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2512 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2513 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2514 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2515 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2516 && same_ugid(tp, addr, next)
2517 && ( !addr->host_list && !next->host_list
2518 || addr->host_list
2519 && next->host_list
2520 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2521 );
059ec3d9
PH
2522
2523 /* If the transport has a batch_id setting, batch_id will be non-NULL
2524 from the expansion outside the loop. Expand for this address and compare.
2525 Expansion failure makes this address ineligible for batching. */
2526
d6c82d7b 2527 if (ok && batch_id)
059ec3d9
PH
2528 {
2529 uschar *bid;
2530 address_item *save_nextnext = next->next;
2531 next->next = NULL; /* Expansion for a single address */
2532 deliver_set_expansions(next);
2533 next->next = save_nextnext;
2534 bid = expand_string(tp->batch_id);
2535 deliver_set_expansions(NULL);
d6c82d7b 2536 if (!bid)
059ec3d9
PH
2537 {
2538 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2539 "in %s transport (%s): %s", tp->name, next->address,
2540 expand_string_message);
2541 ok = FALSE;
2542 }
2543 else ok = (Ustrcmp(batch_id, bid) == 0);
2544 }
2545
2546 /* Take address into batch if OK. */
2547
2548 if (ok)
2549 {
2550 *anchor = next->next; /* Include the address */
2551 next->next = NULL;
2552 last->next = next;
2553 last = next;
2554 batch_count++;
2555 }
fa41615d 2556 else anchor = &next->next; /* Skip the address */
059ec3d9
PH
2557 }
2558 }
2559
2560 /* We now have one or more addresses that can be delivered in a batch. Check
2561 whether the transport is prepared to accept a message of this size. If not,
2562 fail them all forthwith. If the expansion fails, or does not yield an
2563 integer, defer delivery. */
2564
d6c82d7b 2565 if (tp->message_size_limit)
059ec3d9
PH
2566 {
2567 int rc = check_message_size(tp, addr);
2568 if (rc != OK)
2569 {
2570 replicate_status(addr);
d6c82d7b 2571 while (addr)
059ec3d9
PH
2572 {
2573 addr2 = addr->next;
2574 post_process_one(addr, rc, logflags, DTYPE_TRANSPORT, 0);
2575 addr = addr2;
2576 }
2577 continue; /* With next batch of addresses */
2578 }
2579 }
2580
2581 /* If we are not running the queue, or if forcing, all deliveries will be
2582 attempted. Otherwise, we must respect the retry times for each address. Even
2583 when not doing this, we need to set up the retry key string, and determine
2584 whether a retry record exists, because after a successful delivery, a delete
2585 retry item must be set up. Keep the retry database open only for the duration
2586 of these checks, rather than for all local deliveries, because some local
2587 deliveries (e.g. to pipes) can take a substantial time. */
2588
d6c82d7b 2589 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
059ec3d9
PH
2590 {
2591 DEBUG(D_deliver|D_retry|D_hints_lookup)
2592 debug_printf("no retry data available\n");
2593 }
2594
2595 addr2 = addr;
2596 addr3 = NULL;
d6c82d7b 2597 while (addr2)
059ec3d9
PH
2598 {
2599 BOOL ok = TRUE; /* to deliver this address */
2600 uschar *retry_key;
2601
2602 /* Set up the retry key to include the domain or not, and change its
2603 leading character from "R" to "T". Must make a copy before doing this,
2604 because the old key may be pointed to from a "delete" retry item after
2605 a routing delay. */
2606
2607 retry_key = string_copy(
d6c82d7b 2608 tp->retry_use_local_part ? addr2->address_retry_key :
059ec3d9
PH
2609 addr2->domain_retry_key);
2610 *retry_key = 'T';
2611
2612 /* Inspect the retry data. If there is no hints file, delivery happens. */
2613
d6c82d7b 2614 if (dbm_file)
059ec3d9
PH
2615 {
2616 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2617
2618 /* If there is no retry record, delivery happens. If there is,
2619 remember it exists so it can be deleted after a successful delivery. */
2620
d6c82d7b 2621 if (retry_record)
059ec3d9
PH
2622 {
2623 setflag(addr2, af_lt_retry_exists);
2624
2625 /* A retry record exists for this address. If queue running and not
2626 forcing, inspect its contents. If the record is too old, or if its
2627 retry time has come, or if it has passed its cutoff time, delivery
2628 will go ahead. */
2629
2630 DEBUG(D_retry)
2631 {
ea49d0e1
PH
2632 debug_printf("retry record exists: age=%s ",
2633 readconf_printtime(now - retry_record->time_stamp));
2634 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2635 debug_printf(" time to retry = %s expired = %d\n",
2636 readconf_printtime(retry_record->next_try - now),
2637 retry_record->expired);
059ec3d9
PH
2638 }
2639
2640 if (queue_running && !deliver_force)
2641 {
d6c82d7b
JH
2642 ok = (now - retry_record->time_stamp > retry_data_expire)
2643 || (now >= retry_record->next_try)
2644 || retry_record->expired;
059ec3d9
PH
2645
2646 /* If we haven't reached the retry time, there is one more check
1ddeb334 2647 to do, which is for the ultimate address timeout. */
059ec3d9
PH
2648
2649 if (!ok)
ba9af0af
TF
2650 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2651 retry_record, now);
059ec3d9
PH
2652 }
2653 }
2654 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2655 }
2656
2657 /* This address is to be delivered. Leave it on the chain. */
2658
2659 if (ok)
2660 {
2661 addr3 = addr2;
2662 addr2 = addr2->next;
2663 }
2664
2665 /* This address is to be deferred. Take it out of the chain, and
2666 post-process it as complete. Must take it out of the chain first,
2667 because post processing puts it on another chain. */
2668
2669 else
2670 {
2671 address_item *this = addr2;
2672 this->message = US"Retry time not yet reached";
2673 this->basic_errno = ERRNO_LRETRY;
d6c82d7b
JH
2674 addr2 = addr3 ? (addr3->next = addr2->next)
2675 : (addr = addr2->next);
059ec3d9
PH
2676 post_process_one(this, DEFER, logflags, DTYPE_TRANSPORT, 0);
2677 }
2678 }
2679
d6c82d7b 2680 if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
2681
2682 /* If there are no addresses left on the chain, they all deferred. Loop
2683 for the next set of addresses. */
2684
d6c82d7b 2685 if (!addr) continue;
059ec3d9 2686
fa41615d
JH
2687 /* If the transport is limited for parallellism, enforce that here.
2688 We use a hints DB entry, incremented here and decremented after
2689 the transport (and any shadow transport) completes. */
2690
3070ceee 2691 if (tpt_parallel_check(tp, addr, &serialize_key))
fa41615d 2692 {
fa41615d
JH
2693 if (expand_string_message)
2694 {
2695 logflags |= LOG_PANIC;
3070ceee
JH
2696 do
2697 {
2698 addr = addr->next;
fa41615d 2699 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
3070ceee 2700 } while ((addr = addr2));
fa41615d 2701 }
3070ceee 2702 continue; /* Loop for the next set of addresses. */
fa41615d
JH
2703 }
2704
2705
059ec3d9
PH
2706 /* So, finally, we do have some addresses that can be passed to the
2707 transport. Before doing so, set up variables that are relevant to a
2708 single delivery. */
2709
2710 deliver_set_expansions(addr);
2711 delivery_start = time(NULL);
2712 deliver_local(addr, FALSE);
2713 deliver_time = (int)(time(NULL) - delivery_start);
2714
2715 /* If a shadow transport (which must perforce be another local transport), is
2716 defined, and its condition is met, we must pass the message to the shadow
2717 too, but only those addresses that succeeded. We do this by making a new
2718 chain of addresses - also to keep the original chain uncontaminated. We must
2719 use a chain rather than doing it one by one, because the shadow transport may
2720 batch.
2721
2722 NOTE: if the condition fails because of a lookup defer, there is nothing we
2723 can do! */
2724
d6c82d7b
JH
2725 if ( tp->shadow
2726 && ( !tp->shadow_condition
2727 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
2728 ) )
059ec3d9
PH
2729 {
2730 transport_instance *stp;
2731 address_item *shadow_addr = NULL;
2732 address_item **last = &shadow_addr;
2733
d6c82d7b 2734 for (stp = transports; stp; stp = stp->next)
059ec3d9
PH
2735 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
2736
d6c82d7b 2737 if (!stp)
059ec3d9
PH
2738 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
2739 tp->shadow);
2740
2741 /* Pick off the addresses that have succeeded, and make clones. Put into
2742 the shadow_message field a pointer to the shadow_message field of the real
2743 address. */
2744
d6c82d7b
JH
2745 else for (addr2 = addr; addr2; addr2 = addr2->next)
2746 if (addr2->transport_return == OK)
2747 {
2748 addr3 = store_get(sizeof(address_item));
2749 *addr3 = *addr2;
2750 addr3->next = NULL;
2751 addr3->shadow_message = (uschar *) &(addr2->shadow_message);
2752 addr3->transport = stp;
2753 addr3->transport_return = DEFER;
2754 addr3->return_filename = NULL;
2755 addr3->return_file = -1;
2756 *last = addr3;
2757 last = &(addr3->next);
2758 }
059ec3d9
PH
2759
2760 /* If we found any addresses to shadow, run the delivery, and stick any
2761 message back into the shadow_message field in the original. */
2762
d6c82d7b 2763 if (shadow_addr)
059ec3d9
PH
2764 {
2765 int save_count = transport_count;
2766
2767 DEBUG(D_deliver|D_transport)
2768 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
2769 deliver_local(shadow_addr, TRUE);
2770
d6c82d7b 2771 for(; shadow_addr; shadow_addr = shadow_addr->next)
059ec3d9
PH
2772 {
2773 int sresult = shadow_addr->transport_return;
d6c82d7b
JH
2774 *(uschar **)shadow_addr->shadow_message =
2775 sresult == OK
2776 ? string_sprintf(" ST=%s", stp->name)
2777 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
2778 shadow_addr->basic_errno <= 0
2779 ? US""
2780 : US strerror(shadow_addr->basic_errno),
2781 shadow_addr->basic_errno <= 0 || !shadow_addr->message
2782 ? US""
2783 : US": ",
2784 shadow_addr->message
2785 ? shadow_addr->message
2786 : shadow_addr->basic_errno <= 0
2787 ? US"unknown error"
2788 : US"");
059ec3d9
PH
2789
2790 DEBUG(D_deliver|D_transport)
2791 debug_printf("%s shadow transport returned %s for %s\n",
2792 stp->name,
d6c82d7b
JH
2793 sresult == OK ? "OK" :
2794 sresult == DEFER ? "DEFER" :
2795 sresult == FAIL ? "FAIL" :
2796 sresult == PANIC ? "PANIC" : "?",
059ec3d9
PH
2797 shadow_addr->address);
2798 }
2799
2800 DEBUG(D_deliver|D_transport)
2801 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
2802
2803 transport_count = save_count; /* Restore original transport count */
2804 }
2805 }
2806
2807 /* Cancel the expansions that were set up for the delivery. */
2808
2809 deliver_set_expansions(NULL);
2810
fa41615d
JH
2811 /* If the transport was parallelism-limited, decrement the hints DB record. */
2812
2813 if (serialize_key) enq_end(serialize_key);
2814
059ec3d9
PH
2815 /* Now we can process the results of the real transport. We must take each
2816 address off the chain first, because post_process_one() puts it on another
2817 chain. */
2818
d6c82d7b 2819 for (addr2 = addr; addr2; addr2 = nextaddr)
059ec3d9
PH
2820 {
2821 int result = addr2->transport_return;
2822 nextaddr = addr2->next;
2823
2824 DEBUG(D_deliver|D_transport)
2825 debug_printf("%s transport returned %s for %s\n",
2826 tp->name,
d6c82d7b
JH
2827 result == OK ? "OK" :
2828 result == DEFER ? "DEFER" :
2829 result == FAIL ? "FAIL" :
2830 result == PANIC ? "PANIC" : "?",
059ec3d9
PH
2831 addr2->address);
2832
2833 /* If there is a retry_record, or if delivery is deferred, build a retry
2834 item for setting a new retry time or deleting the old retry record from
2835 the database. These items are handled all together after all addresses
2836 have been handled (so the database is open just for a short time for
2837 updating). */
2838
2839 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
2840 {
d6c82d7b
JH
2841 int flags = result == DEFER ? 0 : rf_delete;
2842 uschar *retry_key = string_copy(tp->retry_use_local_part
2843 ? addr2->address_retry_key : addr2->domain_retry_key);
059ec3d9
PH
2844 *retry_key = 'T';
2845 retry_add_item(addr2, retry_key, flags);
2846 }
2847
2848 /* Done with this address */
2849
2850 if (result == OK) addr2->more_errno = deliver_time;
2851 post_process_one(addr2, result, logflags, DTYPE_TRANSPORT, logchar);
2852
2853 /* If a pipe delivery generated text to be sent back, the result may be
2854 changed to FAIL, and we must copy this for subsequent addresses in the
2855 batch. */
2856
2857 if (addr2->transport_return != result)
2858 {
d6c82d7b 2859 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
059ec3d9
PH
2860 {
2861 addr3->transport_return = addr2->transport_return;
2862 addr3->basic_errno = addr2->basic_errno;
2863 addr3->message = addr2->message;
2864 }
2865 result = addr2->transport_return;
2866 }
2867
2868 /* Whether or not the result was changed to FAIL, we need to copy the
2869 return_file value from the first address into all the addresses of the
2870 batch, so they are all listed in the error message. */
2871
2872 addr2->return_file = addr->return_file;
2873
2874 /* Change log character for recording successful deliveries. */
2875
2876 if (result == OK) logchar = '-';
2877 }
2878 } /* Loop back for next batch of addresses */
2879}
2880
2881
2882
2883
2884/*************************************************
2885* Sort remote deliveries *
2886*************************************************/
2887
2888/* This function is called if remote_sort_domains is set. It arranges that the
2889chain of addresses for remote deliveries is ordered according to the strings
2890specified. Try to make this shuffling reasonably efficient by handling
2891sequences of addresses rather than just single ones.
2892
2893Arguments: None
2894Returns: Nothing
2895*/
2896
2897static void
2898sort_remote_deliveries(void)
2899{
2900int sep = 0;
2901address_item **aptr = &addr_remote;
55414b25 2902const uschar *listptr = remote_sort_domains;
059ec3d9
PH
2903uschar *pattern;
2904uschar patbuf[256];
2905
d6c82d7b
JH
2906while ( *aptr
2907 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
2908 )
059ec3d9
PH
2909 {
2910 address_item *moved = NULL;
2911 address_item **bptr = &moved;
2912
d6c82d7b 2913 while (*aptr)
059ec3d9
PH
2914 {
2915 address_item **next;
2916 deliver_domain = (*aptr)->domain; /* set $domain */
55414b25 2917 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
059ec3d9
PH
2918 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
2919 {
d6c82d7b 2920 aptr = &(*aptr)->next;
059ec3d9
PH
2921 continue;
2922 }
2923
d6c82d7b
JH
2924 next = &(*aptr)->next;
2925 while ( *next
2926 && (deliver_domain = (*next)->domain, /* Set $domain */
55414b25 2927 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
d6c82d7b
JH
2928 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
2929 )
2930 next = &(*next)->next;
059ec3d9
PH
2931
2932 /* If the batch of non-matchers is at the end, add on any that were
2933 extracted further up the chain, and end this iteration. Otherwise,
2934 extract them from the chain and hang on the moved chain. */
2935
d6c82d7b 2936 if (!*next)
059ec3d9
PH
2937 {
2938 *next = moved;
2939 break;
2940 }
2941
2942 *bptr = *aptr;
2943 *aptr = *next;
2944 *next = NULL;
2945 bptr = next;
d6c82d7b 2946 aptr = &(*aptr)->next;
059ec3d9
PH
2947 }
2948
2949 /* If the loop ended because the final address matched, *aptr will
2950 be NULL. Add on to the end any extracted non-matching addresses. If
2951 *aptr is not NULL, the loop ended via "break" when *next is null, that
2952 is, there was a string of non-matching addresses at the end. In this
2953 case the extracted addresses have already been added on the end. */
2954
d6c82d7b 2955 if (!*aptr) *aptr = moved;
059ec3d9
PH
2956 }
2957
2958DEBUG(D_deliver)
2959 {
2960 address_item *addr;
2961 debug_printf("remote addresses after sorting:\n");
d6c82d7b 2962 for (addr = addr_remote; addr; addr = addr->next)
059ec3d9
PH
2963 debug_printf(" %s\n", addr->address);
2964 }
2965}
2966
2967
2968
2969/*************************************************
2970* Read from pipe for remote delivery subprocess *
2971*************************************************/
2972
2973/* This function is called when the subprocess is complete, but can also be
2974called before it is complete, in order to empty a pipe that is full (to prevent
2975deadlock). It must therefore keep track of its progress in the parlist data
2976block.
2977
2978We read the pipe to get the delivery status codes and a possible error message
2979for each address, optionally preceded by unusability data for the hosts and
2980also by optional retry data.
2981
2982Read in large chunks into the big buffer and then scan through, interpreting
2983the data therein. In most cases, only a single read will be necessary. No
1c5466b9
PH
2984individual item will ever be anywhere near 2500 bytes in length, so by ensuring
2985that we read the next chunk when there is less than 2500 bytes left in the
2986non-final chunk, we can assume each item is complete in the buffer before
2987handling it. Each item is written using a single write(), which is atomic for
2988small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
2989often bigger) so even if we are reading while the subprocess is still going, we
2990should never have only a partial item in the buffer.
059ec3d9
PH
2991
2992Argument:
2993 poffset the offset of the parlist item
2994 eop TRUE if the process has completed
2995
2996Returns: TRUE if the terminating 'Z' item has been read,
2997 or there has been a disaster (i.e. no more data needed);
2998 FALSE otherwise
2999*/
3000
3001static BOOL
3002par_read_pipe(int poffset, BOOL eop)
3003{
3004host_item *h;
3005pardata *p = parlist + poffset;
3006address_item *addrlist = p->addrlist;
3007address_item *addr = p->addr;
3008pid_t pid = p->pid;
3009int fd = p->fd;
3010uschar *endptr = big_buffer;
3011uschar *ptr = endptr;
3012uschar *msg = p->msg;
3013BOOL done = p->done;
3014BOOL unfinished = TRUE;
bd21a787
WB
3015/* minimum size to read is header size including id, subid and length */
3016int required = PIPE_HEADER_SIZE;
059ec3d9
PH
3017
3018/* Loop through all items, reading from the pipe when necessary. The pipe
3019is set up to be non-blocking, but there are two different Unix mechanisms in
3020use. Exim uses O_NONBLOCK if it is defined. This returns 0 for end of file,
3021and EAGAIN for no more data. If O_NONBLOCK is not defined, Exim uses O_NDELAY,
3022which returns 0 for both end of file and no more data. We distinguish the
3023two cases by taking 0 as end of file only when we know the process has
3024completed.
3025
3026Each separate item is written to the pipe in a single write(), and as they are
3027all short items, the writes will all be atomic and we should never find
8e669ac1
PH
3028ourselves in the position of having read an incomplete item. "Short" in this
3029case can mean up to about 1K in the case when there is a long error message
1c5466b9 3030associated with an address. */
059ec3d9
PH
3031
3032DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
3033 (int)p->pid, eop? "ended" : "not ended");
3034
3035while (!done)
3036 {
3037 retry_item *r, **rp;
3038 int remaining = endptr - ptr;
bd21a787
WB
3039 uschar header[PIPE_HEADER_SIZE + 1];
3040 uschar id, subid;
3041 uschar *endc;
059ec3d9
PH
3042
3043 /* Read (first time) or top up the chars in the buffer if necessary.
3044 There will be only one read if we get all the available data (i.e. don't
3045 fill the buffer completely). */
3046
bd21a787 3047 if (remaining < required && unfinished)
059ec3d9
PH
3048 {
3049 int len;
3050 int available = big_buffer_size - remaining;
3051
3052 if (remaining > 0) memmove(big_buffer, ptr, remaining);
3053
3054 ptr = big_buffer;
3055 endptr = big_buffer + remaining;
3056 len = read(fd, endptr, available);
3057
3058 DEBUG(D_deliver) debug_printf("read() yielded %d\n", len);
3059
3060 /* If the result is EAGAIN and the process is not complete, just
3061 stop reading any more and process what we have already. */
3062
3063 if (len < 0)
3064 {
3065 if (!eop && errno == EAGAIN) len = 0; else
3066 {
3067 msg = string_sprintf("failed to read pipe from transport process "
3068 "%d for transport %s: %s", pid, addr->transport->driver_name,
3069 strerror(errno));
3070 break;
3071 }
3072 }
3073
3074 /* If the length is zero (eof or no-more-data), just process what we
3075 already have. Note that if the process is still running and we have
3076 read all the data in the pipe (but less that "available") then we
3077 won't read any more, as "unfinished" will get set FALSE. */
3078
3079 endptr += len;
bd21a787 3080 remaining += len;
059ec3d9
PH
3081 unfinished = len == available;
3082 }
3083
3084 /* If we are at the end of the available data, exit the loop. */
059ec3d9
PH
3085 if (ptr >= endptr) break;
3086
bd21a787
WB
3087 /* copy and read header */
3088 memcpy(header, ptr, PIPE_HEADER_SIZE);
94431adb 3089 header[PIPE_HEADER_SIZE] = '\0';
bd21a787
WB
3090 id = header[0];
3091 subid = header[1];
3092 required = Ustrtol(header + 2, &endc, 10) + PIPE_HEADER_SIZE; /* header + data */
3093 if (*endc)
3094 {
3095 msg = string_sprintf("failed to read pipe from transport process "
3096 "%d for transport %s: error reading size from header", pid, addr->transport->driver_name);
3097 done = TRUE;
3098 break;
3099 }
3100
3101 DEBUG(D_deliver)
94431adb 3102 debug_printf("header read id:%c,subid:%c,size:%s,required:%d,remaining:%d,unfinished:%d\n",
bd21a787
WB
3103 id, subid, header+2, required, remaining, unfinished);
3104
3105 /* is there room for the dataset we want to read ? */
3106 if (required > big_buffer_size - PIPE_HEADER_SIZE)
3107 {
3108 msg = string_sprintf("failed to read pipe from transport process "
3109 "%d for transport %s: big_buffer too small! required size=%d buffer size=%d", pid, addr->transport->driver_name,
3110 required, big_buffer_size - PIPE_HEADER_SIZE);
3111 done = TRUE;
3112 break;
3113 }
3114
94431adb 3115 /* we wrote all datasets with atomic write() calls
bd21a787 3116 remaining < required only happens if big_buffer was too small
94431adb 3117 to get all available data from pipe. unfinished has to be true
bd21a787
WB
3118 as well. */
3119 if (remaining < required)
8fa0a4d4 3120 {
bd21a787
WB
3121 if (unfinished)
3122 continue;
8fa0a4d4 3123 msg = string_sprintf("failed to read pipe from transport process "
94431adb 3124 "%d for transport %s: required size=%d > remaining size=%d and unfinished=false",
8fa0a4d4
JH
3125 pid, addr->transport->driver_name, required, remaining);
3126 done = TRUE;
3127 break;
3128 }
bd21a787
WB
3129
3130 /* step behind the header */
3131 ptr += PIPE_HEADER_SIZE;
94431adb 3132
059ec3d9
PH
3133 /* Handle each possible type of item, assuming the complete item is
3134 available in store. */
3135
bd21a787 3136 switch (id)
059ec3d9
PH
3137 {
3138 /* Host items exist only if any hosts were marked unusable. Match
3139 up by checking the IP address. */
3140
3141 case 'H':
d6c82d7b 3142 for (h = addrlist->host_list; h; h = h->next)
059ec3d9 3143 {
d6c82d7b 3144 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
059ec3d9
PH
3145 h->status = ptr[0];
3146 h->why = ptr[1];
3147 }
3148 ptr += 2;
3149 while (*ptr++);
3150 break;
3151
3152 /* Retry items are sent in a preceding R item for each address. This is
3153 kept separate to keep each message short enough to guarantee it won't
3154 be split in the pipe. Hopefully, in the majority of cases, there won't in
3155 fact be any retry items at all.
3156
3157 The complete set of retry items might include an item to delete a
3158 routing retry if there was a previous routing delay. However, routing
3159 retries are also used when a remote transport identifies an address error.
3160 In that case, there may also be an "add" item for the same key. Arrange
3161 that a "delete" item is dropped in favour of an "add" item. */
3162
3163 case 'R':
d6c82d7b 3164 if (!addr) goto ADDR_MISMATCH;
059ec3d9
PH
3165
3166 DEBUG(D_deliver|D_retry)
3167 debug_printf("reading retry information for %s from subprocess\n",
3168 ptr+1);
3169
3170 /* Cut out any "delete" items on the list. */
3171
d6c82d7b 3172 for (rp = &(addr->retries); (r = *rp); rp = &r->next)
059ec3d9
PH
3173 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3174 {
3175 if ((r->flags & rf_delete) == 0) break; /* It was not "delete" */
3176 *rp = r->next; /* Excise a delete item */
3177 DEBUG(D_deliver|D_retry)
3178 debug_printf(" existing delete item dropped\n");
3179 }
059ec3d9
PH
3180
3181 /* We want to add a delete item only if there is no non-delete item;
3182 however we still have to step ptr through the data. */
3183
d6c82d7b 3184 if (!r || (*ptr & rf_delete) == 0)
059ec3d9
PH
3185 {
3186 r = store_get(sizeof(retry_item));
3187 r->next = addr->retries;
3188 addr->retries = r;
3189 r->flags = *ptr++;
3190 r->key = string_copy(ptr);
3191 while (*ptr++);
3192 memcpy(&(r->basic_errno), ptr, sizeof(r->basic_errno));
3193 ptr += sizeof(r->basic_errno);
3194 memcpy(&(r->more_errno), ptr, sizeof(r->more_errno));
3195 ptr += sizeof(r->more_errno);
3196 r->message = (*ptr)? string_copy(ptr) : NULL;
3197 DEBUG(D_deliver|D_retry)
3198 debug_printf(" added %s item\n",
3199 ((r->flags & rf_delete) == 0)? "retry" : "delete");
3200 }
3201
3202 else
3203 {
3204 DEBUG(D_deliver|D_retry)
3205 debug_printf(" delete item not added: non-delete item exists\n");
3206 ptr++;
3207 while(*ptr++);
3208 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3209 }
3210
3211 while(*ptr++);
3212 break;
3213
3214 /* Put the amount of data written into the parlist block */
3215
3216 case 'S':
3217 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3218 ptr += sizeof(transport_count);
3219 break;
3220
3221 /* Address items are in the order of items on the address chain. We
3222 remember the current address value in case this function is called
3223 several times to empty the pipe in stages. Information about delivery
3224 over TLS is sent in a preceding X item for each address. We don't put
3225 it in with the other info, in order to keep each message short enough to
3226 guarantee it won't be split in the pipe. */
3227
a7538db1 3228#ifdef SUPPORT_TLS
059ec3d9 3229 case 'X':
d6c82d7b 3230 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
bd21a787 3231 switch (subid)
9d1c15ef
JH
3232 {
3233 case '1':
3234 addr->cipher = NULL;
3235 addr->peerdn = NULL;
3236
3237 if (*ptr)
3238 addr->cipher = string_copy(ptr);
3239 while (*ptr++);
3240 if (*ptr)
9d1c15ef 3241 addr->peerdn = string_copy(ptr);
9d1c15ef
JH
3242 break;
3243
3244 case '2':
9d1c15ef
JH
3245 if (*ptr)
3246 (void) tls_import_cert(ptr, &addr->peercert);
152e7604
JH
3247 else
3248 addr->peercert = NULL;
9d1c15ef
JH
3249 break;
3250
3251 case '3':
9d1c15ef
JH
3252 if (*ptr)
3253 (void) tls_import_cert(ptr, &addr->ourcert);
152e7604
JH
3254 else
3255 addr->ourcert = NULL;
9d1c15ef 3256 break;
018058b2 3257
a7538db1 3258# ifndef DISABLE_OCSP
018058b2
JH
3259 case '4':
3260 addr->ocsp = OCSP_NOT_REQ;
3261 if (*ptr)
3262 addr->ocsp = *ptr - '0';
3263 break;
a7538db1 3264# endif
9d1c15ef 3265 }
059ec3d9
PH
3266 while (*ptr++);
3267 break;
a7538db1 3268#endif /*SUPPORT_TLS*/
059ec3d9 3269
6f123593 3270 case 'C': /* client authenticator information */
bd21a787 3271 switch (subid)
9d1c15ef
JH
3272 {
3273 case '1':
3274 addr->authenticator = (*ptr)? string_copy(ptr) : NULL;
3275 break;
3276 case '2':
3277 addr->auth_id = (*ptr)? string_copy(ptr) : NULL;
3278 break;
3279 case '3':
3280 addr->auth_sndr = (*ptr)? string_copy(ptr) : NULL;
3281 break;
3282 }
6f123593
JH
3283 while (*ptr++);
3284 break;
3285
8ccd00b1 3286#ifndef DISABLE_PRDR
fd98a5c6 3287 case 'P':
8ccd00b1
JH
3288 addr->flags |= af_prdr_used;
3289 break;
fd98a5c6
JH
3290#endif
3291
6c1c3d1d 3292 case 'D':
d6c82d7b 3293 if (!addr) goto ADDR_MISMATCH;
6c1c3d1d
WB
3294 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3295 ptr += sizeof(addr->dsn_aware);
3296 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
3297 break;
6c1c3d1d 3298
059ec3d9 3299 case 'A':
d6c82d7b 3300 if (!addr)
059ec3d9
PH
3301 {
3302 ADDR_MISMATCH:
3303 msg = string_sprintf("address count mismatch for data read from pipe "
3304 "for transport process %d for transport %s", pid,
3305 addrlist->transport->driver_name);
3306 done = TRUE;
3307 break;
3308 }
3309
895fbaf2 3310 switch (subid)
059ec3d9 3311 {
e6d2a989
JH
3312#ifdef SUPPORT_SOCKS
3313 case '2': /* proxy information; must arrive before A0 and applies to that addr XXX oops*/
3314 proxy_session = TRUE; /*XXX shouod this be cleared somewhere? */
3315 if (*ptr == 0)
3316 ptr++;
3317 else
3318 {
3319 proxy_local_address = string_copy(ptr);
3320 while(*ptr++);
3321 memcpy(&proxy_local_port, ptr, sizeof(proxy_local_port));
3322 ptr += sizeof(proxy_local_port);
3323 }
3324 break;
3325#endif
3326
895fbaf2
JH
3327#ifdef EXPERIMENTAL_DSN_INFO
3328 case '1': /* must arrive before A0, and applies to that addr */
3329 /* Two strings: smtp_greeting and helo_response */
3330 addr->smtp_greeting = string_copy(ptr);
3331 while(*ptr++);
3332 addr->helo_response = string_copy(ptr);
3333 while(*ptr++);
3334 break;
3335#endif
059ec3d9 3336
895fbaf2
JH
3337 case '0':
3338 addr->transport_return = *ptr++;
3339 addr->special_action = *ptr++;
3340 memcpy(&(addr->basic_errno), ptr, sizeof(addr->basic_errno));
3341 ptr += sizeof(addr->basic_errno);
3342 memcpy(&(addr->more_errno), ptr, sizeof(addr->more_errno));
3343 ptr += sizeof(addr->more_errno);
3344 memcpy(&(addr->flags), ptr, sizeof(addr->flags));
3345 ptr += sizeof(addr->flags);
3346 addr->message = (*ptr)? string_copy(ptr) : NULL;
3347 while(*ptr++);
3348 addr->user_message = (*ptr)? string_copy(ptr) : NULL;
3349 while(*ptr++);
059ec3d9 3350
895fbaf2
JH
3351 /* Always two strings for host information, followed by the port number and DNSSEC mark */
3352
3353 if (*ptr != 0)
3354 {
3355 h = store_get(sizeof(host_item));
3356 h->name = string_copy(ptr);
3357 while (*ptr++);
3358 h->address = string_copy(ptr);
3359 while(*ptr++);
3360 memcpy(&(h->port), ptr, sizeof(h->port));
3361 ptr += sizeof(h->port);
3362 h->dnssec = *ptr == '2' ? DS_YES
3363 : *ptr == '1' ? DS_NO
3364 : DS_UNK;
3365 ptr++;
3366 addr->host_used = h;
3367 }
3368 else ptr++;
3369
3370 /* Finished with this address */
3371
3372 addr = addr->next;
3373 break;
3374 }
059ec3d9
PH
3375 break;
3376
aeaf5db3
JH
3377 /* Local interface address/port */
3378 case 'I':
3379 if (*ptr) sending_ip_address = string_copy(ptr);
3380 while (*ptr++) ;
93a6fce2 3381 if (*ptr) sending_port = atoi(CS ptr);
aeaf5db3
JH
3382 while (*ptr++) ;
3383 break;
3384
059ec3d9
PH
3385 /* Z marks the logical end of the data. It is followed by '0' if
3386 continue_transport was NULL at the end of transporting, otherwise '1'.
3387 We need to know when it becomes NULL during a delivery down a passed SMTP
3388 channel so that we don't try to pass anything more down it. Of course, for
3389 most normal messages it will remain NULL all the time. */
3390
3391 case 'Z':
3392 if (*ptr == '0')
3393 {
3394 continue_transport = NULL;
3395 continue_hostname = NULL;
3396 }
3397 done = TRUE;
bd21a787 3398 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
059ec3d9
PH
3399 break;
3400
3401 /* Anything else is a disaster. */
3402
3403 default:
3404 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3405 "process %d for transport %s", ptr[-1], pid,
3406 addr->transport->driver_name);
3407 done = TRUE;
3408 break;
3409 }
3410 }
3411
3412/* The done flag is inspected externally, to determine whether or not to
3413call the function again when the process finishes. */
3414
3415p->done = done;
3416
3417/* If the process hadn't finished, and we haven't seen the end of the data
3418or suffered a disaster, update the rest of the state, and return FALSE to
3419indicate "not finished". */
3420
3421if (!eop && !done)
3422 {
3423 p->addr = addr;
3424 p->msg = msg;
3425 return FALSE;
3426 }
3427
3428/* Close our end of the pipe, to prevent deadlock if the far end is still
3429pushing stuff into it. */
3430
f1e894f3 3431(void)close(fd);
059ec3d9
PH
3432p->fd = -1;
3433
3434/* If we have finished without error, but haven't had data for every address,
3435something is wrong. */
3436
d6c82d7b 3437if (!msg && addr)
059ec3d9
PH
3438 msg = string_sprintf("insufficient address data read from pipe "
3439 "for transport process %d for transport %s", pid,
3440 addr->transport->driver_name);
3441
3442/* If an error message is set, something has gone wrong in getting back
3443the delivery data. Put the message into each address and freeze it. */
3444
d6c82d7b
JH
3445if (msg)
3446 for (addr = addrlist; addr; addr = addr->next)
059ec3d9
PH
3447 {
3448 addr->transport_return = DEFER;
3449 addr->special_action = SPECIAL_FREEZE;
3450 addr->message = msg;
3451 }
059ec3d9
PH
3452
3453/* Return TRUE to indicate we have got all we need from this process, even
3454if it hasn't actually finished yet. */
3455
3456return TRUE;
3457}
3458
3459
3460
3461/*************************************************
3462* Post-process a set of remote addresses *
3463*************************************************/
3464
3465/* Do what has to be done immediately after a remote delivery for each set of
3466addresses, then re-write the spool if necessary. Note that post_process_one
3467puts the address on an appropriate queue; hence we must fish off the next
3468one first. This function is also called if there is a problem with setting
3469up a subprocess to do a remote delivery in parallel. In this case, the final
3470argument contains a message, and the action must be forced to DEFER.
3471
3472Argument:
3473 addr pointer to chain of address items
3474 logflags flags for logging
3475 msg NULL for normal cases; -> error message for unexpected problems
3476 fallback TRUE if processing fallback hosts
3477
3478Returns: nothing
3479*/
3480
3481static void
3482remote_post_process(address_item *addr, int logflags, uschar *msg,
3483 BOOL fallback)
3484{
3485host_item *h;
3486
3487/* If any host addresses were found to be unusable, add them to the unusable
3488tree so that subsequent deliveries don't try them. */
3489
d6c82d7b
JH
3490for (h = addr->host_list; h; h = h->next)
3491 if (h->address)
3492 if (h->status >= hstatus_unusable) tree_add_unusable(h);
059ec3d9
PH
3493
3494/* Now handle each address on the chain. The transport has placed '=' or '-'
3495into the special_action field for each successful delivery. */
3496
d6c82d7b 3497while (addr)
059ec3d9
PH
3498 {
3499 address_item *next = addr->next;
3500
3501 /* If msg == NULL (normal processing) and the result is DEFER and we are
3502 processing the main hosts and there are fallback hosts available, put the
3503 address on the list for fallback delivery. */
3504
d6c82d7b
JH
3505 if ( addr->transport_return == DEFER
3506 && addr->fallback_hosts
3507 && !fallback
3508 && !msg
3509 )
059ec3d9
PH
3510 {
3511 addr->host_list = addr->fallback_hosts;
3512 addr->next = addr_fallback;
3513 addr_fallback = addr;
3514 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3515 }
3516
3517 /* If msg is set (=> unexpected problem), set it in the address before
3518 doing the ordinary post processing. */
3519
3520 else
3521 {
d6c82d7b 3522 if (msg)
059ec3d9
PH
3523 {
3524 addr->message = msg;
3525 addr->transport_return = DEFER;
3526 }
3527 (void)post_process_one(addr, addr->transport_return, logflags,
3528 DTYPE_TRANSPORT, addr->special_action);
3529 }
3530
3531 /* Next address */
3532
3533 addr = next;
3534 }
3535
3536/* If we have just delivered down a passed SMTP channel, and that was
3537the last address, the channel will have been closed down. Now that
3538we have logged that delivery, set continue_sequence to 1 so that
3539any subsequent deliveries don't get "*" incorrectly logged. */
3540
d6c82d7b 3541if (!continue_transport) continue_sequence = 1;
059ec3d9
PH
3542}
3543
3544
3545
3546/*************************************************
3547* Wait for one remote delivery subprocess *
3548*************************************************/
3549
3550/* This function is called while doing remote deliveries when either the
3551maximum number of processes exist and we need one to complete so that another
3552can be created, or when waiting for the last ones to complete. It must wait for
3553the completion of one subprocess, empty the control block slot, and return a
3554pointer to the address chain.
3555
3556Arguments: none
3557Returns: pointer to the chain of addresses handled by the process;
3558 NULL if no subprocess found - this is an unexpected error
3559*/
3560
3561static address_item *
3562par_wait(void)
3563{
3564int poffset, status;
3565address_item *addr, *addrlist;
3566pid_t pid;
3567
3568set_process_info("delivering %s: waiting for a remote delivery subprocess "
3569 "to finish", message_id);
3570
3571/* Loop until either a subprocess completes, or there are no subprocesses in
3572existence - in which case give an error return. We cannot proceed just by
3573waiting for a completion, because a subprocess may have filled up its pipe, and
3574be waiting for it to be emptied. Therefore, if no processes have finished, we
3575wait for one of the pipes to acquire some data by calling select(), with a
3576timeout just in case.
3577
3578The simple approach is just to iterate after reading data from a ready pipe.
3579This leads to non-ideal behaviour when the subprocess has written its final Z
3580item, closed the pipe, and is in the process of exiting (the common case). A
3581call to waitpid() yields nothing completed, but select() shows the pipe ready -
3582reading it yields EOF, so you end up with busy-waiting until the subprocess has
3583actually finished.
3584
3585To avoid this, if all the data that is needed has been read from a subprocess
3586after select(), an explicit wait() for it is done. We know that all it is doing
3587is writing to the pipe and then exiting, so the wait should not be long.
3588
3589The non-blocking waitpid() is to some extent just insurance; if we could
3590reliably detect end-of-file on the pipe, we could always know when to do a
3591blocking wait() for a completed process. However, because some systems use
3592NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3593use code that functions without the need to recognize EOF.
3594
3595There's a double loop here just in case we end up with a process that is not in
3596the list of remote delivery processes. Something has obviously gone wrong if
3597this is the case. (For example, a process that is incorrectly left over from
3598routing or local deliveries might be found.) The damage can be minimized by
3599looping back and looking for another process. If there aren't any, the error
3600return will happen. */
3601
3602for (;;) /* Normally we do not repeat this loop */
3603 {
3604 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3605 {
3606 struct timeval tv;
3607 fd_set select_pipes;
3608 int maxpipe, readycount;
3609
3610 /* A return value of -1 can mean several things. If errno != ECHILD, it
3611 either means invalid options (which we discount), or that this process was
3612 interrupted by a signal. Just loop to try the waitpid() again.
3613
3614 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3615 in existence. This should never happen, and is an unexpected error.
3616 However, there is a nasty complication when running under Linux. If "strace
3617 -f" is being used under Linux to trace this process and its children,
3618 subprocesses are "stolen" from their parents and become the children of the
3619 tracing process. A general wait such as the one we've just obeyed returns
3620 as if there are no children while subprocesses are running. Once a
3621 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3622 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3623 palliative.
3624
3625 This does not happen using "truss" on Solaris, nor (I think) with other
3626 tracing facilities on other OS. It seems to be specific to Linux.
3627
3628 What we do to get round this is to use kill() to see if any of our
3629 subprocesses are still in existence. If kill() gives an OK return, we know
3630 it must be for one of our processes - it can't be for a re-use of the pid,
3631 because if our process had finished, waitpid() would have found it. If any
3632 of our subprocesses are in existence, we proceed to use select() as if
3633 waitpid() had returned zero. I think this is safe. */
3634
3635 if (pid < 0)
3636 {
3637 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3638
3639 DEBUG(D_deliver)
3640 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3641 "for process existence\n");
3642
3643 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3644 {
3645 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3646 {
3647 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3648 "stolen by strace\n", (int)pid);
3649 break; /* With poffset set */