Logging: support +outgoing_port on defer lines and per-host error lines
[exim.git] / src / src / deliver.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
3386088d 5/* Copyright (c) University of Cambridge 1995 - 2015 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* The main code for delivering a message. */
9
10
11#include "exim.h"
dadff1d4 12#include <assert.h>
059ec3d9
PH
13
14
15/* Data block for keeping track of subprocesses for parallel remote
16delivery. */
17
18typedef struct pardata {
19 address_item *addrlist; /* chain of addresses */
20 address_item *addr; /* next address data expected for */
21 pid_t pid; /* subprocess pid */
22 int fd; /* pipe fd for getting result from subprocess */
23 int transport_count; /* returned transport count value */
24 BOOL done; /* no more data needed */
25 uschar *msg; /* error message */
26 uschar *return_path; /* return_path for these addresses */
27} pardata;
28
29/* Values for the process_recipients variable */
30
31enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
32 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
33 RECIP_FAIL_LOOP};
34
35/* Mutually recursive functions for marking addresses done. */
36
37static void child_done(address_item *, uschar *);
38static void address_done(address_item *, uschar *);
39
40/* Table for turning base-62 numbers into binary */
41
42static uschar tab62[] =
43 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
44 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
45 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
46 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
47 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
48 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
49 59,60,61}; /* x-z */
50
51
52/*************************************************
53* Local static variables *
54*************************************************/
55
56/* addr_duplicate is global because it needs to be seen from the Envelope-To
57writing code. */
58
59static address_item *addr_defer = NULL;
60static address_item *addr_failed = NULL;
61static address_item *addr_fallback = NULL;
62static address_item *addr_local = NULL;
63static address_item *addr_new = NULL;
64static address_item *addr_remote = NULL;
65static address_item *addr_route = NULL;
66static address_item *addr_succeed = NULL;
6c1c3d1d
WB
67static address_item *addr_dsntmp = NULL;
68static address_item *addr_senddsn = NULL;
059ec3d9
PH
69
70static FILE *message_log = NULL;
71static BOOL update_spool;
72static BOOL remove_journal;
73static int parcount = 0;
74static pardata *parlist = NULL;
75static int return_count;
76static uschar *frozen_info = US"";
77static uschar *used_return_path = NULL;
78
79static uschar spoolname[PATH_MAX];
80
81
82
83/*************************************************
84* Make a new address item *
85*************************************************/
86
87/* This function gets the store and initializes with default values. The
88transport_return value defaults to DEFER, so that any unexpected failure to
89deliver does not wipe out the message. The default unique string is set to a
90copy of the address, so that its domain can be lowercased.
91
92Argument:
93 address the RFC822 address string
94 copy force a copy of the address
95
96Returns: a pointer to an initialized address_item
97*/
98
99address_item *
100deliver_make_addr(uschar *address, BOOL copy)
101{
102address_item *addr = store_get(sizeof(address_item));
103*addr = address_defaults;
104if (copy) address = string_copy(address);
105addr->address = address;
106addr->unique = string_copy(address);
107return addr;
108}
109
110
111
112
113/*************************************************
114* Set expansion values for an address *
115*************************************************/
116
117/* Certain expansion variables are valid only when handling an address or
118address list. This function sets them up or clears the values, according to its
119argument.
120
121Arguments:
122 addr the address in question, or NULL to clear values
123Returns: nothing
124*/
125
126void
127deliver_set_expansions(address_item *addr)
128{
d6c82d7b 129if (!addr)
059ec3d9 130 {
55414b25 131 const uschar ***p = address_expansions;
d6c82d7b 132 while (*p) **p++ = NULL;
059ec3d9
PH
133 return;
134 }
135
136/* Exactly what gets set depends on whether there is one or more addresses, and
137what they contain. These first ones are always set, taking their values from
138the first address. */
139
d6c82d7b 140if (!addr->host_list)
059ec3d9
PH
141 {
142 deliver_host = deliver_host_address = US"";
a7538db1 143 deliver_host_port = 0;
059ec3d9
PH
144 }
145else
146 {
147 deliver_host = addr->host_list->name;
148 deliver_host_address = addr->host_list->address;
a7538db1 149 deliver_host_port = addr->host_list->port;
059ec3d9
PH
150 }
151
152deliver_recipients = addr;
d43cbe25
JH
153deliver_address_data = addr->prop.address_data;
154deliver_domain_data = addr->prop.domain_data;
155deliver_localpart_data = addr->prop.localpart_data;
059ec3d9
PH
156
157/* These may be unset for multiple addresses */
158
159deliver_domain = addr->domain;
160self_hostname = addr->self_hostname;
161
8523533c
TK
162#ifdef EXPERIMENTAL_BRIGHTMAIL
163bmi_deliver = 1; /* deliver by default */
164bmi_alt_location = NULL;
165bmi_base64_verdict = NULL;
166bmi_base64_tracker_verdict = NULL;
167#endif
168
059ec3d9
PH
169/* If there's only one address we can set everything. */
170
d6c82d7b 171if (!addr->next)
059ec3d9
PH
172 {
173 address_item *addr_orig;
174
175 deliver_localpart = addr->local_part;
176 deliver_localpart_prefix = addr->prefix;
177 deliver_localpart_suffix = addr->suffix;
178
d6c82d7b 179 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
059ec3d9
PH
180 deliver_domain_orig = addr_orig->domain;
181
182 /* Re-instate any prefix and suffix in the original local part. In all
183 normal cases, the address will have a router associated with it, and we can
184 choose the caseful or caseless version accordingly. However, when a system
185 filter sets up a pipe, file, or autoreply delivery, no router is involved.
186 In this case, though, there won't be any prefix or suffix to worry about. */
187
d6c82d7b
JH
188 deliver_localpart_orig = !addr_orig->router
189 ? addr_orig->local_part
190 : addr_orig->router->caseful_local_part
191 ? addr_orig->cc_local_part
192 : addr_orig->lc_local_part;
059ec3d9
PH
193
194 /* If there's a parent, make its domain and local part available, and if
195 delivering to a pipe or file, or sending an autoreply, get the local
196 part from the parent. For pipes and files, put the pipe or file string
197 into address_pipe and address_file. */
198
d6c82d7b 199 if (addr->parent)
059ec3d9
PH
200 {
201 deliver_domain_parent = addr->parent->domain;
d6c82d7b
JH
202 deliver_localpart_parent = !addr->parent->router
203 ? addr->parent->local_part
204 : addr->parent->router->caseful_local_part
205 ? addr->parent->cc_local_part
206 : addr->parent->lc_local_part;
059ec3d9
PH
207
208 /* File deliveries have their own flag because they need to be picked out
209 as special more often. */
210
211 if (testflag(addr, af_pfr))
212 {
d6c82d7b
JH
213 if (testflag(addr, af_file)) address_file = addr->local_part;
214 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
059ec3d9
PH
215 deliver_localpart = addr->parent->local_part;
216 deliver_localpart_prefix = addr->parent->prefix;
217 deliver_localpart_suffix = addr->parent->suffix;
218 }
219 }
8523533c
TK
220
221#ifdef EXPERIMENTAL_BRIGHTMAIL
222 /* Set expansion variables related to Brightmail AntiSpam */
223 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
224 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
225 /* get message delivery status (0 - don't deliver | 1 - deliver) */
226 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
227 /* if message is to be delivered, get eventual alternate location */
d6c82d7b 228 if (bmi_deliver == 1)
8523533c 229 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
8523533c
TK
230#endif
231
059ec3d9
PH
232 }
233
234/* For multiple addresses, don't set local part, and leave the domain and
f7fd3850
PH
235self_hostname set only if it is the same for all of them. It is possible to
236have multiple pipe and file addresses, but only when all addresses have routed
237to the same pipe or file. */
059ec3d9
PH
238
239else
240 {
241 address_item *addr2;
f7fd3850
PH
242 if (testflag(addr, af_pfr))
243 {
d6c82d7b
JH
244 if (testflag(addr, af_file)) address_file = addr->local_part;
245 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
f7fd3850 246 }
d6c82d7b 247 for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 248 {
d6c82d7b 249 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
059ec3d9 250 deliver_domain = NULL;
d6c82d7b
JH
251 if ( self_hostname
252 && ( !addr2->self_hostname
253 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
254 ) )
059ec3d9 255 self_hostname = NULL;
d6c82d7b 256 if (!deliver_domain && !self_hostname) break;
059ec3d9
PH
257 }
258 }
259}
260
261
262
263
264/*************************************************
265* Open a msglog file *
266*************************************************/
267
268/* This function is used both for normal message logs, and for files in the
269msglog directory that are used to catch output from pipes. Try to create the
270directory if it does not exist. From release 4.21, normal message logs should
271be created when the message is received.
272
273Argument:
274 filename the file name
275 mode the mode required
276 error used for saying what failed
277
278Returns: a file descriptor, or -1 (with errno set)
279*/
280
281static int
282open_msglog_file(uschar *filename, int mode, uschar **error)
283{
284int fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
285
286if (fd < 0 && errno == ENOENT)
287 {
288 uschar temp[16];
289 sprintf(CS temp, "msglog/%s", message_subdir);
290 if (message_subdir[0] == 0) temp[6] = 0;
291 (void)directory_make(spool_directory, temp, MSGLOG_DIRECTORY_MODE, TRUE);
292 fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
293 }
294
295/* Set the close-on-exec flag and change the owner to the exim uid/gid (this
296function is called as root). Double check the mode, because the group setting
297doesn't always get set automatically. */
298
299if (fd >= 0)
300 {
ff790e47 301 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
059ec3d9
PH
302 if (fchown(fd, exim_uid, exim_gid) < 0)
303 {
304 *error = US"chown";
305 return -1;
306 }
307 if (fchmod(fd, mode) < 0)
308 {
309 *error = US"chmod";
310 return -1;
311 }
312 }
313else *error = US"create";
314
315return fd;
316}
317
318
319
320
321/*************************************************
322* Write to msglog if required *
323*************************************************/
324
325/* Write to the message log, if configured. This function may also be called
326from transports.
327
328Arguments:
329 format a string format
330
331Returns: nothing
332*/
333
334void
335deliver_msglog(const char *format, ...)
336{
337va_list ap;
338if (!message_logs) return;
339va_start(ap, format);
340vfprintf(message_log, format, ap);
341fflush(message_log);
342va_end(ap);
343}
344
345
346
347
348/*************************************************
349* Replicate status for batch *
350*************************************************/
351
352/* When a transport handles a batch of addresses, it may treat them
353individually, or it may just put the status in the first one, and return FALSE,
354requesting that the status be copied to all the others externally. This is the
355replication function. As well as the status, it copies the transport pointer,
356which may have changed if appendfile passed the addresses on to a different
357transport.
358
359Argument: pointer to the first address in a chain
360Returns: nothing
361*/
362
363static void
364replicate_status(address_item *addr)
365{
366address_item *addr2;
d6c82d7b 367for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 368 {
d6c82d7b 369 addr2->transport = addr->transport;
059ec3d9 370 addr2->transport_return = addr->transport_return;
d6c82d7b
JH
371 addr2->basic_errno = addr->basic_errno;
372 addr2->more_errno = addr->more_errno;
373 addr2->special_action = addr->special_action;
374 addr2->message = addr->message;
375 addr2->user_message = addr->user_message;
059ec3d9
PH
376 }
377}
378
379
380
381/*************************************************
382* Compare lists of hosts *
383*************************************************/
384
385/* This function is given two pointers to chains of host items, and it yields
386TRUE if the lists refer to the same hosts in the same order, except that
387
388(1) Multiple hosts with the same non-negative MX values are permitted to appear
389 in different orders. Round-robinning nameservers can cause this to happen.
390
391(2) Multiple hosts with the same negative MX values less than MX_NONE are also
392 permitted to appear in different orders. This is caused by randomizing
393 hosts lists.
394
395This enables Exim to use a single SMTP transaction for sending to two entirely
396different domains that happen to end up pointing at the same hosts.
397
398Arguments:
399 one points to the first host list
400 two points to the second host list
401
402Returns: TRUE if the lists refer to the same host set
403*/
404
405static BOOL
406same_hosts(host_item *one, host_item *two)
407{
d6c82d7b 408while (one && two)
059ec3d9
PH
409 {
410 if (Ustrcmp(one->name, two->name) != 0)
411 {
412 int mx = one->mx;
413 host_item *end_one = one;
414 host_item *end_two = two;
415
416 /* Batch up only if there was no MX and the list was not randomized */
417
418 if (mx == MX_NONE) return FALSE;
419
420 /* Find the ends of the shortest sequence of identical MX values */
421
d6c82d7b
JH
422 while ( end_one->next && end_one->next->mx == mx
423 && end_two->next && end_two->next->mx == mx)
059ec3d9
PH
424 {
425 end_one = end_one->next;
426 end_two = end_two->next;
427 }
428
429 /* If there aren't any duplicates, there's no match. */
430
431 if (end_one == one) return FALSE;
432
433 /* For each host in the 'one' sequence, check that it appears in the 'two'
434 sequence, returning FALSE if not. */
435
436 for (;;)
437 {
438 host_item *hi;
439 for (hi = two; hi != end_two->next; hi = hi->next)
440 if (Ustrcmp(one->name, hi->name) == 0) break;
441 if (hi == end_two->next) return FALSE;
442 if (one == end_one) break;
443 one = one->next;
444 }
445
446 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
447 Ensure both are pointing at the last host, and carry on as for equality. */
448
449 two = end_two;
450 }
451
452 /* Hosts matched */
453
454 one = one->next;
455 two = two->next;
456 }
457
458/* True if both are NULL */
459
460return (one == two);
461}
462
463
464
465/*************************************************
466* Compare header lines *
467*************************************************/
468
469/* This function is given two pointers to chains of header items, and it yields
470TRUE if they are the same header texts in the same order.
471
472Arguments:
473 one points to the first header list
474 two points to the second header list
475
476Returns: TRUE if the lists refer to the same header set
477*/
478
479static BOOL
480same_headers(header_line *one, header_line *two)
481{
d6c82d7b 482for (;; one = one->next, two = two->next)
059ec3d9
PH
483 {
484 if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 485 if (!one || !two) return FALSE;
059ec3d9 486 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
059ec3d9
PH
487 }
488}
489
490
491
492/*************************************************
493* Compare string settings *
494*************************************************/
495
496/* This function is given two pointers to strings, and it returns
497TRUE if they are the same pointer, or if the two strings are the same.
498
499Arguments:
500 one points to the first string
501 two points to the second string
502
503Returns: TRUE or FALSE
504*/
505
506static BOOL
507same_strings(uschar *one, uschar *two)
508{
509if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 510if (!one || !two) return FALSE;
059ec3d9
PH
511return (Ustrcmp(one, two) == 0);
512}
513
514
515
516/*************************************************
517* Compare uid/gid for addresses *
518*************************************************/
519
520/* This function is given a transport and two addresses. It yields TRUE if the
521uid/gid/initgroups settings for the two addresses are going to be the same when
522they are delivered.
523
524Arguments:
525 tp the transort
526 addr1 the first address
527 addr2 the second address
528
529Returns: TRUE or FALSE
530*/
531
532static BOOL
533same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
534{
d6c82d7b
JH
535if ( !tp->uid_set && !tp->expand_uid
536 && !tp->deliver_as_creator
537 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
538 || ( testflag(addr1, af_uid_set)
539 && ( addr1->uid != addr2->uid
540 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
541 ) ) ) )
542 return FALSE;
059ec3d9 543
d6c82d7b
JH
544if ( !tp->gid_set && !tp->expand_gid
545 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
546 || ( testflag(addr1, af_gid_set)
547 && addr1->gid != addr2->gid
548 ) ) )
549 return FALSE;
059ec3d9
PH
550
551return TRUE;
552}
553
554
555
556
557/*************************************************
558* Record that an address is complete *
559*************************************************/
560
561/* This function records that an address is complete. This is straightforward
562for most addresses, where the unique address is just the full address with the
563domain lower cased. For homonyms (addresses that are the same as one of their
564ancestors) their are complications. Their unique addresses have \x\ prepended
565(where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
566cousins.
567
568Exim used to record the unique addresses of homonyms as "complete". This,
569however, fails when the pattern of redirection varies over time (e.g. if taking
570unseen copies at only some times of day) because the prepended numbers may vary
571from one delivery run to the next. This problem is solved by never recording
572prepended unique addresses as complete. Instead, when a homonymic address has
573actually been delivered via a transport, we record its basic unique address
574followed by the name of the transport. This is checked in subsequent delivery
575runs whenever an address is routed to a transport.
576
577If the completed address is a top-level one (has no parent, which means it
578cannot be homonymic) we also add the original address to the non-recipients
579tree, so that it gets recorded in the spool file and therefore appears as
580"done" in any spool listings. The original address may differ from the unique
581address in the case of the domain.
582
583Finally, this function scans the list of duplicates, marks as done any that
584match this address, and calls child_done() for their ancestors.
585
586Arguments:
587 addr address item that has been completed
588 now current time as a string
589
590Returns: nothing
591*/
592
593static void
594address_done(address_item *addr, uschar *now)
595{
596address_item *dup;
597
598update_spool = TRUE; /* Ensure spool gets updated */
599
600/* Top-level address */
601
d6c82d7b 602if (!addr->parent)
059ec3d9
PH
603 {
604 tree_add_nonrecipient(addr->unique);
605 tree_add_nonrecipient(addr->address);
606 }
607
608/* Homonymous child address */
609
610else if (testflag(addr, af_homonym))
611 {
d6c82d7b 612 if (addr->transport)
059ec3d9
PH
613 tree_add_nonrecipient(
614 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
059ec3d9
PH
615 }
616
617/* Non-homonymous child address */
618
619else tree_add_nonrecipient(addr->unique);
620
621/* Check the list of duplicate addresses and ensure they are now marked
622done as well. */
623
d6c82d7b 624for (dup = addr_duplicate; dup; dup = dup->next)
059ec3d9
PH
625 if (Ustrcmp(addr->unique, dup->unique) == 0)
626 {
57730b52 627 tree_add_nonrecipient(dup->unique);
059ec3d9
PH
628 child_done(dup, now);
629 }
059ec3d9
PH
630}
631
632
633
634
635/*************************************************
636* Decrease counts in parents and mark done *
637*************************************************/
638
639/* This function is called when an address is complete. If there is a parent
640address, its count of children is decremented. If there are still other
641children outstanding, the function exits. Otherwise, if the count has become
642zero, address_done() is called to mark the parent and its duplicates complete.
643Then loop for any earlier ancestors.
644
645Arguments:
646 addr points to the completed address item
647 now the current time as a string, for writing to the message log
648
649Returns: nothing
650*/
651
652static void
653child_done(address_item *addr, uschar *now)
654{
655address_item *aa;
d6c82d7b 656while (addr->parent)
059ec3d9
PH
657 {
658 addr = addr->parent;
659 if ((addr->child_count -= 1) > 0) return; /* Incomplete parent */
660 address_done(addr, now);
661
662 /* Log the completion of all descendents only when there is no ancestor with
663 the same original address. */
664
d6c82d7b 665 for (aa = addr->parent; aa; aa = aa->parent)
059ec3d9 666 if (Ustrcmp(aa->address, addr->address) == 0) break;
d6c82d7b 667 if (aa) continue;
059ec3d9
PH
668
669 deliver_msglog("%s %s: children all complete\n", now, addr->address);
670 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
671 }
672}
673
674
675
ac881e27
TF
676/*************************************************
677* Delivery logging support functions *
678*************************************************/
679
680/* The LOGGING() checks in d_log_interface() are complicated for backwards
681compatibility. When outgoing interface logging was originally added, it was
682conditional on just incoming_interface (which is off by default). The
683outgoing_interface option is on by default to preserve this behaviour, but
684you can enable incoming_interface and disable outgoing_interface to get I=
685fields on incoming lines only.
686
687Arguments:
688 s The log line buffer
689 sizep Pointer to the buffer size
690 ptrp Pointer to current index into buffer
691 addr The address to be logged
692
693Returns: New value for s
694*/
059ec3d9 695
67d81c10 696static uschar *
ac881e27 697d_log_interface(uschar *s, int *sizep, int *ptrp)
67d81c10 698{
ac881e27 699if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
d6c82d7b 700 && sending_ip_address)
ac881e27
TF
701 {
702 s = string_append(s, sizep, ptrp, 2, US" I=[", sending_ip_address);
d6c82d7b
JH
703 s = LOGGING(outgoing_port)
704 ? string_append(s, sizep, ptrp, 2, US"]:",
705 string_sprintf("%d", sending_port))
706 : string_cat(s, sizep, ptrp, "]", 1);
ac881e27
TF
707 }
708return s;
67d81c10
JH
709}
710
ac881e27
TF
711
712
713static uschar *
714d_hostlog(uschar *s, int *sizep, int *ptrp, address_item *addr)
715{
716s = string_append(s, sizep, ptrp, 5, US" H=", addr->host_used->name,
717 US" [", addr->host_used->address, US"]");
718if (LOGGING(outgoing_port))
719 s = string_append(s, sizep, ptrp, 2, US":", string_sprintf("%d",
720 addr->host_used->port));
721return d_log_interface(s, sizep, ptrp);
722}
723
724
725
67d81c10
JH
726#ifdef SUPPORT_TLS
727static uschar *
728d_tlslog(uschar * s, int * sizep, int * ptrp, address_item * addr)
729{
d6c82d7b 730if (LOGGING(tls_cipher) && addr->cipher)
ac881e27 731 s = string_append(s, sizep, ptrp, 2, US" X=", addr->cipher);
d6c82d7b 732if (LOGGING(tls_certificate_verified) && addr->cipher)
ac881e27
TF
733 s = string_append(s, sizep, ptrp, 2, US" CV=",
734 testflag(addr, af_cert_verified)
735 ?
53a7196b 736#ifdef EXPERIMENTAL_DANE
ac881e27
TF
737 testflag(addr, af_dane_verified)
738 ? "dane"
739 :
53a7196b 740#endif
ac881e27
TF
741 "yes"
742 : "no");
d6c82d7b 743if (LOGGING(tls_peerdn) && addr->peerdn)
ac881e27
TF
744 s = string_append(s, sizep, ptrp, 3, US" DN=\"",
745 string_printing(addr->peerdn), US"\"");
746return s;
67d81c10
JH
747}
748#endif
749
a7538db1 750
14a465c3
JH
751
752
774ef2d7 753#ifdef EXPERIMENTAL_EVENT
b30275b8 754uschar *
55414b25 755event_raise(uschar * action, const uschar * event, uschar * ev_data)
a7538db1
JH
756{
757uschar * s;
758if (action)
759 {
760 DEBUG(D_deliver)
774ef2d7 761 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
a7538db1
JH
762 event,
763 action, deliver_host_address);
764
774ef2d7
JH
765 event_name = event;
766 event_data = ev_data;
a7538db1
JH
767
768 if (!(s = expand_string(action)) && *expand_string_message)
769 log_write(0, LOG_MAIN|LOG_PANIC,
774ef2d7 770 "failed to expand event_action %s in %s: %s\n",
a7538db1
JH
771 event, transport_name, expand_string_message);
772
774ef2d7 773 event_name = event_data = NULL;
a7538db1
JH
774
775 /* If the expansion returns anything but an empty string, flag for
776 the caller to modify his normal processing
777 */
778 if (s && *s)
779 {
780 DEBUG(D_deliver)
774ef2d7 781 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
b30275b8 782 return s;
a7538db1
JH
783 }
784 }
b30275b8 785return NULL;
a7538db1 786}
14a465c3
JH
787
788static void
55414b25 789msg_event_raise(const uschar * event, const address_item * addr)
14a465c3 790{
55414b25 791const uschar * save_domain = deliver_domain;
14a465c3 792uschar * save_local = deliver_localpart;
55414b25 793const uschar * save_host = deliver_host;
14a465c3
JH
794
795if (!addr->transport)
796 return;
797
798router_name = addr->router ? addr->router->name : NULL;
799transport_name = addr->transport->name;
800deliver_domain = addr->domain;
801deliver_localpart = addr->local_part;
c562fd30 802deliver_host = addr->host_used ? addr->host_used->name : NULL;
14a465c3 803
774ef2d7 804(void) event_raise(addr->transport->event_action, event,
14a465c3
JH
805 addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
806 ? addr->message : NULL);
807
c562fd30 808deliver_host = save_host;
14a465c3
JH
809deliver_localpart = save_local;
810deliver_domain = save_domain;
811router_name = transport_name = NULL;
812}
774ef2d7 813#endif /*EXPERIMENTAL_EVENT*/
14a465c3
JH
814
815
a7538db1 816
817d9f57 817/* If msg is NULL this is a delivery log and logchar is used. Otherwise
67d81c10 818this is a nonstandard call; no two-character delivery flag is written
817d9f57
JH
819but sender-host and sender are prefixed and "msg" is inserted in the log line.
820
821Arguments:
822 flags passed to log_write()
823*/
e4bdf652 824void
817d9f57 825delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
e4bdf652
JH
826{
827uschar *log_address;
828int size = 256; /* Used for a temporary, */
829int ptr = 0; /* expanding buffer, for */
830uschar *s; /* building log lines; */
831void *reset_point; /* released afterwards. */
832
e4bdf652
JH
833/* Log the delivery on the main log. We use an extensible string to build up
834the log line, and reset the store afterwards. Remote deliveries should always
835have a pointer to the host item that succeeded; local deliveries can have a
836pointer to a single host item in their host list, for use by the transport. */
837
774ef2d7 838#ifdef EXPERIMENTAL_EVENT
a7538db1 839 /* presume no successful remote delivery */
783b385f 840 lookup_dnssec_authenticated = NULL;
d68218c7
JH
841#endif
842
e4bdf652 843s = reset_point = store_get(size);
e4bdf652 844
6c6d6e48 845log_address = string_log_address(addr, LOGGING(all_parents), TRUE);
817d9f57
JH
846if (msg)
847 s = string_append(s, &size, &ptr, 3, host_and_ident(TRUE), US" ", log_address);
848else
849 {
850 s[ptr++] = logchar;
851 s = string_append(s, &size, &ptr, 2, US"> ", log_address);
852 }
e4bdf652 853
6c6d6e48 854if (LOGGING(sender_on_delivery) || msg)
3c8b3577
JH
855 s = string_append(s, &size, &ptr, 3, US" F=<",
856#ifdef EXPERIMENTAL_INTERNATIONAL
857 testflag(addr, af_utf8_downcvt)
858 ? string_address_utf8_to_alabel(sender_address, NULL)
859 :
860#endif
861 sender_address,
862 US">");
e4bdf652
JH
863
864#ifdef EXPERIMENTAL_SRS
d43cbe25
JH
865if(addr->prop.srs_sender)
866 s = string_append(s, &size, &ptr, 3, US" SRS=<", addr->prop.srs_sender, US">");
e4bdf652
JH
867#endif
868
869/* You might think that the return path must always be set for a successful
870delivery; indeed, I did for some time, until this statement crashed. The case
871when it is not set is for a delivery to /dev/null which is optimised by not
872being run at all. */
873
d6c82d7b 874if (used_return_path && LOGGING(return_path_on_delivery))
e4bdf652
JH
875 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
876
817d9f57
JH
877if (msg)
878 s = string_append(s, &size, &ptr, 2, US" ", msg);
e4bdf652 879
817d9f57 880/* For a delivery from a system filter, there may not be a router */
d6c82d7b 881if (addr->router)
e4bdf652
JH
882 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
883
884s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
885
6c6d6e48 886if (LOGGING(delivery_size))
e4bdf652
JH
887 s = string_append(s, &size, &ptr, 2, US" S=",
888 string_sprintf("%d", transport_count));
889
890/* Local delivery */
891
892if (addr->transport->info->local)
893 {
a7538db1 894 if (addr->host_list)
e4bdf652 895 s = string_append(s, &size, &ptr, 2, US" H=", addr->host_list->name);
ac881e27 896 s = d_log_interface(s, &size, &ptr);
d6c82d7b 897 if (addr->shadow_message)
e4bdf652
JH
898 s = string_cat(s, &size, &ptr, addr->shadow_message,
899 Ustrlen(addr->shadow_message));
900 }
901
902/* Remote delivery */
903
904else
905 {
783b385f 906 if (addr->host_used)
e4bdf652 907 {
67d81c10 908 s = d_hostlog(s, &size, &ptr, addr);
e4bdf652
JH
909 if (continue_sequence > 1)
910 s = string_cat(s, &size, &ptr, US"*", 1);
d68218c7 911
774ef2d7 912#ifdef EXPERIMENTAL_EVENT
a7538db1
JH
913 deliver_host_address = addr->host_used->address;
914 deliver_host_port = addr->host_used->port;
029f4192 915 deliver_host = addr->host_used->name;
783b385f
JH
916
917 /* DNS lookup status */
918 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
919 : addr->host_used->dnssec==DS_NO ? US"no"
920 : NULL;
a7538db1 921#endif
e4bdf652
JH
922 }
923
a7538db1 924#ifdef SUPPORT_TLS
67d81c10 925 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 926#endif
e4bdf652 927
018c60d7 928 if (addr->authenticator)
6f123593 929 {
018c60d7
JH
930 s = string_append(s, &size, &ptr, 2, US" A=", addr->authenticator);
931 if (addr->auth_id)
c8e2fc1e 932 {
018c60d7 933 s = string_append(s, &size, &ptr, 2, US":", addr->auth_id);
6c6d6e48 934 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
018c60d7 935 s = string_append(s, &size, &ptr, 2, US":", addr->auth_sndr);
c8e2fc1e 936 }
6f123593
JH
937 }
938
a7538db1 939#ifndef DISABLE_PRDR
fd98a5c6
JH
940 if (addr->flags & af_prdr_used)
941 s = string_append(s, &size, &ptr, 1, US" PRDR");
a7538db1 942#endif
76f44207
WB
943 }
944
945/* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
fd98a5c6 946
d6c82d7b
JH
947if ( LOGGING(smtp_confirmation)
948 && addr->message
949 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
950 )
76f44207 951 {
b980ed83 952 unsigned i;
1c5b1499 953 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
76f44207
WB
954 uschar *p = big_buffer;
955 uschar *ss = addr->message;
956 *p++ = '\"';
b980ed83 957 for (i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
e4bdf652 958 {
27520a68 959 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
76f44207 960 *p++ = ss[i];
e4bdf652 961 }
76f44207
WB
962 *p++ = '\"';
963 *p = 0;
964 s = string_append(s, &size, &ptr, 2, US" C=", big_buffer);
e4bdf652
JH
965 }
966
967/* Time on queue and actual time taken to deliver */
968
6c6d6e48 969if (LOGGING(queue_time))
e4bdf652 970 s = string_append(s, &size, &ptr, 2, US" QT=",
19050083 971 readconf_printtime( (int) ((long)time(NULL) - (long)received_time)) );
e4bdf652 972
6c6d6e48 973if (LOGGING(deliver_time))
e4bdf652
JH
974 s = string_append(s, &size, &ptr, 2, US" DT=",
975 readconf_printtime(addr->more_errno));
e4bdf652
JH
976
977/* string_cat() always leaves room for the terminator. Release the
978store we used to build the line after writing it. */
979
980s[ptr] = 0;
817d9f57 981log_write(0, flags, "%s", s);
d68218c7 982
774ef2d7
JH
983#ifdef EXPERIMENTAL_EVENT
984if (!msg) msg_event_raise(US"msg:delivery", addr);
d68218c7 985#endif
14a465c3 986
e4bdf652
JH
987store_reset(reset_point);
988return;
989}
990
991
992
059ec3d9
PH
993/*************************************************
994* Actions at the end of handling an address *
995*************************************************/
996
997/* This is a function for processing a single address when all that can be done
998with it has been done.
999
1000Arguments:
1001 addr points to the address block
1002 result the result of the delivery attempt
1003 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1004 driver_type indicates which type of driver (transport, or router) was last
1005 to process the address
1006 logchar '=' or '-' for use when logging deliveries with => or ->
1007
1008Returns: nothing
1009*/
1010
1011static void
1012post_process_one(address_item *addr, int result, int logflags, int driver_type,
1013 int logchar)
1014{
1015uschar *now = tod_stamp(tod_log);
1016uschar *driver_kind = NULL;
1017uschar *driver_name = NULL;
1018uschar *log_address;
1019
1020int size = 256; /* Used for a temporary, */
1021int ptr = 0; /* expanding buffer, for */
1022uschar *s; /* building log lines; */
1023void *reset_point; /* released afterwards. */
1024
059ec3d9
PH
1025DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1026
1027/* Set up driver kind and name for logging. Disable logging if the router or
1028transport has disabled it. */
1029
1030if (driver_type == DTYPE_TRANSPORT)
1031 {
d6c82d7b 1032 if (addr->transport)
059ec3d9
PH
1033 {
1034 driver_name = addr->transport->name;
1035 driver_kind = US" transport";
1036 disable_logging = addr->transport->disable_logging;
1037 }
1038 else driver_kind = US"transporting";
1039 }
1040else if (driver_type == DTYPE_ROUTER)
1041 {
d6c82d7b 1042 if (addr->router)
059ec3d9
PH
1043 {
1044 driver_name = addr->router->name;
1045 driver_kind = US" router";
1046 disable_logging = addr->router->disable_logging;
1047 }
1048 else driver_kind = US"routing";
1049 }
1050
1051/* If there's an error message set, ensure that it contains only printing
1052characters - it should, but occasionally things slip in and this at least
49c2d5ea
PH
1053stops the log format from getting wrecked. We also scan the message for an LDAP
1054expansion item that has a password setting, and flatten the password. This is a
1055fudge, but I don't know a cleaner way of doing this. (If the item is badly
1056malformed, it won't ever have gone near LDAP.) */
059ec3d9 1057
d6c82d7b 1058if (addr->message)
49c2d5ea 1059 {
55414b25
JH
1060 const uschar * s = string_printing(addr->message);
1061 if (s != addr->message)
1062 addr->message = US s;
1063 /* deconst cast ok as string_printing known to have alloc'n'copied */
d6c82d7b
JH
1064 if ( ( Ustrstr(s, "failed to expand") != NULL
1065 || Ustrstr(s, "expansion of ") != NULL
1066 )
1067 && ( Ustrstr(s, "mysql") != NULL
1068 || Ustrstr(s, "pgsql") != NULL
9bdd29ad 1069#ifdef EXPERIMENTAL_REDIS
d6c82d7b 1070 || Ustrstr(s, "redis") != NULL
9bdd29ad 1071#endif
d6c82d7b
JH
1072 || Ustrstr(s, "sqlite") != NULL
1073 || Ustrstr(s, "ldap:") != NULL
1074 || Ustrstr(s, "ldapdn:") != NULL
1075 || Ustrstr(s, "ldapm:") != NULL
1076 ) )
1077 addr->message = string_sprintf("Temporary internal error");
49c2d5ea 1078 }
059ec3d9
PH
1079
1080/* If we used a transport that has one of the "return_output" options set, and
1081if it did in fact generate some output, then for return_output we treat the
1082message as failed if it was not already set that way, so that the output gets
1083returned to the sender, provided there is a sender to send it to. For
1084return_fail_output, do this only if the delivery failed. Otherwise we just
1085unlink the file, and remove the name so that if the delivery failed, we don't
1086try to send back an empty or unwanted file. The log_output options operate only
1087on a non-empty file.
1088
1089In any case, we close the message file, because we cannot afford to leave a
1090file-descriptor for one address while processing (maybe very many) others. */
1091
d6c82d7b 1092if (addr->return_file >= 0 && addr->return_filename)
059ec3d9
PH
1093 {
1094 BOOL return_output = FALSE;
1095 struct stat statbuf;
54fc8428 1096 (void)EXIMfsync(addr->return_file);
059ec3d9
PH
1097
1098 /* If there is no output, do nothing. */
1099
1100 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1101 {
1102 transport_instance *tb = addr->transport;
1103
1104 /* Handle logging options */
1105
d6c82d7b
JH
1106 if ( tb->log_output
1107 || result == FAIL && tb->log_fail_output
1108 || result == DEFER && tb->log_defer_output
1109 )
059ec3d9
PH
1110 {
1111 uschar *s;
1112 FILE *f = Ufopen(addr->return_filename, "rb");
d6c82d7b 1113 if (!f)
059ec3d9
PH
1114 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1115 "from %s transport: %s", addr->return_filename, tb->name,
1116 strerror(errno));
1117 else
d6c82d7b 1118 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
059ec3d9
PH
1119 {
1120 uschar *p = big_buffer + Ustrlen(big_buffer);
55414b25 1121 const uschar * sp;
059ec3d9
PH
1122 while (p > big_buffer && isspace(p[-1])) p--;
1123 *p = 0;
55414b25 1124 sp = string_printing(big_buffer);
059ec3d9 1125 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
55414b25 1126 addr->address, tb->name, sp);
059ec3d9 1127 }
f1e894f3 1128 (void)fclose(f);
059ec3d9
PH
1129 }
1130
1131 /* Handle returning options, but only if there is an address to return
1132 the text to. */
1133
d6c82d7b 1134 if (sender_address[0] != 0 || addr->prop.errors_address)
059ec3d9
PH
1135 if (tb->return_output)
1136 {
1137 addr->transport_return = result = FAIL;
d6c82d7b 1138 if (addr->basic_errno == 0 && !addr->message)
059ec3d9
PH
1139 addr->message = US"return message generated";
1140 return_output = TRUE;
1141 }
1142 else
1143 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
059ec3d9
PH
1144 }
1145
1146 /* Get rid of the file unless it might be returned, but close it in
1147 all cases. */
1148
1149 if (!return_output)
1150 {
1151 Uunlink(addr->return_filename);
1152 addr->return_filename = NULL;
1153 addr->return_file = -1;
1154 }
1155
f1e894f3 1156 (void)close(addr->return_file);
059ec3d9
PH
1157 }
1158
9d1c15ef 1159/* The success case happens only after delivery by a transport. */
059ec3d9
PH
1160
1161if (result == OK)
1162 {
1163 addr->next = addr_succeed;
1164 addr_succeed = addr;
1165
1166 /* Call address_done() to ensure that we don't deliver to this address again,
1167 and write appropriate things to the message log. If it is a child address, we
1168 call child_done() to scan the ancestors and mark them complete if this is the
1169 last child to complete. */
1170
1171 address_done(addr, now);
1172 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1173
d6c82d7b 1174 if (!addr->parent)
059ec3d9
PH
1175 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1176 driver_name, driver_kind);
059ec3d9
PH
1177 else
1178 {
1179 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1180 addr->parent->address, driver_name, driver_kind);
1181 child_done(addr, now);
1182 }
1183
774ef2d7 1184 /* Certificates for logging (via events) */
a7538db1 1185#ifdef SUPPORT_TLS
9d1c15ef
JH
1186 tls_out.ourcert = addr->ourcert;
1187 addr->ourcert = NULL;
1188 tls_out.peercert = addr->peercert;
1189 addr->peercert = NULL;
018058b2 1190
018058b2
JH
1191 tls_out.cipher = addr->cipher;
1192 tls_out.peerdn = addr->peerdn;
1193 tls_out.ocsp = addr->ocsp;
83b27293
JH
1194# ifdef EXPERIMENTAL_DANE
1195 tls_out.dane_verified = testflag(addr, af_dane_verified);
1196# endif
a7538db1 1197#endif
9d1c15ef 1198
817d9f57 1199 delivery_log(LOG_MAIN, addr, logchar, NULL);
9d1c15ef 1200
a7538db1 1201#ifdef SUPPORT_TLS
790fbb71
JH
1202 tls_free_cert(&tls_out.ourcert);
1203 tls_free_cert(&tls_out.peercert);
018058b2
JH
1204 tls_out.cipher = NULL;
1205 tls_out.peerdn = NULL;
1206 tls_out.ocsp = OCSP_NOT_REQ;
83b27293
JH
1207# ifdef EXPERIMENTAL_DANE
1208 tls_out.dane_verified = FALSE;
1209# endif
a7538db1 1210#endif
059ec3d9
PH
1211 }
1212
1213
1214/* Soft failure, or local delivery process failed; freezing may be
1215requested. */
1216
1217else if (result == DEFER || result == PANIC)
1218 {
1219 if (result == PANIC) logflags |= LOG_PANIC;
1220
1221 /* This puts them on the chain in reverse order. Do not change this, because
1222 the code for handling retries assumes that the one with the retry
1223 information is last. */
1224
1225 addr->next = addr_defer;
1226 addr_defer = addr;
1227
1228 /* The only currently implemented special action is to freeze the
1229 message. Logging of this is done later, just before the -H file is
1230 updated. */
1231
1232 if (addr->special_action == SPECIAL_FREEZE)
1233 {
1234 deliver_freeze = TRUE;
1235 deliver_frozen_at = time(NULL);
1236 update_spool = TRUE;
1237 }
1238
1239 /* If doing a 2-stage queue run, we skip writing to either the message
1240 log or the main log for SMTP defers. */
1241
1242 if (!queue_2stage || addr->basic_errno != 0)
1243 {
1244 uschar ss[32];
1245
1246 /* For errors of the type "retry time not reached" (also remotes skipped
1247 on queue run), logging is controlled by L_retry_defer. Note that this kind
1248 of error number is negative, and all the retry ones are less than any
1249 others. */
1250
d6c82d7b
JH
1251 unsigned int use_log_selector = addr->basic_errno <= ERRNO_RETRY_BASE
1252 ? L_retry_defer : 0;
059ec3d9
PH
1253
1254 /* Build up the line that is used for both the message log and the main
1255 log. */
1256
1257 s = reset_point = store_get(size);
e4bdf652
JH
1258
1259 /* Create the address string for logging. Must not do this earlier, because
1260 an OK result may be changed to FAIL when a pipe returns text. */
1261
6c6d6e48 1262 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1263
059ec3d9
PH
1264 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
1265
1266 /* Either driver_name contains something and driver_kind contains
1267 " router" or " transport" (note the leading space), or driver_name is
1268 a null string and driver_kind contains "routing" without the leading
1269 space, if all routing has been deferred. When a domain has been held,
1270 so nothing has been done at all, both variables contain null strings. */
1271
d6c82d7b 1272 if (driver_name)
059ec3d9 1273 {
d6c82d7b 1274 if (driver_kind[1] == 't' && addr->router)
059ec3d9
PH
1275 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
1276 Ustrcpy(ss, " ?=");
1277 ss[1] = toupper(driver_kind[1]);
1278 s = string_append(s, &size, &ptr, 2, ss, driver_name);
1279 }
d6c82d7b
JH
1280 else if (driver_kind)
1281 s = string_append(s, &size, &ptr, 2, US" ", driver_kind);
059ec3d9
PH
1282
1283 sprintf(CS ss, " defer (%d)", addr->basic_errno);
1284 s = string_cat(s, &size, &ptr, ss, Ustrlen(ss));
1285
1286 if (addr->basic_errno > 0)
1287 s = string_append(s, &size, &ptr, 2, US": ",
1288 US strerror(addr->basic_errno));
1289
c562fd30 1290 if (addr->host_used)
1a47b633 1291 {
c562fd30
JH
1292 s = string_append(s, &size, &ptr, 5,
1293 US" H=", addr->host_used->name,
1294 US" [", addr->host_used->address, US"]");
1a47b633
JH
1295 if (LOGGING(outgoing_port))
1296 {
1297 int port = addr->host_used->port;
1298 s = string_append(s, &size, &ptr, 2,
1299 US":", port == PORT_NONE ? US"25" : string_sprintf("%d", port));
1300 }
1301 }
c562fd30 1302
d6c82d7b 1303 if (addr->message)
059ec3d9
PH
1304 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1305
1306 s[ptr] = 0;
1307
1308 /* Log the deferment in the message log, but don't clutter it
1309 up with retry-time defers after the first delivery attempt. */
1310
1311 if (deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1312 deliver_msglog("%s %s\n", now, s);
1313
1314 /* Write the main log and reset the store */
1315
1316 log_write(use_log_selector, logflags, "== %s", s);
1317 store_reset(reset_point);
1318 }
1319 }
1320
1321
1322/* Hard failure. If there is an address to which an error message can be sent,
1323put this address on the failed list. If not, put it on the deferred list and
1324freeze the mail message for human attention. The latter action can also be
1325explicitly requested by a router or transport. */
1326
1327else
1328 {
1329 /* If this is a delivery error, or a message for which no replies are
1330 wanted, and the message's age is greater than ignore_bounce_errors_after,
1331 force the af_ignore_error flag. This will cause the address to be discarded
1332 later (with a log entry). */
1333
1334 if (sender_address[0] == 0 && message_age >= ignore_bounce_errors_after)
1335 setflag(addr, af_ignore_error);
1336
1337 /* Freeze the message if requested, or if this is a bounce message (or other
652e1b65
PH
1338 message with null sender) and this address does not have its own errors
1339 address. However, don't freeze if errors are being ignored. The actual code
1340 to ignore occurs later, instead of sending a message. Logging of freezing
1341 occurs later, just before writing the -H file. */
059ec3d9 1342
d6c82d7b
JH
1343 if ( !testflag(addr, af_ignore_error)
1344 && ( addr->special_action == SPECIAL_FREEZE
1345 || (sender_address[0] == 0 && !addr->prop.errors_address)
1346 ) )
059ec3d9 1347 {
d6c82d7b
JH
1348 frozen_info = addr->special_action == SPECIAL_FREEZE
1349 ? US""
1350 : sender_local && !local_error_message
1351 ? US" (message created with -f <>)"
1352 : US" (delivery error message)";
059ec3d9
PH
1353 deliver_freeze = TRUE;
1354 deliver_frozen_at = time(NULL);
1355 update_spool = TRUE;
1356
1357 /* The address is put on the defer rather than the failed queue, because
1358 the message is being retained. */
1359
1360 addr->next = addr_defer;
1361 addr_defer = addr;
1362 }
1363
1364 /* Don't put the address on the nonrecipients tree yet; wait until an
1365 error message has been successfully sent. */
1366
1367 else
1368 {
1369 addr->next = addr_failed;
1370 addr_failed = addr;
1371 }
1372
1373 /* Build up the log line for the message and main logs */
1374
1375 s = reset_point = store_get(size);
e4bdf652
JH
1376
1377 /* Create the address string for logging. Must not do this earlier, because
1378 an OK result may be changed to FAIL when a pipe returns text. */
1379
6c6d6e48 1380 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1381
059ec3d9
PH
1382 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
1383
6c6d6e48 1384 if (LOGGING(sender_on_delivery))
059ec3d9
PH
1385 s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
1386
1387 /* Return path may not be set if no delivery actually happened */
1388
d6c82d7b 1389 if (used_return_path && LOGGING(return_path_on_delivery))
059ec3d9 1390 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
059ec3d9 1391
d6c82d7b 1392 if (addr->router)
059ec3d9 1393 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
d6c82d7b 1394 if (addr->transport)
059ec3d9
PH
1395 s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
1396
d6c82d7b 1397 if (addr->host_used)
67d81c10
JH
1398 s = d_hostlog(s, &size, &ptr, addr);
1399
a7538db1 1400#ifdef SUPPORT_TLS
67d81c10 1401 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 1402#endif
059ec3d9
PH
1403
1404 if (addr->basic_errno > 0)
1405 s = string_append(s, &size, &ptr, 2, US": ",
1406 US strerror(addr->basic_errno));
1407
d6c82d7b 1408 if (addr->message)
059ec3d9
PH
1409 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1410
1411 s[ptr] = 0;
1412
1413 /* Do the logging. For the message log, "routing failed" for those cases,
1414 just to make it clearer. */
1415
d6c82d7b 1416 if (driver_name)
059ec3d9 1417 deliver_msglog("%s %s\n", now, s);
d6c82d7b
JH
1418 else
1419 deliver_msglog("%s %s failed for %s\n", now, driver_kind, s);
059ec3d9
PH
1420
1421 log_write(0, LOG_MAIN, "** %s", s);
14a465c3 1422
774ef2d7
JH
1423#ifdef EXPERIMENTAL_EVENT
1424 msg_event_raise(US"msg:fail:delivery", addr);
14a465c3
JH
1425#endif
1426
059ec3d9
PH
1427 store_reset(reset_point);
1428 }
1429
1430/* Ensure logging is turned on again in all cases */
1431
1432disable_logging = FALSE;
1433}
1434
1435
1436
1437
1438/*************************************************
1439* Address-independent error *
1440*************************************************/
1441
1442/* This function is called when there's an error that is not dependent on a
1443particular address, such as an expansion string failure. It puts the error into
1444all the addresses in a batch, logs the incident on the main and panic logs, and
1445clears the expansions. It is mostly called from local_deliver(), but can be
1446called for a remote delivery via findugid().
1447
1448Arguments:
1449 logit TRUE if (MAIN+PANIC) logging required
1450 addr the first of the chain of addresses
1451 code the error code
1452 format format string for error message, or NULL if already set in addr
1453 ... arguments for the format
1454
1455Returns: nothing
1456*/
1457
1458static void
1459common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1460{
1461address_item *addr2;
1462addr->basic_errno = code;
1463
d6c82d7b 1464if (format)
059ec3d9
PH
1465 {
1466 va_list ap;
1467 uschar buffer[512];
1468 va_start(ap, format);
1469 if (!string_vformat(buffer, sizeof(buffer), CS format, ap))
1470 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
81f91683 1471 "common_error expansion was longer than " SIZE_T_FMT, sizeof(buffer));
059ec3d9
PH
1472 va_end(ap);
1473 addr->message = string_copy(buffer);
1474 }
1475
d6c82d7b 1476for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9
PH
1477 {
1478 addr2->basic_errno = code;
1479 addr2->message = addr->message;
1480 }
1481
1482if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1483deliver_set_expansions(NULL);
1484}
1485
1486
1487
1488
1489/*************************************************
1490* Check a "never users" list *
1491*************************************************/
1492
1493/* This function is called to check whether a uid is on one of the two "never
1494users" lists.
1495
1496Arguments:
1497 uid the uid to be checked
1498 nusers the list to be scanned; the first item in the list is the count
1499
1500Returns: TRUE if the uid is on the list
1501*/
1502
1503static BOOL
1504check_never_users(uid_t uid, uid_t *nusers)
1505{
1506int i;
d6c82d7b 1507if (!nusers) return FALSE;
059ec3d9
PH
1508for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1509return FALSE;
1510}
1511
1512
1513
1514/*************************************************
1515* Find uid and gid for a transport *
1516*************************************************/
1517
1518/* This function is called for both local and remote deliveries, to find the
1519uid/gid under which to run the delivery. The values are taken preferentially
1520from the transport (either explicit or deliver_as_creator), then from the
1521address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1522the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1523panic error is logged, and the function fails (which normally leads to delivery
1524deferral).
1525
1526Arguments:
1527 addr the address (possibly a chain)
1528 tp the transport
1529 uidp pointer to uid field
1530 gidp pointer to gid field
1531 igfp pointer to the use_initgroups field
1532
1533Returns: FALSE if failed - error has been set in address(es)
1534*/
1535
1536static BOOL
1537findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1538 BOOL *igfp)
1539{
d6c82d7b 1540uschar *nuname;
059ec3d9
PH
1541BOOL gid_set = FALSE;
1542
1543/* Default initgroups flag comes from the transport */
1544
1545*igfp = tp->initgroups;
1546
1547/* First see if there's a gid on the transport, either fixed or expandable.
1548The expanding function always logs failure itself. */
1549
1550if (tp->gid_set)
1551 {
1552 *gidp = tp->gid;
1553 gid_set = TRUE;
1554 }
d6c82d7b 1555else if (tp->expand_gid)
059ec3d9 1556 {
d6c82d7b
JH
1557 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1558 &(addr->message)))
059ec3d9
PH
1559 {
1560 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1561 return FALSE;
1562 }
d6c82d7b 1563 gid_set = TRUE;
059ec3d9
PH
1564 }
1565
911f6fde
PH
1566/* If the transport did not set a group, see if the router did. */
1567
1568if (!gid_set && testflag(addr, af_gid_set))
1569 {
1570 *gidp = addr->gid;
1571 gid_set = TRUE;
1572 }
1573
059ec3d9
PH
1574/* Pick up a uid from the transport if one is set. */
1575
1576if (tp->uid_set) *uidp = tp->uid;
1577
1578/* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1579it does not provide a passwd value from which a gid can be taken. */
1580
d6c82d7b 1581else if (tp->expand_uid)
059ec3d9
PH
1582 {
1583 struct passwd *pw;
1584 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1585 uidp, &(addr->message)))
1586 {
1587 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1588 return FALSE;
1589 }
d6c82d7b 1590 if (!gid_set && pw)
059ec3d9
PH
1591 {
1592 *gidp = pw->pw_gid;
1593 gid_set = TRUE;
1594 }
1595 }
1596
1597/* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1598
1599else if (tp->deliver_as_creator)
1600 {
1601 *uidp = originator_uid;
1602 if (!gid_set)
1603 {
1604 *gidp = originator_gid;
1605 gid_set = TRUE;
1606 }
1607 }
1608
911f6fde
PH
1609/* Otherwise see if the address specifies the uid and if so, take it and its
1610initgroups flag. */
059ec3d9
PH
1611
1612else if (testflag(addr, af_uid_set))
1613 {
1614 *uidp = addr->uid;
1615 *igfp = testflag(addr, af_initgroups);
059ec3d9
PH
1616 }
1617
1618/* Nothing has specified the uid - default to the Exim user, and group if the
1619gid is not set. */
1620
1621else
1622 {
1623 *uidp = exim_uid;
1624 if (!gid_set)
1625 {
1626 *gidp = exim_gid;
1627 gid_set = TRUE;
1628 }
1629 }
1630
911f6fde
PH
1631/* If no gid is set, it is a disaster. We default to the Exim gid only if
1632defaulting to the Exim uid. In other words, if the configuration has specified
1633a uid, it must also provide a gid. */
059ec3d9
PH
1634
1635if (!gid_set)
1636 {
1637 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1638 "%s transport", tp->name);
1639 return FALSE;
1640 }
1641
1642/* Check that the uid is not on the lists of banned uids that may not be used
1643for delivery processes. */
1644
d6c82d7b
JH
1645nuname = check_never_users(*uidp, never_users)
1646 ? US"never_users"
1647 : check_never_users(*uidp, fixed_never_users)
1648 ? US"fixed_never_users"
1649 : NULL;
1650if (nuname)
059ec3d9
PH
1651 {
1652 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1653 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1654 return FALSE;
1655 }
1656
1657/* All is well */
1658
1659return TRUE;
1660}
1661
1662
1663
1664
1665/*************************************************
1666* Check the size of a message for a transport *
1667*************************************************/
1668
1669/* Checks that the message isn't too big for the selected transport.
1670This is called only when it is known that the limit is set.
1671
1672Arguments:
1673 tp the transport
1674 addr the (first) address being delivered
1675
1676Returns: OK
1677 DEFER expansion failed or did not yield an integer
1678 FAIL message too big
1679*/
1680
1681int
1682check_message_size(transport_instance *tp, address_item *addr)
1683{
1684int rc = OK;
1685int size_limit;
1686
1687deliver_set_expansions(addr);
d45b1de8 1688size_limit = expand_string_integer(tp->message_size_limit, TRUE);
059ec3d9
PH
1689deliver_set_expansions(NULL);
1690
d6c82d7b 1691if (expand_string_message)
059ec3d9
PH
1692 {
1693 rc = DEFER;
d6c82d7b
JH
1694 addr->message = size_limit == -1
1695 ? string_sprintf("failed to expand message_size_limit "
1696 "in %s transport: %s", tp->name, expand_string_message)
1697 : string_sprintf("invalid message_size_limit "
059ec3d9
PH
1698 "in %s transport: %s", tp->name, expand_string_message);
1699 }
1700else if (size_limit > 0 && message_size > size_limit)
1701 {
1702 rc = FAIL;
1703 addr->message =
1704 string_sprintf("message is too big (transport limit = %d)",
1705 size_limit);
1706 }
1707
1708return rc;
1709}
1710
1711
1712
1713/*************************************************
1714* Transport-time check for a previous delivery *
1715*************************************************/
1716
1717/* Check that this base address hasn't previously been delivered to its routed
c2c19e9d
PH
1718transport. If it has been delivered, mark it done. The check is necessary at
1719delivery time in order to handle homonymic addresses correctly in cases where
1720the pattern of redirection changes between delivery attempts (so the unique
1721fields change). Non-homonymic previous delivery is detected earlier, at routing
1722time (which saves unnecessary routing).
1723
1724Arguments:
1725 addr the address item
1726 testing TRUE if testing wanted only, without side effects
059ec3d9 1727
059ec3d9
PH
1728Returns: TRUE if previously delivered by the transport
1729*/
1730
1731static BOOL
c2c19e9d 1732previously_transported(address_item *addr, BOOL testing)
059ec3d9
PH
1733{
1734(void)string_format(big_buffer, big_buffer_size, "%s/%s",
1735 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
1736
1737if (tree_search(tree_nonrecipients, big_buffer) != 0)
1738 {
1739 DEBUG(D_deliver|D_route|D_transport)
1740 debug_printf("%s was previously delivered (%s transport): discarded\n",
1741 addr->address, addr->transport->name);
c2c19e9d 1742 if (!testing) child_done(addr, tod_stamp(tod_log));
059ec3d9
PH
1743 return TRUE;
1744 }
1745
1746return FALSE;
1747}
1748
1749
1750
064a94c9
PH
1751/******************************************************
1752* Check for a given header in a header string *
1753******************************************************/
1754
1755/* This function is used when generating quota warnings. The configuration may
1756specify any header lines it likes in quota_warn_message. If certain of them are
1757missing, defaults are inserted, so we need to be able to test for the presence
1758of a given header.
1759
1760Arguments:
1761 hdr the required header name
1762 hstring the header string
1763
1764Returns: TRUE the header is in the string
1765 FALSE the header is not in the string
1766*/
1767
1768static BOOL
1769contains_header(uschar *hdr, uschar *hstring)
1770{
1771int len = Ustrlen(hdr);
1772uschar *p = hstring;
1773while (*p != 0)
1774 {
1775 if (strncmpic(p, hdr, len) == 0)
1776 {
1777 p += len;
1778 while (*p == ' ' || *p == '\t') p++;
1779 if (*p == ':') return TRUE;
1780 }
1781 while (*p != 0 && *p != '\n') p++;
1782 if (*p == '\n') p++;
1783 }
1784return FALSE;
1785}
1786
1787
1788
059ec3d9
PH
1789
1790/*************************************************
1791* Perform a local delivery *
1792*************************************************/
1793
1794/* Each local delivery is performed in a separate process which sets its
1795uid and gid as specified. This is a safer way than simply changing and
1796restoring using seteuid(); there is a body of opinion that seteuid() cannot be
1797used safely. From release 4, Exim no longer makes any use of it. Besides, not
1798all systems have seteuid().
1799
1800If the uid/gid are specified in the transport_instance, they are used; the
1801transport initialization must ensure that either both or neither are set.
1802Otherwise, the values associated with the address are used. If neither are set,
1803it is a configuration error.
1804
1805The transport or the address may specify a home directory (transport over-
1806rides), and if they do, this is set as $home. If neither have set a working
1807directory, this value is used for that as well. Otherwise $home is left unset
1808and the cwd is set to "/" - a directory that should be accessible to all users.
1809
1810Using a separate process makes it more complicated to get error information
1811back. We use a pipe to pass the return code and also an error code and error
1812text string back to the parent process.
1813
1814Arguments:
1815 addr points to an address block for this delivery; for "normal" local
1816 deliveries this is the only address to be delivered, but for
1817 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
1818 a number of addresses can be handled simultaneously, and in this
1819 case addr will point to a chain of addresses with the same
1820 characteristics.
1821
1822 shadowing TRUE if running a shadow transport; this causes output from pipes
1823 to be ignored.
1824
1825Returns: nothing
1826*/
1827
1828static void
1829deliver_local(address_item *addr, BOOL shadowing)
1830{
1831BOOL use_initgroups;
1832uid_t uid;
1833gid_t gid;
1834int status, len, rc;
1835int pfd[2];
1836pid_t pid;
1837uschar *working_directory;
1838address_item *addr2;
1839transport_instance *tp = addr->transport;
1840
1841/* Set up the return path from the errors or sender address. If the transport
1842has its own return path setting, expand it and replace the existing value. */
1843
d6c82d7b 1844if(addr->prop.errors_address)
d43cbe25 1845 return_path = addr->prop.errors_address;
384152a6 1846#ifdef EXPERIMENTAL_SRS
d6c82d7b 1847else if (addr->prop.srs_sender)
d43cbe25 1848 return_path = addr->prop.srs_sender;
384152a6
TK
1849#endif
1850else
1851 return_path = sender_address;
059ec3d9 1852
d6c82d7b 1853if (tp->return_path)
059ec3d9
PH
1854 {
1855 uschar *new_return_path = expand_string(tp->return_path);
d6c82d7b 1856 if (!new_return_path)
059ec3d9
PH
1857 {
1858 if (!expand_string_forcedfail)
1859 {
1860 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
1861 US"Failed to expand return path \"%s\" in %s transport: %s",
1862 tp->return_path, tp->name, expand_string_message);
1863 return;
1864 }
1865 }
1866 else return_path = new_return_path;
1867 }
1868
1869/* For local deliveries, one at a time, the value used for logging can just be
1870set directly, once and for all. */
1871
1872used_return_path = return_path;
1873
1874/* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
1875gets put into the address(es), and the expansions are unset, so we can just
1876return. */
1877
1878if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
1879
5418e93b
PH
1880/* See if either the transport or the address specifies a home directory. A
1881home directory set in the address may already be expanded; a flag is set to
1882indicate that. In other cases we must expand it. */
059ec3d9 1883
d6c82d7b
JH
1884if ( (deliver_home = tp->home_dir) /* Set in transport, or */
1885 || ( (deliver_home = addr->home_dir) /* Set in address and */
1886 && !testflag(addr, af_home_expanded) /* not expanded */
1887 ) )
059ec3d9
PH
1888 {
1889 uschar *rawhome = deliver_home;
1890 deliver_home = NULL; /* in case it contains $home */
d6c82d7b 1891 if (!(deliver_home = expand_string(rawhome)))
059ec3d9
PH
1892 {
1893 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
1894 "to expand for %s transport: %s", rawhome, tp->name,
1895 expand_string_message);
1896 return;
1897 }
1898 if (*deliver_home != '/')
1899 {
1900 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
1901 "is not absolute for %s transport", deliver_home, tp->name);
1902 return;
1903 }
1904 }
1905
5418e93b
PH
1906/* See if either the transport or the address specifies a current directory,
1907and if so, expand it. If nothing is set, use the home directory, unless it is
1908also unset in which case use "/", which is assumed to be a directory to which
1909all users have access. It is necessary to be in a visible directory for some
1910operating systems when running pipes, as some commands (e.g. "rm" under Solaris
19112.5) require this. */
1912
d6c82d7b
JH
1913working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
1914if (working_directory)
059ec3d9
PH
1915 {
1916 uschar *raw = working_directory;
d6c82d7b 1917 if (!(working_directory = expand_string(raw)))
059ec3d9
PH
1918 {
1919 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
1920 "failed to expand for %s transport: %s", raw, tp->name,
1921 expand_string_message);
1922 return;
1923 }
1924 if (*working_directory != '/')
1925 {
1926 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
1927 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
1928 return;
1929 }
1930 }
d6c82d7b 1931else working_directory = deliver_home ? deliver_home : US"/";
059ec3d9
PH
1932
1933/* If one of the return_output flags is set on the transport, create and open a
1934file in the message log directory for the transport to write its output onto.
1935This is mainly used by pipe transports. The file needs to be unique to the
1936address. This feature is not available for shadow transports. */
1937
d6c82d7b
JH
1938if ( !shadowing
1939 && ( tp->return_output || tp->return_fail_output
ec0eb1a3 1940 || tp->log_output || tp->log_fail_output || tp->log_defer_output
d6c82d7b 1941 ) )
059ec3d9
PH
1942 {
1943 uschar *error;
1944 addr->return_filename =
1945 string_sprintf("%s/msglog/%s/%s-%d-%d", spool_directory, message_subdir,
1946 message_id, getpid(), return_count++);
1947 addr->return_file = open_msglog_file(addr->return_filename, 0400, &error);
1948 if (addr->return_file < 0)
1949 {
1950 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
1951 "to return message: %s", error, tp->name, strerror(errno));
1952 return;
1953 }
1954 }
1955
1956/* Create the pipe for inter-process communication. */
1957
1958if (pipe(pfd) != 0)
1959 {
1960 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
1961 strerror(errno));
1962 return;
1963 }
1964
1965/* Now fork the process to do the real work in the subprocess, but first
1966ensure that all cached resources are freed so that the subprocess starts with
1967a clean slate and doesn't interfere with the parent process. */
1968
1969search_tidyup();
1970
1971if ((pid = fork()) == 0)
1972 {
1973 BOOL replicate = TRUE;
1974
1975 /* Prevent core dumps, as we don't want them in users' home directories.
1976 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
1977 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
1978 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
a29e5231
PP
1979 complain if the error is "not supported".
1980
1981 There are two scenarios where changing the max limit has an effect. In one,
1982 the user is using a .forward and invoking a command of their choice via pipe;
1983 for these, we do need the max limit to be 0 unless the admin chooses to
1984 permit an increased limit. In the other, the command is invoked directly by
1985 the transport and is under administrator control, thus being able to raise
1986 the limit aids in debugging. So there's no general always-right answer.
1987
1988 Thus we inhibit core-dumps completely but let individual transports, while
1989 still root, re-raise the limits back up to aid debugging. We make the
1990 default be no core-dumps -- few enough people can use core dumps in
1991 diagnosis that it's reasonable to make them something that has to be explicitly requested.
1992 */
059ec3d9 1993
a7538db1 1994#ifdef RLIMIT_CORE
059ec3d9
PH
1995 struct rlimit rl;
1996 rl.rlim_cur = 0;
1997 rl.rlim_max = 0;
1998 if (setrlimit(RLIMIT_CORE, &rl) < 0)
1999 {
a7538db1 2000# ifdef SETRLIMIT_NOT_SUPPORTED
059ec3d9 2001 if (errno != ENOSYS && errno != ENOTSUP)
a7538db1 2002# endif
059ec3d9
PH
2003 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
2004 strerror(errno));
2005 }
a7538db1 2006#endif
059ec3d9
PH
2007
2008 /* Reset the random number generator, so different processes don't all
2009 have the same sequence. */
2010
2011 random_seed = 0;
2012
2013 /* If the transport has a setup entry, call this first, while still
2014 privileged. (Appendfile uses this to expand quota, for example, while
2015 able to read private files.) */
2016
d6c82d7b 2017 if (addr->transport->setup)
929ba01c 2018 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
059ec3d9
PH
2019 &(addr->message)))
2020 {
2021 case DEFER:
d6c82d7b
JH
2022 addr->transport_return = DEFER;
2023 goto PASS_BACK;
059ec3d9
PH
2024
2025 case FAIL:
d6c82d7b
JH
2026 addr->transport_return = PANIC;
2027 goto PASS_BACK;
059ec3d9 2028 }
059ec3d9
PH
2029
2030 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2031 when the process becomes unprivileged, it won't be able to write to the
2032 process log. SIGHUP is ignored throughout exim, except when it is being
2033 run as a daemon. */
2034
2035 signal(SIGINT, SIG_IGN);
2036 signal(SIGTERM, SIG_IGN);
2037 signal(SIGUSR1, SIG_IGN);
2038
2039 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2040 half - for transports that exec things (e.g. pipe). Then set the required
2041 gid/uid. */
2042
f1e894f3 2043 (void)close(pfd[pipe_read]);
ff790e47 2044 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
059ec3d9
PH
2045 FD_CLOEXEC);
2046 exim_setugid(uid, gid, use_initgroups,
2047 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2048 addr->address, addr->transport->name));
2049
2050 DEBUG(D_deliver)
2051 {
2052 address_item *batched;
2053 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
d6c82d7b 2054 for (batched = addr->next; batched; batched = batched->next)
059ec3d9
PH
2055 debug_printf("additional batched address: %s\n", batched->address);
2056 }
2057
2058 /* Set an appropriate working directory. */
2059
2060 if (Uchdir(working_directory) < 0)
2061 {
2062 addr->transport_return = DEFER;
2063 addr->basic_errno = errno;
2064 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2065 }
2066
2067 /* If successful, call the transport */
2068
2069 else
2070 {
2071 BOOL ok = TRUE;
2072 set_process_info("delivering %s to %s using %s", message_id,
2073 addr->local_part, addr->transport->name);
2074
2a47f028
JH
2075 /* Setting this global in the subprocess means we need never clear it */
2076 transport_name = addr->transport->name;
2077
059ec3d9
PH
2078 /* If a transport filter has been specified, set up its argument list.
2079 Any errors will get put into the address, and FALSE yielded. */
2080
d6c82d7b 2081 if (addr->transport->filter_command)
059ec3d9
PH
2082 {
2083 ok = transport_set_up_command(&transport_filter_argv,
2084 addr->transport->filter_command,
2085 TRUE, PANIC, addr, US"transport filter", NULL);
2086 transport_filter_timeout = addr->transport->filter_timeout;
2087 }
2088 else transport_filter_argv = NULL;
2089
2090 if (ok)
2091 {
2092 debug_print_string(addr->transport->debug_string);
2093 replicate = !(addr->transport->info->code)(addr->transport, addr);
2094 }
2095 }
2096
2097 /* Pass the results back down the pipe. If necessary, first replicate the
2098 status in the top address to the others in the batch. The label is the
2099 subject of a goto when a call to the transport's setup function fails. We
2100 pass the pointer to the transport back in case it got changed as a result of
2101 file_format in appendfile. */
2102
2103 PASS_BACK:
2104
2105 if (replicate) replicate_status(addr);
d6c82d7b 2106 for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9
PH
2107 {
2108 int i;
2109 int local_part_length = Ustrlen(addr2->local_part);
2110 uschar *s;
1ac6b2e7 2111 int ret;
059ec3d9 2112
d6c82d7b
JH
2113 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2114 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2115 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2116 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2117 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
2118 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2119 || (ret = write(pfd[pipe_write], &addr2->transport,
1ac6b2e7 2120 sizeof(transport_instance *))) != sizeof(transport_instance *)
059ec3d9
PH
2121
2122 /* For a file delivery, pass back the local part, in case the original
2123 was only part of the final delivery path. This gives more complete
2124 logging. */
2125
1ac6b2e7 2126 || (testflag(addr2, af_file)
d6c82d7b 2127 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
1ac6b2e7
JH
2128 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2129 )
2130 )
2131 )
2132 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2133 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2134
2135 /* Now any messages */
2136
2137 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2138 {
d6c82d7b
JH
2139 int message_length = s ? Ustrlen(s) + 1 : 0;
2140 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2141 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
1ac6b2e7
JH
2142 )
2143 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2144 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2145 }
2146 }
2147
2148 /* OK, this process is now done. Free any cached resources that it opened,
2149 and close the pipe we were writing down before exiting. */
2150
f1e894f3 2151 (void)close(pfd[pipe_write]);
059ec3d9
PH
2152 search_tidyup();
2153 exit(EXIT_SUCCESS);
2154 }
2155
2156/* Back in the main process: panic if the fork did not succeed. This seems
2157better than returning an error - if forking is failing it is probably best
2158not to try other deliveries for this message. */
2159
2160if (pid < 0)
2161 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2162 addr->address);
2163
2164/* Read the pipe to get the delivery status codes and error messages. Our copy
2165of the writing end must be closed first, as otherwise read() won't return zero
2166on an empty pipe. We check that a status exists for each address before
2167overwriting the address structure. If data is missing, the default DEFER status
2168will remain. Afterwards, close the reading end. */
2169
f1e894f3 2170(void)close(pfd[pipe_write]);
059ec3d9 2171
d6c82d7b 2172for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9 2173 {
d6c82d7b 2174 len = read(pfd[pipe_read], &status, sizeof(int));
059ec3d9
PH
2175 if (len > 0)
2176 {
2177 int i;
2178 uschar **sptr;
2179
2180 addr2->transport_return = status;
d6c82d7b 2181 len = read(pfd[pipe_read], &transport_count,
059ec3d9 2182 sizeof(transport_count));
d6c82d7b
JH
2183 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2184 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2185 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
2186 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2187 len = read(pfd[pipe_read], &addr2->transport,
059ec3d9
PH
2188 sizeof(transport_instance *));
2189
2190 if (testflag(addr2, af_file))
2191 {
2192 int local_part_length;
d6c82d7b
JH
2193 len = read(pfd[pipe_read], &local_part_length, sizeof(int));
2194 len = read(pfd[pipe_read], big_buffer, local_part_length);
059ec3d9
PH
2195 big_buffer[local_part_length] = 0;
2196 addr2->local_part = string_copy(big_buffer);
2197 }
2198
d6c82d7b 2199 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
059ec3d9
PH
2200 {
2201 int message_length;
d6c82d7b 2202 len = read(pfd[pipe_read], &message_length, sizeof(int));
059ec3d9
PH
2203 if (message_length > 0)
2204 {
d6c82d7b 2205 len = read(pfd[pipe_read], big_buffer, message_length);
059ec3d9
PH
2206 if (len > 0) *sptr = string_copy(big_buffer);
2207 }
2208 }
2209 }
2210
2211 else
2212 {
2213 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2214 "from delivery subprocess", addr2->unique);
2215 break;
2216 }
2217 }
2218
f1e894f3 2219(void)close(pfd[pipe_read]);
059ec3d9
PH
2220
2221/* Unless shadowing, write all successful addresses immediately to the journal
2222file, to ensure they are recorded asap. For homonymic addresses, use the base
2223address plus the transport name. Failure to write the journal is panic-worthy,
2224but don't stop, as it may prove possible subsequently to update the spool file
2225in order to record the delivery. */
2226
2227if (!shadowing)
2228 {
d6c82d7b
JH
2229 for (addr2 = addr; addr2; addr2 = addr2->next)
2230 if (addr2->transport_return == OK)
2231 {
2232 if (testflag(addr2, af_homonym))
2233 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2234 else
2235 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
059ec3d9 2236
d6c82d7b
JH
2237 /* In the test harness, wait just a bit to let the subprocess finish off
2238 any debug output etc first. */
059ec3d9 2239
d6c82d7b 2240 if (running_in_test_harness) millisleep(300);
059ec3d9 2241
d6c82d7b
JH
2242 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2243 len = Ustrlen(big_buffer);
2244 if (write(journal_fd, big_buffer, len) != len)
2245 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2246 big_buffer, strerror(errno));
2247 }
059ec3d9
PH
2248
2249 /* Ensure the journal file is pushed out to disk. */
2250
54fc8428 2251 if (EXIMfsync(journal_fd) < 0)
059ec3d9
PH
2252 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2253 strerror(errno));
2254 }
2255
2256/* Wait for the process to finish. If it terminates with a non-zero code,
2257freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2258status values of all the addresses as they are. Take care to handle the case
2259when the subprocess doesn't seem to exist. This has been seen on one system
2260when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2261happens, wait() doesn't recognize the termination of child processes. Exim now
2262resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2263
2264while ((rc = wait(&status)) != pid)
059ec3d9
PH
2265 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2266 {
2267 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2268 addr->transport->driver_name);
2269 status = 0;
2270 break;
2271 }
059ec3d9
PH
2272
2273if ((status & 0xffff) != 0)
2274 {
2275 int msb = (status >> 8) & 255;
2276 int lsb = status & 255;
2277 int code = (msb == 0)? (lsb & 0x7f) : msb;
2278 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2279 addr->special_action = SPECIAL_FREEZE;
2280 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2281 "status 0x%04x: %s %d",
2282 addr->transport->driver_name,
2283 status,
d6c82d7b 2284 msb == 0 ? "terminated by signal" : "exit code",
059ec3d9
PH
2285 code);
2286 }
2287
2288/* If SPECIAL_WARN is set in the top address, send a warning message. */
2289
d6c82d7b 2290if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
059ec3d9
PH
2291 {
2292 int fd;
2293 uschar *warn_message;
d6c82d7b 2294 pid_t pid;
059ec3d9
PH
2295
2296 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2297
d6c82d7b 2298 if (!(warn_message = expand_string(addr->transport->warn_message)))
059ec3d9
PH
2299 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2300 "message for %s transport): %s", addr->transport->warn_message,
2301 addr->transport->name, expand_string_message);
d6c82d7b
JH
2302
2303 else if ((pid = child_open_exim(&fd)) > 0)
059ec3d9 2304 {
d6c82d7b
JH
2305 FILE *f = fdopen(fd, "wb");
2306 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2307 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2308 fprintf(f, "Auto-Submitted: auto-replied\n");
2309 if (!contains_header(US"From", warn_message))
2310 moan_write_from(f);
2311 fprintf(f, "%s", CS warn_message);
059ec3d9 2312
d6c82d7b 2313 /* Close and wait for child process to complete, without a timeout. */
059ec3d9 2314
d6c82d7b
JH
2315 (void)fclose(f);
2316 (void)child_close(pid, 0);
059ec3d9
PH
2317 }
2318
2319 addr->special_action = SPECIAL_NONE;
2320 }
2321}
2322
2323
2324
fa41615d 2325
3070ceee
JH
2326/* Check transport for the given concurrency limit. Return TRUE if over
2327the limit (or an expansion failure), else FALSE and if there was a limit,
2328the key for the hints database used for the concurrency count. */
fa41615d 2329
3070ceee
JH
2330static BOOL
2331tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key)
fa41615d 2332{
3070ceee
JH
2333unsigned max_parallel;
2334
2335if (!tp->max_parallel) return FALSE;
2336
2337max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE);
2338if (expand_string_message)
2339 {
2340 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2341 "in %s transport (%s): %s", tp->name, addr->address,
2342 expand_string_message);
2343 return TRUE;
2344 }
2345
2346if (max_parallel > 0)
2347 {
2348 uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name);
2349 if (!enq_start(serialize_key, max_parallel))
2350 {
2351 address_item * next;
2352 DEBUG(D_transport)
2353 debug_printf("skipping tpt %s because concurrency limit %u reached\n",
2354 tp->name, max_parallel);
2355 do
2356 {
2357 next = addr->next;
2358 addr->message = US"concurrency limit reached for transport";
2359 addr->basic_errno = ERRNO_TRETRY;
2360 post_process_one(addr, DEFER, LOG_MAIN, DTYPE_TRANSPORT, 0);
2361 } while ((addr = next));
2362 return TRUE;
2363 }
2364 *key = serialize_key;
2365 }
2366return FALSE;
fa41615d
JH
2367}
2368
2369
2370
059ec3d9
PH
2371/*************************************************
2372* Do local deliveries *
2373*************************************************/
2374
2375/* This function processes the list of addresses in addr_local. True local
2376deliveries are always done one address at a time. However, local deliveries can
2377be batched up in some cases. Typically this is when writing batched SMTP output
2378files for use by some external transport mechanism, or when running local
2379deliveries over LMTP.
2380
2381Arguments: None
2382Returns: Nothing
2383*/
2384
2385static void
2386do_local_deliveries(void)
2387{
2388open_db dbblock;
2389open_db *dbm_file = NULL;
2390time_t now = time(NULL);
2391
2392/* Loop until we have exhausted the supply of local deliveries */
2393
d6c82d7b 2394while (addr_local)
059ec3d9
PH
2395 {
2396 time_t delivery_start;
2397 int deliver_time;
2398 address_item *addr2, *addr3, *nextaddr;
2399 int logflags = LOG_MAIN;
2400 int logchar = dont_deliver? '*' : '=';
2401 transport_instance *tp;
fa41615d 2402 uschar * serialize_key = NULL;
059ec3d9
PH
2403
2404 /* Pick the first undelivered address off the chain */
2405
2406 address_item *addr = addr_local;
2407 addr_local = addr->next;
2408 addr->next = NULL;
2409
2410 DEBUG(D_deliver|D_transport)
2411 debug_printf("--------> %s <--------\n", addr->address);
2412
2413 /* An internal disaster if there is no transport. Should not occur! */
2414
d6c82d7b 2415 if (!(tp = addr->transport))
059ec3d9
PH
2416 {
2417 logflags |= LOG_PANIC;
2418 disable_logging = FALSE; /* Jic */
d6c82d7b
JH
2419 addr->message = addr->router
2420 ? string_sprintf("No transport set by %s router", addr->router->name)
2421 : string_sprintf("No transport set by system filter");
059ec3d9
PH
2422 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
2423 continue;
2424 }
2425
2426 /* Check that this base address hasn't previously been delivered to this
2427 transport. The check is necessary at this point to handle homonymic addresses
2428 correctly in cases where the pattern of redirection changes between delivery
2429 attempts. Non-homonymic previous delivery is detected earlier, at routing
2430 time. */
2431
c2c19e9d 2432 if (previously_transported(addr, FALSE)) continue;
059ec3d9
PH
2433
2434 /* There are weird cases where logging is disabled */
2435
2436 disable_logging = tp->disable_logging;
2437
f7fd3850
PH
2438 /* Check for batched addresses and possible amalgamation. Skip all the work
2439 if either batch_max <= 1 or there aren't any other addresses for local
2440 delivery. */
059ec3d9 2441
d6c82d7b 2442 if (tp->batch_max > 1 && addr_local)
059ec3d9
PH
2443 {
2444 int batch_count = 1;
2445 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
d6c82d7b
JH
2446 BOOL uses_lp = ( testflag(addr, af_pfr)
2447 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2448 )
2449 || readconf_depends((driver_instance *)tp, US"local_part");
059ec3d9
PH
2450 uschar *batch_id = NULL;
2451 address_item **anchor = &addr_local;
2452 address_item *last = addr;
2453 address_item *next;
2454
2455 /* Expand the batch_id string for comparison with other addresses.
2456 Expansion failure suppresses batching. */
2457
d6c82d7b 2458 if (tp->batch_id)
059ec3d9
PH
2459 {
2460 deliver_set_expansions(addr);
2461 batch_id = expand_string(tp->batch_id);
2462 deliver_set_expansions(NULL);
d6c82d7b 2463 if (!batch_id)
059ec3d9
PH
2464 {
2465 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2466 "in %s transport (%s): %s", tp->name, addr->address,
2467 expand_string_message);
2468 batch_count = tp->batch_max;
2469 }
2470 }
2471
2472 /* Until we reach the batch_max limit, pick off addresses which have the
2473 same characteristics. These are:
2474
2475 same transport
7816e254 2476 not previously delivered (see comment about 50 lines above)
059ec3d9 2477 same local part if the transport's configuration contains $local_part
f7fd3850 2478 or if this is a file or pipe delivery from a redirection
059ec3d9
PH
2479 same domain if the transport's configuration contains $domain
2480 same errors address
2481 same additional headers
2482 same headers to be removed
2483 same uid/gid for running the transport
2484 same first host if a host list is set
2485 */
2486
d6c82d7b 2487 while ((next = *anchor) && batch_count < tp->batch_max)
059ec3d9
PH
2488 {
2489 BOOL ok =
d6c82d7b
JH
2490 tp == next->transport
2491 && !previously_transported(next, TRUE)
2492 && (addr->flags & (af_pfr|af_file)) == (next->flags & (af_pfr|af_file))
2493 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2494 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2495 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2496 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2497 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2498 && same_ugid(tp, addr, next)
2499 && ( !addr->host_list && !next->host_list
2500 || addr->host_list
2501 && next->host_list
2502 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2503 );
059ec3d9
PH
2504
2505 /* If the transport has a batch_id setting, batch_id will be non-NULL
2506 from the expansion outside the loop. Expand for this address and compare.
2507 Expansion failure makes this address ineligible for batching. */
2508
d6c82d7b 2509 if (ok && batch_id)
059ec3d9
PH
2510 {
2511 uschar *bid;
2512 address_item *save_nextnext = next->next;
2513 next->next = NULL; /* Expansion for a single address */
2514 deliver_set_expansions(next);
2515 next->next = save_nextnext;
2516 bid = expand_string(tp->batch_id);
2517 deliver_set_expansions(NULL);
d6c82d7b 2518 if (!bid)
059ec3d9
PH
2519 {
2520 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2521 "in %s transport (%s): %s", tp->name, next->address,
2522 expand_string_message);
2523 ok = FALSE;
2524 }
2525 else ok = (Ustrcmp(batch_id, bid) == 0);
2526 }
2527
2528 /* Take address into batch if OK. */
2529
2530 if (ok)
2531 {
2532 *anchor = next->next; /* Include the address */
2533 next->next = NULL;
2534 last->next = next;
2535 last = next;
2536 batch_count++;
2537 }
fa41615d 2538 else anchor = &next->next; /* Skip the address */
059ec3d9
PH
2539 }
2540 }
2541
2542 /* We now have one or more addresses that can be delivered in a batch. Check
2543 whether the transport is prepared to accept a message of this size. If not,
2544 fail them all forthwith. If the expansion fails, or does not yield an
2545 integer, defer delivery. */
2546
d6c82d7b 2547 if (tp->message_size_limit)
059ec3d9
PH
2548 {
2549 int rc = check_message_size(tp, addr);
2550 if (rc != OK)
2551 {
2552 replicate_status(addr);
d6c82d7b 2553 while (addr)
059ec3d9
PH
2554 {
2555 addr2 = addr->next;
2556 post_process_one(addr, rc, logflags, DTYPE_TRANSPORT, 0);
2557 addr = addr2;
2558 }
2559 continue; /* With next batch of addresses */
2560 }
2561 }
2562
2563 /* If we are not running the queue, or if forcing, all deliveries will be
2564 attempted. Otherwise, we must respect the retry times for each address. Even
2565 when not doing this, we need to set up the retry key string, and determine
2566 whether a retry record exists, because after a successful delivery, a delete
2567 retry item must be set up. Keep the retry database open only for the duration
2568 of these checks, rather than for all local deliveries, because some local
2569 deliveries (e.g. to pipes) can take a substantial time. */
2570
d6c82d7b 2571 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
059ec3d9
PH
2572 {
2573 DEBUG(D_deliver|D_retry|D_hints_lookup)
2574 debug_printf("no retry data available\n");
2575 }
2576
2577 addr2 = addr;
2578 addr3 = NULL;
d6c82d7b 2579 while (addr2)
059ec3d9
PH
2580 {
2581 BOOL ok = TRUE; /* to deliver this address */
2582 uschar *retry_key;
2583
2584 /* Set up the retry key to include the domain or not, and change its
2585 leading character from "R" to "T". Must make a copy before doing this,
2586 because the old key may be pointed to from a "delete" retry item after
2587 a routing delay. */
2588
2589 retry_key = string_copy(
d6c82d7b 2590 tp->retry_use_local_part ? addr2->address_retry_key :
059ec3d9
PH
2591 addr2->domain_retry_key);
2592 *retry_key = 'T';
2593
2594 /* Inspect the retry data. If there is no hints file, delivery happens. */
2595
d6c82d7b 2596 if (dbm_file)
059ec3d9
PH
2597 {
2598 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2599
2600 /* If there is no retry record, delivery happens. If there is,
2601 remember it exists so it can be deleted after a successful delivery. */
2602
d6c82d7b 2603 if (retry_record)
059ec3d9
PH
2604 {
2605 setflag(addr2, af_lt_retry_exists);
2606
2607 /* A retry record exists for this address. If queue running and not
2608 forcing, inspect its contents. If the record is too old, or if its
2609 retry time has come, or if it has passed its cutoff time, delivery
2610 will go ahead. */
2611
2612 DEBUG(D_retry)
2613 {
ea49d0e1
PH
2614 debug_printf("retry record exists: age=%s ",
2615 readconf_printtime(now - retry_record->time_stamp));
2616 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2617 debug_printf(" time to retry = %s expired = %d\n",
2618 readconf_printtime(retry_record->next_try - now),
2619 retry_record->expired);
059ec3d9
PH
2620 }
2621
2622 if (queue_running && !deliver_force)
2623 {
d6c82d7b
JH
2624 ok = (now - retry_record->time_stamp > retry_data_expire)
2625 || (now >= retry_record->next_try)
2626 || retry_record->expired;
059ec3d9
PH
2627
2628 /* If we haven't reached the retry time, there is one more check
1ddeb334 2629 to do, which is for the ultimate address timeout. */
059ec3d9
PH
2630
2631 if (!ok)
ba9af0af
TF
2632 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2633 retry_record, now);
059ec3d9
PH
2634 }
2635 }
2636 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2637 }
2638
2639 /* This address is to be delivered. Leave it on the chain. */
2640
2641 if (ok)
2642 {
2643 addr3 = addr2;
2644 addr2 = addr2->next;
2645 }
2646
2647 /* This address is to be deferred. Take it out of the chain, and
2648 post-process it as complete. Must take it out of the chain first,
2649 because post processing puts it on another chain. */
2650
2651 else
2652 {
2653 address_item *this = addr2;
2654 this->message = US"Retry time not yet reached";
2655 this->basic_errno = ERRNO_LRETRY;
d6c82d7b
JH
2656 addr2 = addr3 ? (addr3->next = addr2->next)
2657 : (addr = addr2->next);
059ec3d9
PH
2658 post_process_one(this, DEFER, logflags, DTYPE_TRANSPORT, 0);
2659 }
2660 }
2661
d6c82d7b 2662 if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
2663
2664 /* If there are no addresses left on the chain, they all deferred. Loop
2665 for the next set of addresses. */
2666
d6c82d7b 2667 if (!addr) continue;
059ec3d9 2668
fa41615d
JH
2669 /* If the transport is limited for parallellism, enforce that here.
2670 We use a hints DB entry, incremented here and decremented after
2671 the transport (and any shadow transport) completes. */
2672
3070ceee 2673 if (tpt_parallel_check(tp, addr, &serialize_key))
fa41615d 2674 {
fa41615d
JH
2675 if (expand_string_message)
2676 {
2677 logflags |= LOG_PANIC;
3070ceee
JH
2678 do
2679 {
2680 addr = addr->next;
fa41615d 2681 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
3070ceee 2682 } while ((addr = addr2));
fa41615d 2683 }
3070ceee 2684 continue; /* Loop for the next set of addresses. */
fa41615d
JH
2685 }
2686
2687
059ec3d9
PH
2688 /* So, finally, we do have some addresses that can be passed to the
2689 transport. Before doing so, set up variables that are relevant to a
2690 single delivery. */
2691
2692 deliver_set_expansions(addr);
2693 delivery_start = time(NULL);
2694 deliver_local(addr, FALSE);
2695 deliver_time = (int)(time(NULL) - delivery_start);
2696
2697 /* If a shadow transport (which must perforce be another local transport), is
2698 defined, and its condition is met, we must pass the message to the shadow
2699 too, but only those addresses that succeeded. We do this by making a new
2700 chain of addresses - also to keep the original chain uncontaminated. We must
2701 use a chain rather than doing it one by one, because the shadow transport may
2702 batch.
2703
2704 NOTE: if the condition fails because of a lookup defer, there is nothing we
2705 can do! */
2706
d6c82d7b
JH
2707 if ( tp->shadow
2708 && ( !tp->shadow_condition
2709 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
2710 ) )
059ec3d9
PH
2711 {
2712 transport_instance *stp;
2713 address_item *shadow_addr = NULL;
2714 address_item **last = &shadow_addr;
2715
d6c82d7b 2716 for (stp = transports; stp; stp = stp->next)
059ec3d9
PH
2717 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
2718
d6c82d7b 2719 if (!stp)
059ec3d9
PH
2720 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
2721 tp->shadow);
2722
2723 /* Pick off the addresses that have succeeded, and make clones. Put into
2724 the shadow_message field a pointer to the shadow_message field of the real
2725 address. */
2726
d6c82d7b
JH
2727 else for (addr2 = addr; addr2; addr2 = addr2->next)
2728 if (addr2->transport_return == OK)
2729 {
2730 addr3 = store_get(sizeof(address_item));
2731 *addr3 = *addr2;
2732 addr3->next = NULL;
2733 addr3->shadow_message = (uschar *) &(addr2->shadow_message);
2734 addr3->transport = stp;
2735 addr3->transport_return = DEFER;
2736 addr3->return_filename = NULL;
2737 addr3->return_file = -1;
2738 *last = addr3;
2739 last = &(addr3->next);
2740 }
059ec3d9
PH
2741
2742 /* If we found any addresses to shadow, run the delivery, and stick any
2743 message back into the shadow_message field in the original. */
2744
d6c82d7b 2745 if (shadow_addr)
059ec3d9
PH
2746 {
2747 int save_count = transport_count;
2748
2749 DEBUG(D_deliver|D_transport)
2750 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
2751 deliver_local(shadow_addr, TRUE);
2752
d6c82d7b 2753 for(; shadow_addr; shadow_addr = shadow_addr->next)
059ec3d9
PH
2754 {
2755 int sresult = shadow_addr->transport_return;
d6c82d7b
JH
2756 *(uschar **)shadow_addr->shadow_message =
2757 sresult == OK
2758 ? string_sprintf(" ST=%s", stp->name)
2759 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
2760 shadow_addr->basic_errno <= 0
2761 ? US""
2762 : US strerror(shadow_addr->basic_errno),
2763 shadow_addr->basic_errno <= 0 || !shadow_addr->message
2764 ? US""
2765 : US": ",
2766 shadow_addr->message
2767 ? shadow_addr->message
2768 : shadow_addr->basic_errno <= 0
2769 ? US"unknown error"
2770 : US"");
059ec3d9
PH
2771
2772 DEBUG(D_deliver|D_transport)
2773 debug_printf("%s shadow transport returned %s for %s\n",
2774 stp->name,
d6c82d7b
JH
2775 sresult == OK ? "OK" :
2776 sresult == DEFER ? "DEFER" :
2777 sresult == FAIL ? "FAIL" :
2778 sresult == PANIC ? "PANIC" : "?",
059ec3d9
PH
2779 shadow_addr->address);
2780 }
2781
2782 DEBUG(D_deliver|D_transport)
2783 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
2784
2785 transport_count = save_count; /* Restore original transport count */
2786 }
2787 }
2788
2789 /* Cancel the expansions that were set up for the delivery. */
2790
2791 deliver_set_expansions(NULL);
2792
fa41615d
JH
2793 /* If the transport was parallelism-limited, decrement the hints DB record. */
2794
2795 if (serialize_key) enq_end(serialize_key);
2796
059ec3d9
PH
2797 /* Now we can process the results of the real transport. We must take each
2798 address off the chain first, because post_process_one() puts it on another
2799 chain. */
2800
d6c82d7b 2801 for (addr2 = addr; addr2; addr2 = nextaddr)
059ec3d9
PH
2802 {
2803 int result = addr2->transport_return;
2804 nextaddr = addr2->next;
2805
2806 DEBUG(D_deliver|D_transport)
2807 debug_printf("%s transport returned %s for %s\n",
2808 tp->name,
d6c82d7b
JH
2809 result == OK ? "OK" :
2810 result == DEFER ? "DEFER" :
2811 result == FAIL ? "FAIL" :
2812 result == PANIC ? "PANIC" : "?",
059ec3d9
PH
2813 addr2->address);
2814
2815 /* If there is a retry_record, or if delivery is deferred, build a retry
2816 item for setting a new retry time or deleting the old retry record from
2817 the database. These items are handled all together after all addresses
2818 have been handled (so the database is open just for a short time for
2819 updating). */
2820
2821 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
2822 {
d6c82d7b
JH
2823 int flags = result == DEFER ? 0 : rf_delete;
2824 uschar *retry_key = string_copy(tp->retry_use_local_part
2825 ? addr2->address_retry_key : addr2->domain_retry_key);
059ec3d9
PH
2826 *retry_key = 'T';
2827 retry_add_item(addr2, retry_key, flags);
2828 }
2829
2830 /* Done with this address */
2831
2832 if (result == OK) addr2->more_errno = deliver_time;
2833 post_process_one(addr2, result, logflags, DTYPE_TRANSPORT, logchar);
2834
2835 /* If a pipe delivery generated text to be sent back, the result may be
2836 changed to FAIL, and we must copy this for subsequent addresses in the
2837 batch. */
2838
2839 if (addr2->transport_return != result)
2840 {
d6c82d7b 2841 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
059ec3d9
PH
2842 {
2843 addr3->transport_return = addr2->transport_return;
2844 addr3->basic_errno = addr2->basic_errno;
2845 addr3->message = addr2->message;
2846 }
2847 result = addr2->transport_return;
2848 }
2849
2850 /* Whether or not the result was changed to FAIL, we need to copy the
2851 return_file value from the first address into all the addresses of the
2852 batch, so they are all listed in the error message. */
2853
2854 addr2->return_file = addr->return_file;
2855
2856 /* Change log character for recording successful deliveries. */
2857
2858 if (result == OK) logchar = '-';
2859 }
2860 } /* Loop back for next batch of addresses */
2861}
2862
2863
2864
2865
2866/*************************************************
2867* Sort remote deliveries *
2868*************************************************/
2869
2870/* This function is called if remote_sort_domains is set. It arranges that the
2871chain of addresses for remote deliveries is ordered according to the strings
2872specified. Try to make this shuffling reasonably efficient by handling
2873sequences of addresses rather than just single ones.
2874
2875Arguments: None
2876Returns: Nothing
2877*/
2878
2879static void
2880sort_remote_deliveries(void)
2881{
2882int sep = 0;
2883address_item **aptr = &addr_remote;
55414b25 2884const uschar *listptr = remote_sort_domains;
059ec3d9
PH
2885uschar *pattern;
2886uschar patbuf[256];
2887
d6c82d7b
JH
2888while ( *aptr
2889 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
2890 )
059ec3d9
PH
2891 {
2892 address_item *moved = NULL;
2893 address_item **bptr = &moved;
2894
d6c82d7b 2895 while (*aptr)
059ec3d9
PH
2896 {
2897 address_item **next;
2898 deliver_domain = (*aptr)->domain; /* set $domain */
55414b25 2899 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
059ec3d9
PH
2900 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
2901 {
d6c82d7b 2902 aptr = &(*aptr)->next;
059ec3d9
PH
2903 continue;
2904 }
2905
d6c82d7b
JH
2906 next = &(*aptr)->next;
2907 while ( *next
2908 && (deliver_domain = (*next)->domain, /* Set $domain */
55414b25 2909 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
d6c82d7b
JH
2910 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
2911 )
2912 next = &(*next)->next;
059ec3d9
PH
2913
2914 /* If the batch of non-matchers is at the end, add on any that were
2915 extracted further up the chain, and end this iteration. Otherwise,
2916 extract them from the chain and hang on the moved chain. */
2917
d6c82d7b 2918 if (!*next)
059ec3d9
PH
2919 {
2920 *next = moved;
2921 break;
2922 }
2923
2924 *bptr = *aptr;
2925 *aptr = *next;
2926 *next = NULL;
2927 bptr = next;
d6c82d7b 2928 aptr = &(*aptr)->next;
059ec3d9
PH
2929 }
2930
2931 /* If the loop ended because the final address matched, *aptr will
2932 be NULL. Add on to the end any extracted non-matching addresses. If
2933 *aptr is not NULL, the loop ended via "break" when *next is null, that
2934 is, there was a string of non-matching addresses at the end. In this
2935 case the extracted addresses have already been added on the end. */
2936
d6c82d7b 2937 if (!*aptr) *aptr = moved;
059ec3d9
PH
2938 }
2939
2940DEBUG(D_deliver)
2941 {
2942 address_item *addr;
2943 debug_printf("remote addresses after sorting:\n");
d6c82d7b 2944 for (addr = addr_remote; addr; addr = addr->next)
059ec3d9
PH
2945 debug_printf(" %s\n", addr->address);
2946 }
2947}
2948
2949
2950
2951/*************************************************
2952* Read from pipe for remote delivery subprocess *
2953*************************************************/
2954
2955/* This function is called when the subprocess is complete, but can also be
2956called before it is complete, in order to empty a pipe that is full (to prevent
2957deadlock). It must therefore keep track of its progress in the parlist data
2958block.
2959
2960We read the pipe to get the delivery status codes and a possible error message
2961for each address, optionally preceded by unusability data for the hosts and
2962also by optional retry data.
2963
2964Read in large chunks into the big buffer and then scan through, interpreting
2965the data therein. In most cases, only a single read will be necessary. No
1c5466b9
PH
2966individual item will ever be anywhere near 2500 bytes in length, so by ensuring
2967that we read the next chunk when there is less than 2500 bytes left in the
2968non-final chunk, we can assume each item is complete in the buffer before
2969handling it. Each item is written using a single write(), which is atomic for
2970small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
2971often bigger) so even if we are reading while the subprocess is still going, we
2972should never have only a partial item in the buffer.
059ec3d9
PH
2973
2974Argument:
2975 poffset the offset of the parlist item
2976 eop TRUE if the process has completed
2977
2978Returns: TRUE if the terminating 'Z' item has been read,
2979 or there has been a disaster (i.e. no more data needed);
2980 FALSE otherwise
2981*/
2982
2983static BOOL
2984par_read_pipe(int poffset, BOOL eop)
2985{
2986host_item *h;
2987pardata *p = parlist + poffset;
2988address_item *addrlist = p->addrlist;
2989address_item *addr = p->addr;
2990pid_t pid = p->pid;
2991int fd = p->fd;
2992uschar *endptr = big_buffer;
2993uschar *ptr = endptr;
2994uschar *msg = p->msg;
2995BOOL done = p->done;
2996BOOL unfinished = TRUE;
bd21a787
WB
2997/* minimum size to read is header size including id, subid and length */
2998int required = PIPE_HEADER_SIZE;
059ec3d9
PH
2999
3000/* Loop through all items, reading from the pipe when necessary. The pipe
3001is set up to be non-blocking, but there are two different Unix mechanisms in
3002use. Exim uses O_NONBLOCK if it is defined. This returns 0 for end of file,
3003and EAGAIN for no more data. If O_NONBLOCK is not defined, Exim uses O_NDELAY,
3004which returns 0 for both end of file and no more data. We distinguish the
3005two cases by taking 0 as end of file only when we know the process has
3006completed.
3007
3008Each separate item is written to the pipe in a single write(), and as they are
3009all short items, the writes will all be atomic and we should never find
8e669ac1
PH
3010ourselves in the position of having read an incomplete item. "Short" in this
3011case can mean up to about 1K in the case when there is a long error message
1c5466b9 3012associated with an address. */
059ec3d9
PH
3013
3014DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
3015 (int)p->pid, eop? "ended" : "not ended");
3016
3017while (!done)
3018 {
3019 retry_item *r, **rp;
3020 int remaining = endptr - ptr;
bd21a787
WB
3021 uschar header[PIPE_HEADER_SIZE + 1];
3022 uschar id, subid;
3023 uschar *endc;
059ec3d9
PH
3024
3025 /* Read (first time) or top up the chars in the buffer if necessary.
3026 There will be only one read if we get all the available data (i.e. don't
3027 fill the buffer completely). */
3028
bd21a787 3029 if (remaining < required && unfinished)
059ec3d9
PH
3030 {
3031 int len;
3032 int available = big_buffer_size - remaining;
3033
3034 if (remaining > 0) memmove(big_buffer, ptr, remaining);
3035
3036 ptr = big_buffer;
3037 endptr = big_buffer + remaining;
3038 len = read(fd, endptr, available);
3039
3040 DEBUG(D_deliver) debug_printf("read() yielded %d\n", len);
3041
3042 /* If the result is EAGAIN and the process is not complete, just
3043 stop reading any more and process what we have already. */
3044
3045 if (len < 0)
3046 {
3047 if (!eop && errno == EAGAIN) len = 0; else
3048 {
3049 msg = string_sprintf("failed to read pipe from transport process "
3050 "%d for transport %s: %s", pid, addr->transport->driver_name,
3051 strerror(errno));
3052 break;
3053 }
3054 }
3055
3056 /* If the length is zero (eof or no-more-data), just process what we
3057 already have. Note that if the process is still running and we have
3058 read all the data in the pipe (but less that "available") then we
3059 won't read any more, as "unfinished" will get set FALSE. */
3060
3061 endptr += len;
bd21a787 3062 remaining += len;
059ec3d9
PH
3063 unfinished = len == available;
3064 }
3065
3066 /* If we are at the end of the available data, exit the loop. */
059ec3d9
PH
3067 if (ptr >= endptr) break;
3068
bd21a787
WB
3069 /* copy and read header */
3070 memcpy(header, ptr, PIPE_HEADER_SIZE);
94431adb 3071 header[PIPE_HEADER_SIZE] = '\0';
bd21a787
WB
3072 id = header[0];
3073 subid = header[1];
3074 required = Ustrtol(header + 2, &endc, 10) + PIPE_HEADER_SIZE; /* header + data */
3075 if (*endc)
3076 {
3077 msg = string_sprintf("failed to read pipe from transport process "
3078 "%d for transport %s: error reading size from header", pid, addr->transport->driver_name);
3079 done = TRUE;
3080 break;
3081 }
3082
3083 DEBUG(D_deliver)
94431adb 3084 debug_printf("header read id:%c,subid:%c,size:%s,required:%d,remaining:%d,unfinished:%d\n",
bd21a787
WB
3085 id, subid, header+2, required, remaining, unfinished);
3086
3087 /* is there room for the dataset we want to read ? */
3088 if (required > big_buffer_size - PIPE_HEADER_SIZE)
3089 {
3090 msg = string_sprintf("failed to read pipe from transport process "
3091 "%d for transport %s: big_buffer too small! required size=%d buffer size=%d", pid, addr->transport->driver_name,
3092 required, big_buffer_size - PIPE_HEADER_SIZE);
3093 done = TRUE;
3094 break;
3095 }
3096
94431adb 3097 /* we wrote all datasets with atomic write() calls
bd21a787 3098 remaining < required only happens if big_buffer was too small
94431adb 3099 to get all available data from pipe. unfinished has to be true
bd21a787
WB
3100 as well. */
3101 if (remaining < required)
8fa0a4d4 3102 {
bd21a787
WB
3103 if (unfinished)
3104 continue;
8fa0a4d4 3105 msg = string_sprintf("failed to read pipe from transport process "
94431adb 3106 "%d for transport %s: required size=%d > remaining size=%d and unfinished=false",
8fa0a4d4
JH
3107 pid, addr->transport->driver_name, required, remaining);
3108 done = TRUE;
3109 break;
3110 }
bd21a787
WB
3111
3112 /* step behind the header */
3113 ptr += PIPE_HEADER_SIZE;
94431adb 3114
059ec3d9
PH
3115 /* Handle each possible type of item, assuming the complete item is
3116 available in store. */
3117
bd21a787 3118 switch (id)
059ec3d9
PH
3119 {
3120 /* Host items exist only if any hosts were marked unusable. Match
3121 up by checking the IP address. */
3122
3123 case 'H':
d6c82d7b 3124 for (h = addrlist->host_list; h; h = h->next)
059ec3d9 3125 {
d6c82d7b 3126 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
059ec3d9
PH
3127 h->status = ptr[0];
3128 h->why = ptr[1];
3129 }
3130 ptr += 2;
3131 while (*ptr++);
3132 break;
3133
3134 /* Retry items are sent in a preceding R item for each address. This is
3135 kept separate to keep each message short enough to guarantee it won't
3136 be split in the pipe. Hopefully, in the majority of cases, there won't in
3137 fact be any retry items at all.
3138
3139 The complete set of retry items might include an item to delete a
3140 routing retry if there was a previous routing delay. However, routing
3141 retries are also used when a remote transport identifies an address error.
3142 In that case, there may also be an "add" item for the same key. Arrange
3143 that a "delete" item is dropped in favour of an "add" item. */
3144
3145 case 'R':
d6c82d7b 3146 if (!addr) goto ADDR_MISMATCH;
059ec3d9
PH
3147
3148 DEBUG(D_deliver|D_retry)
3149 debug_printf("reading retry information for %s from subprocess\n",
3150 ptr+1);
3151
3152 /* Cut out any "delete" items on the list. */
3153
d6c82d7b 3154 for (rp = &(addr->retries); (r = *rp); rp = &r->next)
059ec3d9
PH
3155 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3156 {
3157 if ((r->flags & rf_delete) == 0) break; /* It was not "delete" */
3158 *rp = r->next; /* Excise a delete item */
3159 DEBUG(D_deliver|D_retry)
3160 debug_printf(" existing delete item dropped\n");
3161 }
059ec3d9
PH
3162
3163 /* We want to add a delete item only if there is no non-delete item;
3164 however we still have to step ptr through the data. */
3165
d6c82d7b 3166 if (!r || (*ptr & rf_delete) == 0)
059ec3d9
PH
3167 {
3168 r = store_get(sizeof(retry_item));
3169 r->next = addr->retries;
3170 addr->retries = r;
3171 r->flags = *ptr++;
3172 r->key = string_copy(ptr);
3173 while (*ptr++);
3174 memcpy(&(r->basic_errno), ptr, sizeof(r->basic_errno));
3175 ptr += sizeof(r->basic_errno);
3176 memcpy(&(r->more_errno), ptr, sizeof(r->more_errno));
3177 ptr += sizeof(r->more_errno);
3178 r->message = (*ptr)? string_copy(ptr) : NULL;
3179 DEBUG(D_deliver|D_retry)
3180 debug_printf(" added %s item\n",
3181 ((r->flags & rf_delete) == 0)? "retry" : "delete");
3182 }
3183
3184 else
3185 {
3186 DEBUG(D_deliver|D_retry)
3187 debug_printf(" delete item not added: non-delete item exists\n");
3188 ptr++;
3189 while(*ptr++);
3190 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3191 }
3192
3193 while(*ptr++);
3194 break;
3195
3196 /* Put the amount of data written into the parlist block */
3197
3198 case 'S':
3199 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3200 ptr += sizeof(transport_count);
3201 break;
3202
3203 /* Address items are in the order of items on the address chain. We
3204 remember the current address value in case this function is called
3205 several times to empty the pipe in stages. Information about delivery
3206 over TLS is sent in a preceding X item for each address. We don't put
3207 it in with the other info, in order to keep each message short enough to
3208 guarantee it won't be split in the pipe. */
3209
a7538db1 3210#ifdef SUPPORT_TLS
059ec3d9 3211 case 'X':
d6c82d7b 3212 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
bd21a787 3213 switch (subid)
9d1c15ef
JH
3214 {
3215 case '1':
3216 addr->cipher = NULL;
3217 addr->peerdn = NULL;
3218
3219 if (*ptr)
3220 addr->cipher = string_copy(ptr);
3221 while (*ptr++);
3222 if (*ptr)
9d1c15ef 3223 addr->peerdn = string_copy(ptr);
9d1c15ef
JH
3224 break;
3225
3226 case '2':
9d1c15ef
JH
3227 if (*ptr)
3228 (void) tls_import_cert(ptr, &addr->peercert);
152e7604
JH
3229 else
3230 addr->peercert = NULL;
9d1c15ef
JH
3231 break;
3232
3233 case '3':
9d1c15ef
JH
3234 if (*ptr)
3235 (void) tls_import_cert(ptr, &addr->ourcert);
152e7604
JH
3236 else
3237 addr->ourcert = NULL;
9d1c15ef 3238 break;
018058b2 3239
a7538db1 3240# ifndef DISABLE_OCSP
018058b2
JH
3241 case '4':
3242 addr->ocsp = OCSP_NOT_REQ;
3243 if (*ptr)
3244 addr->ocsp = *ptr - '0';
3245 break;
a7538db1 3246# endif
9d1c15ef 3247 }
059ec3d9
PH
3248 while (*ptr++);
3249 break;
a7538db1 3250#endif /*SUPPORT_TLS*/
059ec3d9 3251
6f123593 3252 case 'C': /* client authenticator information */
bd21a787 3253 switch (subid)
9d1c15ef
JH
3254 {
3255 case '1':
3256 addr->authenticator = (*ptr)? string_copy(ptr) : NULL;
3257 break;
3258 case '2':
3259 addr->auth_id = (*ptr)? string_copy(ptr) : NULL;
3260 break;
3261 case '3':
3262 addr->auth_sndr = (*ptr)? string_copy(ptr) : NULL;
3263 break;
3264 }
6f123593
JH
3265 while (*ptr++);
3266 break;
3267
8ccd00b1 3268#ifndef DISABLE_PRDR
fd98a5c6 3269 case 'P':
8ccd00b1
JH
3270 addr->flags |= af_prdr_used;
3271 break;
fd98a5c6
JH
3272#endif
3273
6c1c3d1d 3274 case 'D':
d6c82d7b 3275 if (!addr) goto ADDR_MISMATCH;
6c1c3d1d
WB
3276 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3277 ptr += sizeof(addr->dsn_aware);
3278 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
3279 break;
6c1c3d1d 3280
059ec3d9 3281 case 'A':
d6c82d7b 3282 if (!addr)
059ec3d9
PH
3283 {
3284 ADDR_MISMATCH:
3285 msg = string_sprintf("address count mismatch for data read from pipe "
3286 "for transport process %d for transport %s", pid,
3287 addrlist->transport->driver_name);
3288 done = TRUE;
3289 break;
3290 }
3291
895fbaf2 3292 switch (subid)
059ec3d9 3293 {
895fbaf2
JH
3294#ifdef EXPERIMENTAL_DSN_INFO
3295 case '1': /* must arrive before A0, and applies to that addr */
3296 /* Two strings: smtp_greeting and helo_response */
3297 addr->smtp_greeting = string_copy(ptr);
3298 while(*ptr++);
3299 addr->helo_response = string_copy(ptr);
3300 while(*ptr++);
3301 break;
3302#endif
059ec3d9 3303
895fbaf2
JH
3304 case '0':
3305 addr->transport_return = *ptr++;
3306 addr->special_action = *ptr++;
3307 memcpy(&(addr->basic_errno), ptr, sizeof(addr->basic_errno));
3308 ptr += sizeof(addr->basic_errno);
3309 memcpy(&(addr->more_errno), ptr, sizeof(addr->more_errno));
3310 ptr += sizeof(addr->more_errno);
3311 memcpy(&(addr->flags), ptr, sizeof(addr->flags));
3312 ptr += sizeof(addr->flags);
3313 addr->message = (*ptr)? string_copy(ptr) : NULL;
3314 while(*ptr++);
3315 addr->user_message = (*ptr)? string_copy(ptr) : NULL;
3316 while(*ptr++);
059ec3d9 3317
895fbaf2
JH
3318 /* Always two strings for host information, followed by the port number and DNSSEC mark */
3319
3320 if (*ptr != 0)
3321 {
3322 h = store_get(sizeof(host_item));
3323 h->name = string_copy(ptr);
3324 while (*ptr++);
3325 h->address = string_copy(ptr);
3326 while(*ptr++);
3327 memcpy(&(h->port), ptr, sizeof(h->port));
3328 ptr += sizeof(h->port);
3329 h->dnssec = *ptr == '2' ? DS_YES
3330 : *ptr == '1' ? DS_NO
3331 : DS_UNK;
3332 ptr++;
3333 addr->host_used = h;
3334 }
3335 else ptr++;
3336
3337 /* Finished with this address */
3338
3339 addr = addr->next;
3340 break;
3341 }
059ec3d9
PH
3342 break;
3343
aeaf5db3
JH
3344 /* Local interface address/port */
3345 case 'I':
3346 if (*ptr) sending_ip_address = string_copy(ptr);
3347 while (*ptr++) ;
93a6fce2 3348 if (*ptr) sending_port = atoi(CS ptr);
aeaf5db3
JH
3349 while (*ptr++) ;
3350 break;
3351
059ec3d9
PH
3352 /* Z marks the logical end of the data. It is followed by '0' if
3353 continue_transport was NULL at the end of transporting, otherwise '1'.
3354 We need to know when it becomes NULL during a delivery down a passed SMTP
3355 channel so that we don't try to pass anything more down it. Of course, for
3356 most normal messages it will remain NULL all the time. */
3357
3358 case 'Z':
3359 if (*ptr == '0')
3360 {
3361 continue_transport = NULL;
3362 continue_hostname = NULL;
3363 }
3364 done = TRUE;
bd21a787 3365 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
059ec3d9
PH
3366 break;
3367
3368 /* Anything else is a disaster. */
3369
3370 default:
3371 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3372 "process %d for transport %s", ptr[-1], pid,
3373 addr->transport->driver_name);
3374 done = TRUE;
3375 break;
3376 }
3377 }
3378
3379/* The done flag is inspected externally, to determine whether or not to
3380call the function again when the process finishes. */
3381
3382p->done = done;
3383
3384/* If the process hadn't finished, and we haven't seen the end of the data
3385or suffered a disaster, update the rest of the state, and return FALSE to
3386indicate "not finished". */
3387
3388if (!eop && !done)
3389 {
3390 p->addr = addr;
3391 p->msg = msg;
3392 return FALSE;
3393 }
3394
3395/* Close our end of the pipe, to prevent deadlock if the far end is still
3396pushing stuff into it. */
3397
f1e894f3 3398(void)close(fd);
059ec3d9
PH
3399p->fd = -1;
3400
3401/* If we have finished without error, but haven't had data for every address,
3402something is wrong. */
3403
d6c82d7b 3404if (!msg && addr)
059ec3d9
PH
3405 msg = string_sprintf("insufficient address data read from pipe "
3406 "for transport process %d for transport %s", pid,
3407 addr->transport->driver_name);
3408
3409/* If an error message is set, something has gone wrong in getting back
3410the delivery data. Put the message into each address and freeze it. */
3411
d6c82d7b
JH
3412if (msg)
3413 for (addr = addrlist; addr; addr = addr->next)
059ec3d9
PH
3414 {
3415 addr->transport_return = DEFER;
3416 addr->special_action = SPECIAL_FREEZE;
3417 addr->message = msg;
3418 }
059ec3d9
PH
3419
3420/* Return TRUE to indicate we have got all we need from this process, even
3421if it hasn't actually finished yet. */
3422
3423return TRUE;
3424}
3425
3426
3427
3428/*************************************************
3429* Post-process a set of remote addresses *
3430*************************************************/
3431
3432/* Do what has to be done immediately after a remote delivery for each set of
3433addresses, then re-write the spool if necessary. Note that post_process_one
3434puts the address on an appropriate queue; hence we must fish off the next
3435one first. This function is also called if there is a problem with setting
3436up a subprocess to do a remote delivery in parallel. In this case, the final
3437argument contains a message, and the action must be forced to DEFER.
3438
3439Argument:
3440 addr pointer to chain of address items
3441 logflags flags for logging
3442 msg NULL for normal cases; -> error message for unexpected problems
3443 fallback TRUE if processing fallback hosts
3444
3445Returns: nothing
3446*/
3447
3448static void
3449remote_post_process(address_item *addr, int logflags, uschar *msg,
3450 BOOL fallback)
3451{
3452host_item *h;
3453
3454/* If any host addresses were found to be unusable, add them to the unusable
3455tree so that subsequent deliveries don't try them. */
3456
d6c82d7b
JH
3457for (h = addr->host_list; h; h = h->next)
3458 if (h->address)
3459 if (h->status >= hstatus_unusable) tree_add_unusable(h);
059ec3d9
PH
3460
3461/* Now handle each address on the chain. The transport has placed '=' or '-'
3462into the special_action field for each successful delivery. */
3463
d6c82d7b 3464while (addr)
059ec3d9
PH
3465 {
3466 address_item *next = addr->next;
3467
3468 /* If msg == NULL (normal processing) and the result is DEFER and we are
3469 processing the main hosts and there are fallback hosts available, put the
3470 address on the list for fallback delivery. */
3471
d6c82d7b
JH
3472 if ( addr->transport_return == DEFER
3473 && addr->fallback_hosts
3474 && !fallback
3475 && !msg
3476 )
059ec3d9
PH
3477 {
3478 addr->host_list = addr->fallback_hosts;
3479 addr->next = addr_fallback;
3480 addr_fallback = addr;
3481 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3482 }
3483
3484 /* If msg is set (=> unexpected problem), set it in the address before
3485 doing the ordinary post processing. */
3486
3487 else
3488 {
d6c82d7b 3489 if (msg)
059ec3d9
PH
3490 {
3491 addr->message = msg;
3492 addr->transport_return = DEFER;
3493 }
3494 (void)post_process_one(addr, addr->transport_return, logflags,
3495 DTYPE_TRANSPORT, addr->special_action);
3496 }
3497
3498 /* Next address */
3499
3500 addr = next;
3501 }
3502
3503/* If we have just delivered down a passed SMTP channel, and that was
3504the last address, the channel will have been closed down. Now that
3505we have logged that delivery, set continue_sequence to 1 so that
3506any subsequent deliveries don't get "*" incorrectly logged. */
3507
d6c82d7b 3508if (!continue_transport) continue_sequence = 1;
059ec3d9
PH
3509}
3510
3511
3512
3513/*************************************************
3514* Wait for one remote delivery subprocess *
3515*************************************************/
3516
3517/* This function is called while doing remote deliveries when either the
3518maximum number of processes exist and we need one to complete so that another
3519can be created, or when waiting for the last ones to complete. It must wait for
3520the completion of one subprocess, empty the control block slot, and return a
3521pointer to the address chain.
3522
3523Arguments: none
3524Returns: pointer to the chain of addresses handled by the process;
3525 NULL if no subprocess found - this is an unexpected error
3526*/
3527
3528static address_item *
3529par_wait(void)
3530{
3531int poffset, status;
3532address_item *addr, *addrlist;
3533pid_t pid;
3534
3535set_process_info("delivering %s: waiting for a remote delivery subprocess "
3536 "to finish", message_id);
3537
3538/* Loop until either a subprocess completes, or there are no subprocesses in
3539existence - in which case give an error return. We cannot proceed just by
3540waiting for a completion, because a subprocess may have filled up its pipe, and
3541be waiting for it to be emptied. Therefore, if no processes have finished, we
3542wait for one of the pipes to acquire some data by calling select(), with a
3543timeout just in case.
3544
3545The simple approach is just to iterate after reading data from a ready pipe.
3546This leads to non-ideal behaviour when the subprocess has written its final Z
3547item, closed the pipe, and is in the process of exiting (the common case). A
3548call to waitpid() yields nothing completed, but select() shows the pipe ready -
3549reading it yields EOF, so you end up with busy-waiting until the subprocess has
3550actually finished.
3551
3552To avoid this, if all the data that is needed has been read from a subprocess
3553after select(), an explicit wait() for it is done. We know that all it is doing
3554is writing to the pipe and then exiting, so the wait should not be long.
3555
3556The non-blocking waitpid() is to some extent just insurance; if we could
3557reliably detect end-of-file on the pipe, we could always know when to do a
3558blocking wait() for a completed process. However, because some systems use
3559NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3560use code that functions without the need to recognize EOF.
3561
3562There's a double loop here just in case we end up with a process that is not in
3563the list of remote delivery processes. Something has obviously gone wrong if
3564this is the case. (For example, a process that is incorrectly left over from
3565routing or local deliveries might be found.) The damage can be minimized by
3566looping back and looking for another process. If there aren't any, the error
3567return will happen. */
3568
3569for (;;) /* Normally we do not repeat this loop */
3570 {
3571 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3572 {
3573 struct timeval tv;
3574 fd_set select_pipes;
3575 int maxpipe, readycount;
3576
3577 /* A return value of -1 can mean several things. If errno != ECHILD, it
3578 either means invalid options (which we discount), or that this process was
3579 interrupted by a signal. Just loop to try the waitpid() again.
3580
3581 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3582 in existence. This should never happen, and is an unexpected error.
3583 However, there is a nasty complication when running under Linux. If "strace
3584 -f" is being used under Linux to trace this process and its children,
3585 subprocesses are "stolen" from their parents and become the children of the
3586 tracing process. A general wait such as the one we've just obeyed returns
3587 as if there are no children while subprocesses are running. Once a
3588 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3589 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3590 palliative.
3591
3592 This does not happen using "truss" on Solaris, nor (I think) with other
3593 tracing facilities on other OS. It seems to be specific to Linux.
3594
3595 What we do to get round this is to use kill() to see if any of our
3596 subprocesses are still in existence. If kill() gives an OK return, we know
3597 it must be for one of our processes - it can't be for a re-use of the pid,
3598 because if our process had finished, waitpid() would have found it. If any
3599 of our subprocesses are in existence, we proceed to use select() as if
3600 waitpid() had returned zero. I think this is safe. */
3601
3602 if (pid < 0)
3603 {
3604 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3605
3606 DEBUG(D_deliver)
3607 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3608 "for process existence\n");
3609
3610 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3611 {
3612 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3613 {
3614 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3615 "stolen by strace\n", (int)pid);
3616 break; /* With poffset set */
3617 }
3618 }
3619
3620 if (poffset >= remote_max_parallel)
3621 {
3622 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3623 return NULL; /* This is the error return */
3624 }
3625 }
3626
3627 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3628 been handled above. A return value of zero means that there is at least one
3629 subprocess, but there are no completed subprocesses. See if any pipes are
3630 ready with any data for reading. */
3631
3632 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3633
3634 maxpipe = 0;
3635 FD_ZERO(&select_pipes);
3636 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3637 {
3638 if (parlist[poffset].pid != 0)
3639 {
3640 int fd = parlist[poffset].fd;
3641 FD_SET(fd, &select_pipes);
3642 if (fd > maxpipe) maxpipe = fd;
3643 }
3644 }
3645
3646 /* Stick in a 60-second timeout, just in case. */
3647
3648 tv.tv_sec = 60;
3649 tv.tv_usec = 0;
3650
3651 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3652 NULL, NULL, &tv);
3653
3654 /* Scan through the pipes and read any that are ready; use the count
3655 returned by select() to stop when there are no more. Select() can return
3656 with no processes (e.g. if interrupted). This shouldn't matter.
3657
3658 If par_read_pipe() returns TRUE, it means that either the terminating Z was
3659 read, or there was a disaster. In either case, we are finished with this
3660 process. Do an explicit wait() for the process and break the main loop if
3661 it succeeds.
3662
3663 It turns out that we have to deal with the case of an interrupted system
3664 call, which can happen on some operating systems if the signal handling is
3665 set up to do that by default. */
3666
3667 for (poffset = 0;
3668 readycount > 0 && poffset < remote_max_parallel;
3669 poffset++)
3670 {
d6c82d7b
JH
3671 if ( (pid = parlist[poffset].pid) != 0
3672 && FD_ISSET(parlist[poffset].fd, &