max_parallel transport option. Bug 1698
[exim.git] / src / src / deliver.c
</
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
3386088d 5/* Copyright (c) University of Cambridge 1995 - 2015 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* The main code for delivering a message. */
9
10
11#include "exim.h"
dadff1d4 12#include <assert.h>
059ec3d9
PH
13
14
15/* Data block for keeping track of subprocesses for parallel remote
16delivery. */
17
18typedef struct pardata {
19 address_item *addrlist; /* chain of addresses */
20 address_item *addr; /* next address data expected for */
21 pid_t pid; /* subprocess pid */
22 int fd; /* pipe fd for getting result from subprocess */
23 int transport_count; /* returned transport count value */
24 BOOL done; /* no more data needed */
25 uschar *msg; /* error message */
26 uschar *return_path; /* return_path for these addresses */
27} pardata;
28
29/* Values for the process_recipients variable */
30
31enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
32 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
33 RECIP_FAIL_LOOP};
34
35/* Mutually recursive functions for marking addresses done. */
36
37static void child_done(address_item *, uschar *);
38static void address_done(address_item *, uschar *);
39
40/* Table for turning base-62 numbers into binary */
41
42static uschar tab62[] =
43 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
44 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
45 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
46 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
47 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
48 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
49 59,60,61}; /* x-z */
50
51
52/*************************************************
53* Local static variables *
54*************************************************/
55
56/* addr_duplicate is global because it needs to be seen from the Envelope-To
57writing code. */
58
59static address_item *addr_defer = NULL;
60static address_item *addr_failed = NULL;
61static address_item *addr_fallback = NULL;
62static address_item *addr_local = NULL;
63static address_item *addr_new = NULL;
64static address_item *addr_remote = NULL;
65static address_item *addr_route = NULL;
66static address_item *addr_succeed = NULL;
6c1c3d1d
WB
67static address_item *addr_dsntmp = NULL;
68static address_item *addr_senddsn = NULL;
059ec3d9
PH
69
70static FILE *message_log = NULL;
71static BOOL update_spool;
72static BOOL remove_journal;
73static int parcount = 0;
74static pardata *parlist = NULL;
75static int return_count;
76static uschar *frozen_info = US"";
77static uschar *used_return_path = NULL;
78
79static uschar spoolname[PATH_MAX];
80
81
82
83/*************************************************
84* Make a new address item *
85*************************************************/
86
87/* This function gets the store and initializes with default values. The
88transport_return value defaults to DEFER, so that any unexpected failure to
89deliver does not wipe out the message. The default unique string is set to a
90copy of the address, so that its domain can be lowercased.
91
92Argument:
93 address the RFC822 address string
94 copy force a copy of the address
95
96Returns: a pointer to an initialized address_item
97*/
98
99address_item *
100deliver_make_addr(uschar *address, BOOL copy)
101{
102address_item *addr = store_get(sizeof(address_item));
103*addr = address_defaults;
104if (copy) address = string_copy(address);
105addr->address = address;
106addr->unique = string_copy(address);
107return addr;
108}
109
110
111
112
113/*************************************************
114* Set expansion values for an address *
115*************************************************/
116
117/* Certain expansion variables are valid only when handling an address or
118address list. This function sets them up or clears the values, according to its
119argument.
120
121Arguments:
122 addr the address in question, or NULL to clear values
123Returns: nothing
124*/
125
126void
127deliver_set_expansions(address_item *addr)
128{
d6c82d7b 129if (!addr)
059ec3d9 130 {
55414b25 131 const uschar ***p = address_expansions;
d6c82d7b 132 while (*p) **p++ = NULL;
059ec3d9
PH
133 return;
134 }
135
136/* Exactly what gets set depends on whether there is one or more addresses, and
137what they contain. These first ones are always set, taking their values from
138the first address. */
139
d6c82d7b 140if (!addr->host_list)
059ec3d9
PH
141 {
142 deliver_host = deliver_host_address = US"";
a7538db1 143 deliver_host_port = 0;
059ec3d9
PH
144 }
145else
146 {
147 deliver_host = addr->host_list->name;
148 deliver_host_address = addr->host_list->address;
a7538db1 149 deliver_host_port = addr->host_list->port;
059ec3d9
PH
150 }
151
152deliver_recipients = addr;
d43cbe25
JH
153deliver_address_data = addr->prop.address_data;
154deliver_domain_data = addr->prop.domain_data;
155deliver_localpart_data = addr->prop.localpart_data;
059ec3d9
PH
156
157/* These may be unset for multiple addresses */
158
159deliver_domain = addr->domain;
160self_hostname = addr->self_hostname;
161
8523533c
TK
162#ifdef EXPERIMENTAL_BRIGHTMAIL
163bmi_deliver = 1; /* deliver by default */
164bmi_alt_location = NULL;
165bmi_base64_verdict = NULL;
166bmi_base64_tracker_verdict = NULL;
167#endif
168
059ec3d9
PH
169/* If there's only one address we can set everything. */
170
d6c82d7b 171if (!addr->next)
059ec3d9
PH
172 {
173 address_item *addr_orig;
174
175 deliver_localpart = addr->local_part;
176 deliver_localpart_prefix = addr->prefix;
177 deliver_localpart_suffix = addr->suffix;
178
d6c82d7b 179 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
059ec3d9
PH
180 deliver_domain_orig = addr_orig->domain;
181
182 /* Re-instate any prefix and suffix in the original local part. In all
183 normal cases, the address will have a router associated with it, and we can
184 choose the caseful or caseless version accordingly. However, when a system
185 filter sets up a pipe, file, or autoreply delivery, no router is involved.
186 In this case, though, there won't be any prefix or suffix to worry about. */
187
d6c82d7b
JH
188 deliver_localpart_orig = !addr_orig->router
189 ? addr_orig->local_part
190 : addr_orig->router->caseful_local_part
191 ? addr_orig->cc_local_part
192 : addr_orig->lc_local_part;
059ec3d9
PH
193
194 /* If there's a parent, make its domain and local part available, and if
195 delivering to a pipe or file, or sending an autoreply, get the local
196 part from the parent. For pipes and files, put the pipe or file string
197 into address_pipe and address_file. */
198
d6c82d7b 199 if (addr->parent)
059ec3d9
PH
200 {
201 deliver_domain_parent = addr->parent->domain;
d6c82d7b
JH
202 deliver_localpart_parent = !addr->parent->router
203 ? addr->parent->local_part
204 : addr->parent->router->caseful_local_part
205 ? addr->parent->cc_local_part
206 : addr->parent->lc_local_part;
059ec3d9
PH
207
208 /* File deliveries have their own flag because they need to be picked out
209 as special more often. */
210
211 if (testflag(addr, af_pfr))
212 {
d6c82d7b
JH
213 if (testflag(addr, af_file)) address_file = addr->local_part;
214 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
059ec3d9
PH
215 deliver_localpart = addr->parent->local_part;
216 deliver_localpart_prefix = addr->parent->prefix;
217 deliver_localpart_suffix = addr->parent->suffix;
218 }
219 }
8523533c
TK
220
221#ifdef EXPERIMENTAL_BRIGHTMAIL
222 /* Set expansion variables related to Brightmail AntiSpam */
223 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
224 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
225 /* get message delivery status (0 - don't deliver | 1 - deliver) */
226 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
227 /* if message is to be delivered, get eventual alternate location */
d6c82d7b 228 if (bmi_deliver == 1)
8523533c 229 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
8523533c
TK
230#endif
231
059ec3d9
PH
232 }
233
234/* For multiple addresses, don't set local part, and leave the domain and
f7fd3850
PH
235self_hostname set only if it is the same for all of them. It is possible to
236have multiple pipe and file addresses, but only when all addresses have routed
237to the same pipe or file. */
059ec3d9
PH
238
239else
240 {
241 address_item *addr2;
f7fd3850
PH
242 if (testflag(addr, af_pfr))
243 {
d6c82d7b
JH
244 if (testflag(addr, af_file)) address_file = addr->local_part;
245 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
f7fd3850 246 }
d6c82d7b 247 for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 248 {
d6c82d7b 249 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
059ec3d9 250 deliver_domain = NULL;
d6c82d7b
JH
251 if ( self_hostname
252 && ( !addr2->self_hostname
253 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
254 ) )
059ec3d9 255 self_hostname = NULL;
d6c82d7b 256 if (!deliver_domain && !self_hostname) break;
059ec3d9
PH
257 }
258 }
259}
260
261
262
263
264/*************************************************
265* Open a msglog file *
266*************************************************/
267
268/* This function is used both for normal message logs, and for files in the
269msglog directory that are used to catch output from pipes. Try to create the
270directory if it does not exist. From release 4.21, normal message logs should
271be created when the message is received.
272
273Argument:
274 filename the file name
275 mode the mode required
276 error used for saying what failed
277
278Returns: a file descriptor, or -1 (with errno set)
279*/
280
281static int
282open_msglog_file(uschar *filename, int mode, uschar **error)
283{
284int fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
285
286if (fd < 0 && errno == ENOENT)
287 {
288 uschar temp[16];
289 sprintf(CS temp, "msglog/%s", message_subdir);
290 if (message_subdir[0] == 0) temp[6] = 0;
291 (void)directory_make(spool_directory, temp, MSGLOG_DIRECTORY_MODE, TRUE);
292 fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
293 }
294
295/* Set the close-on-exec flag and change the owner to the exim uid/gid (this
296function is called as root). Double check the mode, because the group setting
297doesn't always get set automatically. */
298
299if (fd >= 0)
300 {
ff790e47 301 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
059ec3d9
PH
302 if (fchown(fd, exim_uid, exim_gid) < 0)
303 {
304 *error = US"chown";
305 return -1;
306 }
307 if (fchmod(fd, mode) < 0)
308 {
309 *error = US"chmod";
310 return -1;
311 }
312 }
313else *error = US"create";
314
315return fd;
316}
317
318
319
320
321/*************************************************
322* Write to msglog if required *
323*************************************************/
324
325/* Write to the message log, if configured. This function may also be called
326from transports.
327
328Arguments:
329 format a string format
330
331Returns: nothing
332*/
333
334void
335deliver_msglog(const char *format, ...)
336{
337va_list ap;
338if (!message_logs) return;
339va_start(ap, format);
340vfprintf(message_log, format, ap);
341fflush(message_log);
342va_end(ap);
343}
344
345
346
347
348/*************************************************
349* Replicate status for batch *
350*************************************************/
351
352/* When a transport handles a batch of addresses, it may treat them
353individually, or it may just put the status in the first one, and return FALSE,
354requesting that the status be copied to all the others externally. This is the
355replication function. As well as the status, it copies the transport pointer,
356which may have changed if appendfile passed the addresses on to a different
357transport.
358
359Argument: pointer to the first address in a chain
360Returns: nothing
361*/
362
363static void
364replicate_status(address_item *addr)
365{
366address_item *addr2;
d6c82d7b 367for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 368 {
d6c82d7b 369 addr2->transport = addr->transport;
059ec3d9 370 addr2->transport_return = addr->transport_return;
d6c82d7b
JH
371 addr2->basic_errno = addr->basic_errno;
372 addr2->more_errno = addr->more_errno;
373 addr2->special_action = addr->special_action;
374 addr2->message = addr->message;
375 addr2->user_message = addr->user_message;
059ec3d9
PH
376 }
377}
378
379
380
381/*************************************************
382* Compare lists of hosts *
383*************************************************/
384
385/* This function is given two pointers to chains of host items, and it yields
386TRUE if the lists refer to the same hosts in the same order, except that
387
388(1) Multiple hosts with the same non-negative MX values are permitted to appear
389 in different orders. Round-robinning nameservers can cause this to happen.
390
391(2) Multiple hosts with the same negative MX values less than MX_NONE are also
392 permitted to appear in different orders. This is caused by randomizing
393 hosts lists.
394
395This enables Exim to use a single SMTP transaction for sending to two entirely
396different domains that happen to end up pointing at the same hosts.
397
398Arguments:
399 one points to the first host list
400 two points to the second host list
401
402Returns: TRUE if the lists refer to the same host set
403*/
404
405static BOOL
406same_hosts(host_item *one, host_item *two)
407{
d6c82d7b 408while (one && two)
059ec3d9
PH
409 {
410 if (Ustrcmp(one->name, two->name) != 0)
411 {
412 int mx = one->mx;
413 host_item *end_one = one;
414 host_item *end_two = two;
415
416 /* Batch up only if there was no MX and the list was not randomized */
417
418 if (mx == MX_NONE) return FALSE;
419
420 /* Find the ends of the shortest sequence of identical MX values */
421
d6c82d7b
JH
422 while ( end_one->next && end_one->next->mx == mx
423 && end_two->next && end_two->next->mx == mx)
059ec3d9
PH
424 {
425 end_one = end_one->next;
426 end_two = end_two->next;
427 }
428
429 /* If there aren't any duplicates, there's no match. */
430
431 if (end_one == one) return FALSE;
432
433 /* For each host in the 'one' sequence, check that it appears in the 'two'
434 sequence, returning FALSE if not. */
435
436 for (;;)
437 {
438 host_item *hi;
439 for (hi = two; hi != end_two->next; hi = hi->next)
440 if (Ustrcmp(one->name, hi->name) == 0) break;
441 if (hi == end_two->next) return FALSE;
442 if (one == end_one) break;
443 one = one->next;
444 }
445
446 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
447 Ensure both are pointing at the last host, and carry on as for equality. */
448
449 two = end_two;
450 }
451
452 /* Hosts matched */
453
454 one = one->next;
455 two = two->next;
456 }
457
458/* True if both are NULL */
459
460return (one == two);
461}
462
463
464
465/*************************************************
466* Compare header lines *
467*************************************************/
468
469/* This function is given two pointers to chains of header items, and it yields
470TRUE if they are the same header texts in the same order.
471
472Arguments:
473 one points to the first header list
474 two points to the second header list
475
476Returns: TRUE if the lists refer to the same header set
477*/
478
479static BOOL
480same_headers(header_line *one, header_line *two)
481{
d6c82d7b 482for (;; one = one->next, two = two->next)
059ec3d9
PH
483 {
484 if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 485 if (!one || !two) return FALSE;
059ec3d9 486 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
059ec3d9
PH
487 }
488}
489
490
491
492/*************************************************
493* Compare string settings *
494*************************************************/
495
496/* This function is given two pointers to strings, and it returns
497TRUE if they are the same pointer, or if the two strings are the same.
498
499Arguments:
500 one points to the first string
501 two points to the second string
502
503Returns: TRUE or FALSE
504*/
505
506static BOOL
507same_strings(uschar *one, uschar *two)
508{
509if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 510if (!one || !two) return FALSE;
059ec3d9
PH
511return (Ustrcmp(one, two) == 0);
512}
513
514
515
516/*************************************************
517* Compare uid/gid for addresses *
518*************************************************/
519
520/* This function is given a transport and two addresses. It yields TRUE if the
521uid/gid/initgroups settings for the two addresses are going to be the same when
522they are delivered.
523
524Arguments:
525 tp the transort
526 addr1 the first address
527 addr2 the second address
528
529Returns: TRUE or FALSE
530*/
531
532static BOOL
533same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
534{
d6c82d7b
JH
535if ( !tp->uid_set && !tp->expand_uid
536 && !tp->deliver_as_creator
537 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
538 || ( testflag(addr1, af_uid_set)
539 && ( addr1->uid != addr2->uid
540 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
541 ) ) ) )
542 return FALSE;
059ec3d9 543
d6c82d7b
JH
544if ( !tp->gid_set && !tp->expand_gid
545 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
546 || ( testflag(addr1, af_gid_set)
547 && addr1->gid != addr2->gid
548 ) ) )
549 return FALSE;
059ec3d9
PH
550
551return TRUE;
552}
553
554
555
556
557/*************************************************
558* Record that an address is complete *
559*************************************************/
560
561/* This function records that an address is complete. This is straightforward
562for most addresses, where the unique address is just the full address with the
563domain lower cased. For homonyms (addresses that are the same as one of their
564ancestors) their are complications. Their unique addresses have \x\ prepended
565(where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
566cousins.
567
568Exim used to record the unique addresses of homonyms as "complete". This,
569however, fails when the pattern of redirection varies over time (e.g. if taking
570unseen copies at only some times of day) because the prepended numbers may vary
571from one delivery run to the next. This problem is solved by never recording
572prepended unique addresses as complete. Instead, when a homonymic address has
573actually been delivered via a transport, we record its basic unique address
574followed by the name of the transport. This is checked in subsequent delivery
575runs whenever an address is routed to a transport.
576
577If the completed address is a top-level one (has no parent, which means it
578cannot be homonymic) we also add the original address to the non-recipients
579tree, so that it gets recorded in the spool file and therefore appears as
580"done" in any spool listings. The original address may differ from the unique
581address in the case of the domain.
582
583Finally, this function scans the list of duplicates, marks as done any that
584match this address, and calls child_done() for their ancestors.
585
586Arguments:
587 addr address item that has been completed
588 now current time as a string
589
590Returns: nothing
591*/
592
593static void
594address_done(address_item *addr, uschar *now)
595{
596address_item *dup;
597
598update_spool = TRUE; /* Ensure spool gets updated */
599
600/* Top-level address */
601
d6c82d7b 602if (!addr->parent)
059ec3d9
PH
603 {
604 tree_add_nonrecipient(addr->unique);
605 tree_add_nonrecipient(addr->address);
606 }
607
608/* Homonymous child address */
609
610else if (testflag(addr, af_homonym))
611 {
d6c82d7b 612 if (addr->transport)
059ec3d9
PH
613 tree_add_nonrecipient(
614 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
059ec3d9
PH
615 }
616
617/* Non-homonymous child address */
618
619else tree_add_nonrecipient(addr->unique);
620
621/* Check the list of duplicate addresses and ensure they are now marked
622done as well. */
623
d6c82d7b 624for (dup = addr_duplicate; dup; dup = dup->next)
059ec3d9
PH
625 if (Ustrcmp(addr->unique, dup->unique) == 0)
626 {
57730b52 627 tree_add_nonrecipient(dup->unique);
059ec3d9
PH
628 child_done(dup, now);
629 }
059ec3d9
PH
630}
631
632
633
634
635/*************************************************
636* Decrease counts in parents and mark done *
637*************************************************/
638
639/* This function is called when an address is complete. If there is a parent
640address, its count of children is decremented. If there are still other
641children outstanding, the function exits. Otherwise, if the count has become
642zero, address_done() is called to mark the parent and its duplicates complete.
643Then loop for any earlier ancestors.
644
645Arguments:
646 addr points to the completed address item
647 now the current time as a string, for writing to the message log
648
649Returns: nothing
650*/
651
652static void
653child_done(address_item *addr, uschar *now)
654{
655address_item *aa;
d6c82d7b 656while (addr->parent)
059ec3d9
PH
657 {
658 addr = addr->parent;
659 if ((addr->child_count -= 1) > 0) return; /* Incomplete parent */
660 address_done(addr, now);
661
662 /* Log the completion of all descendents only when there is no ancestor with
663 the same original address. */
664
d6c82d7b 665 for (aa = addr->parent; aa; aa = aa->parent)
059ec3d9 666 if (Ustrcmp(aa->address, addr->address) == 0) break;
d6c82d7b 667 if (aa) continue;
059ec3d9
PH
668
669 deliver_msglog("%s %s: children all complete\n", now, addr->address);
670 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
671 }
672}
673
674
675
ac881e27
TF
676/*************************************************
677* Delivery logging support functions *
678*************************************************/
679
680/* The LOGGING() checks in d_log_interface() are complicated for backwards
681compatibility. When outgoing interface logging was originally added, it was
682conditional on just incoming_interface (which is off by default). The
683outgoing_interface option is on by default to preserve this behaviour, but
684you can enable incoming_interface and disable outgoing_interface to get I=
685fields on incoming lines only.
686
687Arguments:
688 s The log line buffer
689 sizep Pointer to the buffer size
690 ptrp Pointer to current index into buffer
691 addr The address to be logged
692
693Returns: New value for s
694*/
059ec3d9 695
67d81c10 696static uschar *
ac881e27 697d_log_interface(uschar *s, int *sizep, int *ptrp)
67d81c10 698{
ac881e27 699if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
d6c82d7b 700 && sending_ip_address)
ac881e27
TF
701 {
702 s = string_append(s, sizep, ptrp, 2, US" I=[", sending_ip_address);
d6c82d7b
JH
703 s = LOGGING(outgoing_port)
704 ? string_append(s, sizep, ptrp, 2, US"]:",
705 string_sprintf("%d", sending_port))
706 : string_cat(s, sizep, ptrp, "]", 1);
ac881e27
TF
707 }
708return s;
67d81c10
JH
709}
710
ac881e27
TF
711
712
713static uschar *
714d_hostlog(uschar *s, int *sizep, int *ptrp, address_item *addr)
715{
716s = string_append(s, sizep, ptrp, 5, US" H=", addr->host_used->name,
717 US" [", addr->host_used->address, US"]");
718if (LOGGING(outgoing_port))
719 s = string_append(s, sizep, ptrp, 2, US":", string_sprintf("%d",
720 addr->host_used->port));
721return d_log_interface(s, sizep, ptrp);
722}
723
724
725
67d81c10
JH
726#ifdef SUPPORT_TLS
727static uschar *
728d_tlslog(uschar * s, int * sizep, int * ptrp, address_item * addr)
729{
d6c82d7b 730if (LOGGING(tls_cipher) && addr->cipher)
ac881e27 731 s = string_append(s, sizep, ptrp, 2, US" X=", addr->cipher);
d6c82d7b 732if (LOGGING(tls_certificate_verified) && addr->cipher)
ac881e27
TF
733 s = string_append(s, sizep, ptrp, 2, US" CV=",
734 testflag(addr, af_cert_verified)
735 ?
53a7196b 736#ifdef EXPERIMENTAL_DANE
ac881e27
TF
737 testflag(addr, af_dane_verified)
738 ? "dane"
739 :
53a7196b 740#endif
ac881e27
TF
741 "yes"
742 : "no");
d6c82d7b 743if (LOGGING(tls_peerdn) && addr->peerdn)
ac881e27
TF
744 s = string_append(s, sizep, ptrp, 3, US" DN=\"",
745 string_printing(addr->peerdn), US"\"");
746return s;
67d81c10
JH
747}
748#endif
749
a7538db1 750
14a465c3
JH
751
752
774ef2d7 753#ifdef EXPERIMENTAL_EVENT
b30275b8 754uschar *
55414b25 755event_raise(uschar * action, const uschar * event, uschar * ev_data)
a7538db1
JH
756{
757uschar * s;
758if (action)
759 {
760 DEBUG(D_deliver)
774ef2d7 761 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
a7538db1
JH
762 event,
763 action, deliver_host_address);
764
774ef2d7
JH
765 event_name = event;
766 event_data = ev_data;
a7538db1
JH
767
768 if (!(s = expand_string(action)) && *expand_string_message)
769 log_write(0, LOG_MAIN|LOG_PANIC,
774ef2d7 770 "failed to expand event_action %s in %s: %s\n",
a7538db1
JH
771 event, transport_name, expand_string_message);
772
774ef2d7 773 event_name = event_data = NULL;
a7538db1
JH
774
775 /* If the expansion returns anything but an empty string, flag for
776 the caller to modify his normal processing
777 */
778 if (s && *s)
779 {
780 DEBUG(D_deliver)
774ef2d7 781 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
b30275b8 782 return s;
a7538db1
JH
783 }
784 }
b30275b8 785return NULL;
a7538db1 786}
14a465c3
JH
787
788static void
55414b25 789msg_event_raise(const uschar * event, const address_item * addr)
14a465c3 790{
55414b25 791const uschar * save_domain = deliver_domain;
14a465c3 792uschar * save_local = deliver_localpart;
55414b25 793const uschar * save_host = deliver_host;
14a465c3
JH
794
795if (!addr->transport)
796 return;
797
798router_name = addr->router ? addr->router->name : NULL;
799transport_name = addr->transport->name;
800deliver_domain = addr->domain;
801deliver_localpart = addr->local_part;
c562fd30 802deliver_host = addr->host_used ? addr->host_used->name : NULL;
14a465c3 803
774ef2d7 804(void) event_raise(addr->transport->event_action, event,
14a465c3
JH
805 addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
806 ? addr->message : NULL);
807
c562fd30 808deliver_host = save_host;
14a465c3
JH
809deliver_localpart = save_local;
810deliver_domain = save_domain;
811router_name = transport_name = NULL;
812}
774ef2d7 813#endif /*EXPERIMENTAL_EVENT*/
14a465c3
JH
814
815
a7538db1 816
817d9f57 817/* If msg is NULL this is a delivery log and logchar is used. Otherwise
67d81c10 818this is a nonstandard call; no two-character delivery flag is written
817d9f57
JH
819but sender-host and sender are prefixed and "msg" is inserted in the log line.
820
821Arguments:
822 flags passed to log_write()
823*/
e4bdf652 824void
817d9f57 825delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
e4bdf652
JH
826{
827uschar *log_address;
828int size = 256; /* Used for a temporary, */
829int ptr = 0; /* expanding buffer, for */
830uschar *s; /* building log lines; */
831void *reset_point; /* released afterwards. */
832
e4bdf652
JH
833/* Log the delivery on the main log. We use an extensible string to build up
834the log line, and reset the store afterwards. Remote deliveries should always
835have a pointer to the host item that succeeded; local deliveries can have a
836pointer to a single host item in their host list, for use by the transport. */
837
774ef2d7 838#ifdef EXPERIMENTAL_EVENT
a7538db1 839 /* presume no successful remote delivery */
783b385f 840 lookup_dnssec_authenticated = NULL;
d68218c7
JH
841#endif
842
e4bdf652 843s = reset_point = store_get(size);
e4bdf652 844
6c6d6e48 845log_address = string_log_address(addr, LOGGING(all_parents), TRUE);
817d9f57
JH
846if (msg)
847 s = string_append(s, &size, &ptr, 3, host_and_ident(TRUE), US" ", log_address);
848else
849 {
850 s[ptr++] = logchar;
851 s = string_append(s, &size, &ptr, 2, US"> ", log_address);
852 }
e4bdf652 853
6c6d6e48 854if (LOGGING(sender_on_delivery) || msg)
3c8b3577
JH
855 s = string_append(s, &size, &ptr, 3, US" F=<",
856#ifdef EXPERIMENTAL_INTERNATIONAL
857 testflag(addr, af_utf8_downcvt)
858 ? string_address_utf8_to_alabel(sender_address, NULL)
859 :
860#endif
861 sender_address,
862 US">");
e4bdf652
JH
863
864#ifdef EXPERIMENTAL_SRS
d43cbe25
JH
865if(addr->prop.srs_sender)
866 s = string_append(s, &size, &ptr, 3, US" SRS=<", addr->prop.srs_sender, US">");
e4bdf652
JH
867#endif
868
869/* You might think that the return path must always be set for a successful
870delivery; indeed, I did for some time, until this statement crashed. The case
871when it is not set is for a delivery to /dev/null which is optimised by not
872being run at all. */
873
d6c82d7b 874if (used_return_path && LOGGING(return_path_on_delivery))
e4bdf652
JH
875 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
876
817d9f57
JH
877if (msg)
878 s = string_append(s, &size, &ptr, 2, US" ", msg);
e4bdf652 879
817d9f57 880/* For a delivery from a system filter, there may not be a router */
d6c82d7b 881if (addr->router)
e4bdf652
JH
882 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
883
884s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
885
6c6d6e48 886if (LOGGING(delivery_size))
e4bdf652
JH
887 s = string_append(s, &size, &ptr, 2, US" S=",
888 string_sprintf("%d", transport_count));
889
890/* Local delivery */
891
892if (addr->transport->info->local)
893 {
a7538db1 894 if (addr->host_list)
e4bdf652 895 s = string_append(s, &size, &ptr, 2, US" H=", addr->host_list->name);
ac881e27 896 s = d_log_interface(s, &size, &ptr);
d6c82d7b 897 if (addr->shadow_message)
e4bdf652
JH
898 s = string_cat(s, &size, &ptr, addr->shadow_message,
899 Ustrlen(addr->shadow_message));
900 }
901
902/* Remote delivery */
903
904else
905 {
783b385f 906 if (addr->host_used)
e4bdf652 907 {
67d81c10 908 s = d_hostlog(s, &size, &ptr, addr);
e4bdf652
JH
909 if (continue_sequence > 1)
910 s = string_cat(s, &size, &ptr, US"*", 1);
d68218c7 911
774ef2d7 912#ifdef EXPERIMENTAL_EVENT
a7538db1
JH
913 deliver_host_address = addr->host_used->address;
914 deliver_host_port = addr->host_used->port;
029f4192 915 deliver_host = addr->host_used->name;
783b385f
JH
916
917 /* DNS lookup status */
918 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
919 : addr->host_used->dnssec==DS_NO ? US"no"
920 : NULL;
a7538db1 921#endif
e4bdf652
JH
922 }
923
a7538db1 924#ifdef SUPPORT_TLS
67d81c10 925 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 926#endif
e4bdf652 927
018c60d7 928 if (addr->authenticator)
6f123593 929 {
018c60d7
JH
930 s = string_append(s, &size, &ptr, 2, US" A=", addr->authenticator);
931 if (addr->auth_id)
c8e2fc1e 932 {
018c60d7 933 s = string_append(s, &size, &ptr, 2, US":", addr->auth_id);
6c6d6e48 934 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
018c60d7 935 s = string_append(s, &size, &ptr, 2, US":", addr->auth_sndr);
c8e2fc1e 936 }
6f123593
JH
937 }
938
a7538db1 939#ifndef DISABLE_PRDR
fd98a5c6
JH
940 if (addr->flags & af_prdr_used)
941 s = string_append(s, &size, &ptr, 1, US" PRDR");
a7538db1 942#endif
76f44207
WB
943 }
944
945/* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
fd98a5c6 946
d6c82d7b
JH
947if ( LOGGING(smtp_confirmation)
948 && addr->message
949 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
950 )
76f44207 951 {
b980ed83 952 unsigned i;
1c5b1499 953 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
76f44207
WB
954 uschar *p = big_buffer;
955 uschar *ss = addr->message;
956 *p++ = '\"';
b980ed83 957 for (i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
e4bdf652 958 {
27520a68 959 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
76f44207 960 *p++ = ss[i];
e4bdf652 961 }
76f44207
WB
962 *p++ = '\"';
963 *p = 0;
964 s = string_append(s, &size, &ptr, 2, US" C=", big_buffer);
e4bdf652
JH
965 }
966
967/* Time on queue and actual time taken to deliver */
968
6c6d6e48 969if (LOGGING(queue_time))
e4bdf652 970 s = string_append(s, &size, &ptr, 2, US" QT=",
19050083 971 readconf_printtime( (int) ((long)time(NULL) - (long)received_time)) );
e4bdf652 972
6c6d6e48 973if (LOGGING(deliver_time))
e4bdf652
JH
974 s = string_append(s, &size, &ptr, 2, US" DT=",
975 readconf_printtime(addr->more_errno));
e4bdf652
JH
976
977/* string_cat() always leaves room for the terminator. Release the
978store we used to build the line after writing it. */
979
980s[ptr] = 0;
817d9f57 981log_write(0, flags, "%s", s);
d68218c7 982
774ef2d7
JH
983#ifdef EXPERIMENTAL_EVENT
984if (!msg) msg_event_raise(US"msg:delivery", addr);
d68218c7 985#endif
14a465c3 986
e4bdf652
JH
987store_reset(reset_point);
988return;
989}
990
991
992
059ec3d9
PH
993/*************************************************
994* Actions at the end of handling an address *
995*************************************************/
996
997/* This is a function for processing a single address when all that can be done
998with it has been done.
999
1000Arguments:
1001 addr points to the address block
1002 result the result of the delivery attempt
1003 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1004 driver_type indicates which type of driver (transport, or router) was last
1005 to process the address
1006 logchar '=' or '-' for use when logging deliveries with => or ->
1007
1008Returns: nothing
1009*/
1010
1011static void
1012post_process_one(address_item *addr, int result, int logflags, int driver_type,
1013 int logchar)
1014{
1015uschar *now = tod_stamp(tod_log);
1016uschar *driver_kind = NULL;
1017uschar *driver_name = NULL;
1018uschar *log_address;
1019
1020int size = 256; /* Used for a temporary, */
1021int ptr = 0; /* expanding buffer, for */
1022uschar *s; /* building log lines; */
1023void *reset_point; /* released afterwards. */
1024
059ec3d9
PH
1025DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1026
1027/* Set up driver kind and name for logging. Disable logging if the router or
1028transport has disabled it. */
1029
1030if (driver_type == DTYPE_TRANSPORT)
1031 {
d6c82d7b 1032 if (addr->transport)
059ec3d9
PH
1033 {
1034 driver_name = addr->transport->name;
1035 driver_kind = US" transport";
1036 disable_logging = addr->transport->disable_logging;
1037 }
1038 else driver_kind = US"transporting";
1039 }
1040else if (driver_type == DTYPE_ROUTER)
1041 {
d6c82d7b 1042 if (addr->router)
059ec3d9
PH
1043 {
1044 driver_name = addr->router->name;
1045 driver_kind = US" router";
1046 disable_logging = addr->router->disable_logging;
1047 }
1048 else driver_kind = US"routing";
1049 }
1050
1051/* If there's an error message set, ensure that it contains only printing
1052characters - it should, but occasionally things slip in and this at least
49c2d5ea
PH
1053stops the log format from getting wrecked. We also scan the message for an LDAP
1054expansion item that has a password setting, and flatten the password. This is a
1055fudge, but I don't know a cleaner way of doing this. (If the item is badly
1056malformed, it won't ever have gone near LDAP.) */
059ec3d9 1057
d6c82d7b 1058if (addr->message)
49c2d5ea 1059 {
55414b25
JH
1060 const uschar * s = string_printing(addr->message);
1061 if (s != addr->message)
1062 addr->message = US s;
1063 /* deconst cast ok as string_printing known to have alloc'n'copied */
d6c82d7b
JH
1064 if ( ( Ustrstr(s, "failed to expand") != NULL
1065 || Ustrstr(s, "expansion of ") != NULL
1066 )
1067 && ( Ustrstr(s, "mysql") != NULL
1068 || Ustrstr(s, "pgsql") != NULL
9bdd29ad 1069#ifdef EXPERIMENTAL_REDIS
d6c82d7b 1070 || Ustrstr(s, "redis") != NULL
9bdd29ad 1071#endif
d6c82d7b
JH
1072 || Ustrstr(s, "sqlite") != NULL
1073 || Ustrstr(s, "ldap:") != NULL
1074 || Ustrstr(s, "ldapdn:") != NULL
1075 || Ustrstr(s, "ldapm:") != NULL
1076 ) )
1077 addr->message = string_sprintf("Temporary internal error");
49c2d5ea 1078 }
059ec3d9
PH
1079
1080/* If we used a transport that has one of the "return_output" options set, and
1081if it did in fact generate some output, then for return_output we treat the
1082message as failed if it was not already set that way, so that the output gets
1083returned to the sender, provided there is a sender to send it to. For
1084return_fail_output, do this only if the delivery failed. Otherwise we just
1085unlink the file, and remove the name so that if the delivery failed, we don't
1086try to send back an empty or unwanted file. The log_output options operate only
1087on a non-empty file.
1088
1089In any case, we close the message file, because we cannot afford to leave a
1090file-descriptor for one address while processing (maybe very many) others. */
1091
d6c82d7b 1092if (addr->return_file >= 0 && addr->return_filename)
059ec3d9
PH
1093 {
1094 BOOL return_output = FALSE;
1095 struct stat statbuf;
54fc8428 1096 (void)EXIMfsync(addr->return_file);
059ec3d9
PH
1097
1098 /* If there is no output, do nothing. */
1099
1100 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1101 {
1102 transport_instance *tb = addr->transport;
1103
1104 /* Handle logging options */
1105
d6c82d7b
JH
1106 if ( tb->log_output
1107 || result == FAIL && tb->log_fail_output
1108 || result == DEFER && tb->log_defer_output
1109 )
059ec3d9
PH
1110 {
1111 uschar *s;
1112 FILE *f = Ufopen(addr->return_filename, "rb");
d6c82d7b 1113 if (!f)
059ec3d9
PH
1114 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1115 "from %s transport: %s", addr->return_filename, tb->name,
1116 strerror(errno));
1117 else
d6c82d7b 1118 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
059ec3d9
PH
1119 {
1120 uschar *p = big_buffer + Ustrlen(big_buffer);
55414b25 1121 const uschar * sp;
059ec3d9
PH
1122 while (p > big_buffer && isspace(p[-1])) p--;
1123 *p = 0;
55414b25 1124 sp = string_printing(big_buffer);
059ec3d9 1125 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
55414b25 1126 addr->address, tb->name, sp);
059ec3d9 1127 }
f1e894f3 1128 (void)fclose(f);
059ec3d9
PH
1129 }
1130
1131 /* Handle returning options, but only if there is an address to return
1132 the text to. */
1133
d6c82d7b 1134 if (sender_address[0] != 0 || addr->prop.errors_address)
059ec3d9
PH
1135 if (tb->return_output)
1136 {
1137 addr->transport_return = result = FAIL;
d6c82d7b 1138 if (addr->basic_errno == 0 && !addr->message)
059ec3d9
PH
1139 addr->message = US"return message generated";
1140 return_output = TRUE;
1141 }
1142 else
1143 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
059ec3d9
PH
1144 }
1145
1146 /* Get rid of the file unless it might be returned, but close it in
1147 all cases. */
1148
1149 if (!return_output)
1150 {
1151 Uunlink(addr->return_filename);
1152 addr->return_filename = NULL;
1153 addr->return_file = -1;
1154 }
1155
f1e894f3 1156 (void)close(addr->return_file);
059ec3d9
PH
1157 }
1158
9d1c15ef 1159/* The success case happens only after delivery by a transport. */
059ec3d9
PH
1160
1161if (result == OK)
1162 {
1163 addr->next = addr_succeed;
1164 addr_succeed = addr;
1165
1166 /* Call address_done() to ensure that we don't deliver to this address again,
1167 and write appropriate things to the message log. If it is a child address, we
1168 call child_done() to scan the ancestors and mark them complete if this is the
1169 last child to complete. */
1170
1171 address_done(addr, now);
1172 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1173
d6c82d7b 1174 if (!addr->parent)
059ec3d9
PH
1175 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1176 driver_name, driver_kind);
059ec3d9
PH
1177 else
1178 {
1179 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1180 addr->parent->address, driver_name, driver_kind);
1181 child_done(addr, now);
1182 }
1183
774ef2d7 1184 /* Certificates for logging (via events) */
a7538db1 1185#ifdef SUPPORT_TLS
9d1c15ef
JH
1186 tls_out.ourcert = addr->ourcert;
1187 addr->ourcert = NULL;
1188 tls_out.peercert = addr->peercert;
1189 addr->peercert = NULL;
018058b2 1190
018058b2
JH
1191 tls_out.cipher = addr->cipher;
1192 tls_out.peerdn = addr->peerdn;
1193 tls_out.ocsp = addr->ocsp;
83b27293
JH
1194# ifdef EXPERIMENTAL_DANE
1195 tls_out.dane_verified = testflag(addr, af_dane_verified);
1196# endif
a7538db1 1197#endif
9d1c15ef 1198
817d9f57 1199 delivery_log(LOG_MAIN, addr, logchar, NULL);
9d1c15ef 1200
a7538db1 1201#ifdef SUPPORT_TLS
790fbb71
JH
1202 tls_free_cert(&tls_out.ourcert);
1203 tls_free_cert(&tls_out.peercert);
018058b2
JH
1204 tls_out.cipher = NULL;
1205 tls_out.peerdn = NULL;
1206 tls_out.ocsp = OCSP_NOT_REQ;
83b27293
JH
1207# ifdef EXPERIMENTAL_DANE
1208 tls_out.dane_verified = FALSE;
1209# endif
a7538db1 1210#endif
059ec3d9
PH
1211 }
1212
1213
1214/* Soft failure, or local delivery process failed; freezing may be
1215requested. */
1216
1217else if (result == DEFER || result == PANIC)
1218 {
1219 if (result == PANIC) logflags |= LOG_PANIC;
1220
1221 /* This puts them on the chain in reverse order. Do not change this, because
1222 the code for handling retries assumes that the one with the retry
1223 information is last. */
1224
1225 addr->next = addr_defer;
1226 addr_defer = addr;
1227
1228 /* The only currently implemented special action is to freeze the
1229 message. Logging of this is done later, just before the -H file is
1230 updated. */
1231
1232 if (addr->special_action == SPECIAL_FREEZE)
1233 {
1234 deliver_freeze = TRUE;
1235 deliver_frozen_at = time(NULL);
1236 update_spool = TRUE;
1237 }
1238
1239 /* If doing a 2-stage queue run, we skip writing to either the message
1240 log or the main log for SMTP defers. */
1241
1242 if (!queue_2stage || addr->basic_errno != 0)
1243 {
1244 uschar ss[32];
1245
1246 /* For errors of the type "retry time not reached" (also remotes skipped
1247 on queue run), logging is controlled by L_retry_defer. Note that this kind
1248 of error number is negative, and all the retry ones are less than any
1249 others. */
1250
d6c82d7b
JH
1251 unsigned int use_log_selector = addr->basic_errno <= ERRNO_RETRY_BASE
1252 ? L_retry_defer : 0;
059ec3d9
PH
1253
1254 /* Build up the line that is used for both the message log and the main
1255 log. */
1256
1257 s = reset_point = store_get(size);
e4bdf652
JH
1258
1259 /* Create the address string for logging. Must not do this earlier, because
1260 an OK result may be changed to FAIL when a pipe returns text. */
1261
6c6d6e48 1262 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1263
059ec3d9
PH
1264 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
1265
1266 /* Either driver_name contains something and driver_kind contains
1267 " router" or " transport" (note the leading space), or driver_name is
1268 a null string and driver_kind contains "routing" without the leading
1269 space, if all routing has been deferred. When a domain has been held,
1270 so nothing has been done at all, both variables contain null strings. */
1271
d6c82d7b 1272 if (driver_name)
059ec3d9 1273 {
d6c82d7b 1274 if (driver_kind[1] == 't' && addr->router)
059ec3d9
PH
1275 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
1276 Ustrcpy(ss, " ?=");
1277 ss[1] = toupper(driver_kind[1]);
1278 s = string_append(s, &size, &ptr, 2, ss, driver_name);
1279 }
d6c82d7b
JH
1280 else if (driver_kind)
1281 s = string_append(s, &size, &ptr, 2, US" ", driver_kind);
059ec3d9
PH
1282
1283 sprintf(CS ss, " defer (%d)", addr->basic_errno);
1284 s = string_cat(s, &size, &ptr, ss, Ustrlen(ss));
1285
1286 if (addr->basic_errno > 0)
1287 s = string_append(s, &size, &ptr, 2, US": ",
1288 US strerror(addr->basic_errno));
1289
c562fd30
JH
1290 if (addr->host_used)
1291 s = string_append(s, &size, &ptr, 5,
1292 US" H=", addr->host_used->name,
1293 US" [", addr->host_used->address, US"]");
1294
d6c82d7b 1295 if (addr->message)
059ec3d9
PH
1296 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1297
1298 s[ptr] = 0;
1299
1300 /* Log the deferment in the message log, but don't clutter it
1301 up with retry-time defers after the first delivery attempt. */
1302
1303 if (deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1304 deliver_msglog("%s %s\n", now, s);
1305
1306 /* Write the main log and reset the store */
1307
1308 log_write(use_log_selector, logflags, "== %s", s);
1309 store_reset(reset_point);
1310 }
1311 }
1312
1313
1314/* Hard failure. If there is an address to which an error message can be sent,
1315put this address on the failed list. If not, put it on the deferred list and
1316freeze the mail message for human attention. The latter action can also be
1317explicitly requested by a router or transport. */
1318
1319else
1320 {
1321 /* If this is a delivery error, or a message for which no replies are
1322 wanted, and the message's age is greater than ignore_bounce_errors_after,
1323 force the af_ignore_error flag. This will cause the address to be discarded
1324 later (with a log entry). */
1325
1326 if (sender_address[0] == 0 && message_age >= ignore_bounce_errors_after)
1327 setflag(addr, af_ignore_error);
1328
1329 /* Freeze the message if requested, or if this is a bounce message (or other
652e1b65
PH
1330 message with null sender) and this address does not have its own errors
1331 address. However, don't freeze if errors are being ignored. The actual code
1332 to ignore occurs later, instead of sending a message. Logging of freezing
1333 occurs later, just before writing the -H file. */
059ec3d9 1334
d6c82d7b
JH
1335 if ( !testflag(addr, af_ignore_error)
1336 && ( addr->special_action == SPECIAL_FREEZE
1337 || (sender_address[0] == 0 && !addr->prop.errors_address)
1338 ) )
059ec3d9 1339 {
d6c82d7b
JH
1340 frozen_info = addr->special_action == SPECIAL_FREEZE
1341 ? US""
1342 : sender_local && !local_error_message
1343 ? US" (message created with -f <>)"
1344 : US" (delivery error message)";
059ec3d9
PH
1345 deliver_freeze = TRUE;
1346 deliver_frozen_at = time(NULL);
1347 update_spool = TRUE;
1348
1349 /* The address is put on the defer rather than the failed queue, because
1350 the message is being retained. */
1351
1352 addr->next = addr_defer;
1353 addr_defer = addr;
1354 }
1355
1356 /* Don't put the address on the nonrecipients tree yet; wait until an
1357 error message has been successfully sent. */
1358
1359 else
1360 {
1361 addr->next = addr_failed;
1362 addr_failed = addr;
1363 }
1364
1365 /* Build up the log line for the message and main logs */
1366
1367 s = reset_point = store_get(size);
e4bdf652
JH
1368
1369 /* Create the address string for logging. Must not do this earlier, because
1370 an OK result may be changed to FAIL when a pipe returns text. */
1371
6c6d6e48 1372 log_address = string_log_address(addr, LOGGING(all_parents), result == OK);
e4bdf652 1373
059ec3d9
PH
1374 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
1375
6c6d6e48 1376 if (LOGGING(sender_on_delivery))
059ec3d9
PH
1377 s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
1378
1379 /* Return path may not be set if no delivery actually happened */
1380
d6c82d7b 1381 if (used_return_path && LOGGING(return_path_on_delivery))
059ec3d9 1382 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
059ec3d9 1383
d6c82d7b 1384 if (addr->router)
059ec3d9 1385 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
d6c82d7b 1386 if (addr->transport)
059ec3d9
PH
1387 s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
1388
d6c82d7b 1389 if (addr->host_used)
67d81c10
JH
1390 s = d_hostlog(s, &size, &ptr, addr);
1391
a7538db1 1392#ifdef SUPPORT_TLS
67d81c10 1393 s = d_tlslog(s, &size, &ptr, addr);
a7538db1 1394#endif
059ec3d9
PH
1395
1396 if (addr->basic_errno > 0)
1397 s = string_append(s, &size, &ptr, 2, US": ",
1398 US strerror(addr->basic_errno));
1399
d6c82d7b 1400 if (addr->message)
059ec3d9
PH
1401 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1402
1403 s[ptr] = 0;
1404
1405 /* Do the logging. For the message log, "routing failed" for those cases,
1406 just to make it clearer. */
1407
d6c82d7b 1408 if (driver_name)
059ec3d9 1409 deliver_msglog("%s %s\n", now, s);
d6c82d7b
JH
1410 else
1411 deliver_msglog("%s %s failed for %s\n", now, driver_kind, s);
059ec3d9
PH
1412
1413 log_write(0, LOG_MAIN, "** %s", s);
14a465c3 1414
774ef2d7
JH
1415#ifdef EXPERIMENTAL_EVENT
1416 msg_event_raise(US"msg:fail:delivery", addr);
14a465c3
JH
1417#endif
1418
059ec3d9
PH
1419 store_reset(reset_point);
1420 }
1421
1422/* Ensure logging is turned on again in all cases */
1423
1424disable_logging = FALSE;
1425}
1426
1427
1428
1429
1430/*************************************************
1431* Address-independent error *
1432*************************************************/
1433
1434/* This function is called when there's an error that is not dependent on a
1435particular address, such as an expansion string failure. It puts the error into
1436all the addresses in a batch, logs the incident on the main and panic logs, and
1437clears the expansions. It is mostly called from local_deliver(), but can be
1438called for a remote delivery via findugid().
1439
1440Arguments:
1441 logit TRUE if (MAIN+PANIC) logging required
1442 addr the first of the chain of addresses
1443 code the error code
1444 format format string for error message, or NULL if already set in addr
1445 ... arguments for the format
1446
1447Returns: nothing
1448*/
1449
1450static void
1451common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1452{
1453address_item *addr2;
1454addr->basic_errno = code;
1455
d6c82d7b 1456if (format)
059ec3d9
PH
1457 {
1458 va_list ap;
1459 uschar buffer[512];
1460 va_start(ap, format);
1461 if (!string_vformat(buffer, sizeof(buffer), CS format, ap))
1462 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
81f91683 1463 "common_error expansion was longer than " SIZE_T_FMT, sizeof(buffer));
059ec3d9
PH
1464 va_end(ap);
1465 addr->message = string_copy(buffer);
1466 }
1467
d6c82d7b 1468for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9
PH
1469 {
1470 addr2->basic_errno = code;
1471 addr2->message = addr->message;
1472 }
1473
1474if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1475deliver_set_expansions(NULL);
1476}
1477
1478
1479
1480
1481/*************************************************
1482* Check a "never users" list *
1483*************************************************/
1484
1485/* This function is called to check whether a uid is on one of the two "never
1486users" lists.
1487
1488Arguments:
1489 uid the uid to be checked
1490 nusers the list to be scanned; the first item in the list is the count
1491
1492Returns: TRUE if the uid is on the list
1493*/
1494
1495static BOOL
1496check_never_users(uid_t uid, uid_t *nusers)
1497{
1498int i;
d6c82d7b 1499if (!nusers) return FALSE;
059ec3d9
PH
1500for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1501return FALSE;
1502}
1503
1504
1505
1506/*************************************************
1507* Find uid and gid for a transport *
1508*************************************************/
1509
1510/* This function is called for both local and remote deliveries, to find the
1511uid/gid under which to run the delivery. The values are taken preferentially
1512from the transport (either explicit or deliver_as_creator), then from the
1513address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1514the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1515panic error is logged, and the function fails (which normally leads to delivery
1516deferral).
1517
1518Arguments:
1519 addr the address (possibly a chain)
1520 tp the transport
1521 uidp pointer to uid field
1522 gidp pointer to gid field
1523 igfp pointer to the use_initgroups field
1524
1525Returns: FALSE if failed - error has been set in address(es)
1526*/
1527
1528static BOOL
1529findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1530 BOOL *igfp)
1531{
d6c82d7b 1532uschar *nuname;
059ec3d9
PH
1533BOOL gid_set = FALSE;
1534
1535/* Default initgroups flag comes from the transport */
1536
1537*igfp = tp->initgroups;
1538
1539/* First see if there's a gid on the transport, either fixed or expandable.
1540The expanding function always logs failure itself. */
1541
1542if (tp->gid_set)
1543 {
1544 *gidp = tp->gid;
1545 gid_set = TRUE;
1546 }
d6c82d7b 1547else if (tp->expand_gid)
059ec3d9 1548 {
d6c82d7b
JH
1549 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1550 &(addr->message)))
059ec3d9
PH
1551 {
1552 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1553 return FALSE;
1554 }
d6c82d7b 1555 gid_set = TRUE;
059ec3d9
PH
1556 }
1557
911f6fde
PH
1558/* If the transport did not set a group, see if the router did. */
1559
1560if (!gid_set && testflag(addr, af_gid_set))
1561 {
1562 *gidp = addr->gid;
1563 gid_set = TRUE;
1564 }
1565
059ec3d9
PH
1566/* Pick up a uid from the transport if one is set. */
1567
1568if (tp->uid_set) *uidp = tp->uid;
1569
1570/* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1571it does not provide a passwd value from which a gid can be taken. */
1572
d6c82d7b 1573else if (tp->expand_uid)
059ec3d9
PH
1574 {
1575 struct passwd *pw;
1576 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1577 uidp, &(addr->message)))
1578 {
1579 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1580 return FALSE;
1581 }
d6c82d7b 1582 if (!gid_set && pw)
059ec3d9
PH
1583 {
1584 *gidp = pw->pw_gid;
1585 gid_set = TRUE;
1586 }
1587 }
1588
1589/* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1590
1591else if (tp->deliver_as_creator)
1592 {
1593 *uidp = originator_uid;
1594 if (!gid_set)
1595 {
1596 *gidp = originator_gid;
1597 gid_set = TRUE;
1598 }
1599 }
1600
911f6fde
PH
1601/* Otherwise see if the address specifies the uid and if so, take it and its
1602initgroups flag. */
059ec3d9
PH
1603
1604else if (testflag(addr, af_uid_set))
1605 {
1606 *uidp = addr->uid;
1607 *igfp = testflag(addr, af_initgroups);
059ec3d9
PH
1608 }
1609
1610/* Nothing has specified the uid - default to the Exim user, and group if the
1611gid is not set. */
1612
1613else
1614 {
1615 *uidp = exim_uid;
1616 if (!gid_set)
1617 {
1618 *gidp = exim_gid;
1619 gid_set = TRUE;
1620 }
1621 }
1622
911f6fde
PH
1623/* If no gid is set, it is a disaster. We default to the Exim gid only if
1624defaulting to the Exim uid. In other words, if the configuration has specified
1625a uid, it must also provide a gid. */
059ec3d9
PH
1626
1627if (!gid_set)
1628 {
1629 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1630 "%s transport", tp->name);
1631 return FALSE;
1632 }
1633
1634/* Check that the uid is not on the lists of banned uids that may not be used
1635for delivery processes. */
1636
d6c82d7b
JH
1637nuname = check_never_users(*uidp, never_users)
1638 ? US"never_users"
1639 : check_never_users(*uidp, fixed_never_users)
1640 ? US"fixed_never_users"
1641 : NULL;
1642if (nuname)
059ec3d9
PH
1643 {
1644 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1645 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1646 return FALSE;
1647 }
1648
1649/* All is well */
1650
1651return TRUE;
1652}
1653
1654
1655
1656
1657/*************************************************
1658* Check the size of a message for a transport *
1659*************************************************/
1660
1661/* Checks that the message isn't too big for the selected transport.
1662This is called only when it is known that the limit is set.
1663
1664Arguments:
1665 tp the transport
1666 addr the (first) address being delivered
1667
1668Returns: OK
1669 DEFER expansion failed or did not yield an integer
1670 FAIL message too big
1671*/
1672
1673int
1674check_message_size(transport_instance *tp, address_item *addr)
1675{
1676int rc = OK;
1677int size_limit;
1678
1679deliver_set_expansions(addr);
d45b1de8 1680size_limit = expand_string_integer(tp->message_size_limit, TRUE);
059ec3d9
PH
1681deliver_set_expansions(NULL);
1682
d6c82d7b 1683if (expand_string_message)
059ec3d9
PH
1684 {
1685 rc = DEFER;
d6c82d7b
JH
1686 addr->message = size_limit == -1
1687 ? string_sprintf("failed to expand message_size_limit "
1688 "in %s transport: %s", tp->name, expand_string_message)
1689 : string_sprintf("invalid message_size_limit "
059ec3d9
PH
1690 "in %s transport: %s", tp->name, expand_string_message);
1691 }
1692else if (size_limit > 0 && message_size > size_limit)
1693 {
1694 rc = FAIL;
1695 addr->message =
1696 string_sprintf("message is too big (transport limit = %d)",
1697 size_limit);
1698 }
1699
1700return rc;
1701}
1702
1703
1704
1705/*************************************************
1706* Transport-time check for a previous delivery *
1707*************************************************/
1708
1709/* Check that this base address hasn't previously been delivered to its routed
c2c19e9d
PH
1710transport. If it has been delivered, mark it done. The check is necessary at
1711delivery time in order to handle homonymic addresses correctly in cases where
1712the pattern of redirection changes between delivery attempts (so the unique
1713fields change). Non-homonymic previous delivery is detected earlier, at routing
1714time (which saves unnecessary routing).
1715
1716Arguments:
1717 addr the address item
1718 testing TRUE if testing wanted only, without side effects
059ec3d9 1719
059ec3d9
PH
1720Returns: TRUE if previously delivered by the transport
1721*/
1722
1723static BOOL
c2c19e9d 1724previously_transported(address_item *addr, BOOL testing)
059ec3d9
PH
1725{
1726(void)string_format(big_buffer, big_buffer_size, "%s/%s",
1727 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
1728
1729if (tree_search(tree_nonrecipients, big_buffer) != 0)
1730 {
1731 DEBUG(D_deliver|D_route|D_transport)
1732 debug_printf("%s was previously delivered (%s transport): discarded\n",
1733 addr->address, addr->transport->name);
c2c19e9d 1734 if (!testing) child_done(addr, tod_stamp(tod_log));
059ec3d9
PH
1735 return TRUE;
1736 }
1737
1738return FALSE;
1739}
1740
1741
1742
064a94c9
PH
1743/******************************************************
1744* Check for a given header in a header string *
1745******************************************************/
1746
1747/* This function is used when generating quota warnings. The configuration may
1748specify any header lines it likes in quota_warn_message. If certain of them are
1749missing, defaults are inserted, so we need to be able to test for the presence
1750of a given header.
1751
1752Arguments:
1753 hdr the required header name
1754 hstring the header string
1755
1756Returns: TRUE the header is in the string
1757 FALSE the header is not in the string
1758*/
1759
1760static BOOL
1761contains_header(uschar *hdr, uschar *hstring)
1762{
1763int len = Ustrlen(hdr);
1764uschar *p = hstring;
1765while (*p != 0)
1766 {
1767 if (strncmpic(p, hdr, len) == 0)
1768 {
1769 p += len;
1770 while (*p == ' ' || *p == '\t') p++;
1771 if (*p == ':') return TRUE;
1772 }
1773 while (*p != 0 && *p != '\n') p++;
1774 if (*p == '\n') p++;
1775 }
1776return FALSE;
1777}
1778
1779
1780
059ec3d9
PH
1781
1782/*************************************************
1783* Perform a local delivery *
1784*************************************************/
1785
1786/* Each local delivery is performed in a separate process which sets its
1787uid and gid as specified. This is a safer way than simply changing and
1788restoring using seteuid(); there is a body of opinion that seteuid() cannot be
1789used safely. From release 4, Exim no longer makes any use of it. Besides, not
1790all systems have seteuid().
1791
1792If the uid/gid are specified in the transport_instance, they are used; the
1793transport initialization must ensure that either both or neither are set.
1794Otherwise, the values associated with the address are used. If neither are set,
1795it is a configuration error.
1796
1797The transport or the address may specify a home directory (transport over-
1798rides), and if they do, this is set as $home. If neither have set a working
1799directory, this value is used for that as well. Otherwise $home is left unset
1800and the cwd is set to "/" - a directory that should be accessible to all users.
1801
1802Using a separate process makes it more complicated to get error information
1803back. We use a pipe to pass the return code and also an error code and error
1804text string back to the parent process.
1805
1806Arguments:
1807 addr points to an address block for this delivery; for "normal" local
1808 deliveries this is the only address to be delivered, but for
1809 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
1810 a number of addresses can be handled simultaneously, and in this
1811 case addr will point to a chain of addresses with the same
1812 characteristics.
1813
1814 shadowing TRUE if running a shadow transport; this causes output from pipes
1815 to be ignored.
1816
1817Returns: nothing
1818*/
1819
1820static void
1821deliver_local(address_item *addr, BOOL shadowing)
1822{
1823BOOL use_initgroups;
1824uid_t uid;
1825gid_t gid;
1826int status, len, rc;
1827int pfd[2];
1828pid_t pid;
1829uschar *working_directory;
1830address_item *addr2;
1831transport_instance *tp = addr->transport;
1832
1833/* Set up the return path from the errors or sender address. If the transport
1834has its own return path setting, expand it and replace the existing value. */
1835
d6c82d7b 1836if(addr->prop.errors_address)
d43cbe25 1837 return_path = addr->prop.errors_address;
384152a6 1838#ifdef EXPERIMENTAL_SRS
d6c82d7b 1839else if (addr->prop.srs_sender)
d43cbe25 1840 return_path = addr->prop.srs_sender;
384152a6
TK
1841#endif
1842else
1843 return_path = sender_address;
059ec3d9 1844
d6c82d7b 1845if (tp->return_path)
059ec3d9
PH
1846 {
1847 uschar *new_return_path = expand_string(tp->return_path);
d6c82d7b 1848 if (!new_return_path)
059ec3d9
PH
1849 {
1850 if (!expand_string_forcedfail)
1851 {
1852 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
1853 US"Failed to expand return path \"%s\" in %s transport: %s",
1854 tp->return_path, tp->name, expand_string_message);
1855 return;
1856 }
1857 }
1858 else return_path = new_return_path;
1859 }
1860
1861/* For local deliveries, one at a time, the value used for logging can just be
1862set directly, once and for all. */
1863
1864used_return_path = return_path;
1865
1866/* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
1867gets put into the address(es), and the expansions are unset, so we can just
1868return. */
1869
1870if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
1871
5418e93b
PH
1872/* See if either the transport or the address specifies a home directory. A
1873home directory set in the address may already be expanded; a flag is set to
1874indicate that. In other cases we must expand it. */
059ec3d9 1875
d6c82d7b
JH
1876if ( (deliver_home = tp->home_dir) /* Set in transport, or */
1877 || ( (deliver_home = addr->home_dir) /* Set in address and */
1878 && !testflag(addr, af_home_expanded) /* not expanded */
1879 ) )
059ec3d9
PH
1880 {
1881 uschar *rawhome = deliver_home;
1882 deliver_home = NULL; /* in case it contains $home */
d6c82d7b 1883 if (!(deliver_home = expand_string(rawhome)))
059ec3d9
PH
1884 {
1885 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
1886 "to expand for %s transport: %s", rawhome, tp->name,
1887 expand_string_message);
1888 return;
1889 }
1890 if (*deliver_home != '/')
1891 {
1892 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
1893 "is not absolute for %s transport", deliver_home, tp->name);
1894 return;
1895 }
1896 }
1897
5418e93b
PH
1898/* See if either the transport or the address specifies a current directory,
1899and if so, expand it. If nothing is set, use the home directory, unless it is
1900also unset in which case use "/", which is assumed to be a directory to which
1901all users have access. It is necessary to be in a visible directory for some
1902operating systems when running pipes, as some commands (e.g. "rm" under Solaris
19032.5) require this. */
1904
d6c82d7b
JH
1905working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
1906if (working_directory)
059ec3d9
PH
1907 {
1908 uschar *raw = working_directory;
d6c82d7b 1909 if (!(working_directory = expand_string(raw)))
059ec3d9
PH
1910 {
1911 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
1912 "failed to expand for %s transport: %s", raw, tp->name,
1913 expand_string_message);
1914 return;
1915 }
1916 if (*working_directory != '/')
1917 {
1918 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
1919 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
1920 return;
1921 }
1922 }
d6c82d7b 1923else working_directory = deliver_home ? deliver_home : US"/";
059ec3d9
PH
1924
1925/* If one of the return_output flags is set on the transport, create and open a
1926file in the message log directory for the transport to write its output onto.
1927This is mainly used by pipe transports. The file needs to be unique to the
1928address. This feature is not available for shadow transports. */
1929
d6c82d7b
JH
1930if ( !shadowing
1931 && ( tp->return_output || tp->return_fail_output
1932 || tp->log_output || tp->log_fail_output
1933 ) )
059ec3d9
PH
1934 {
1935 uschar *error;
1936 addr->return_filename =
1937 string_sprintf("%s/msglog/%s/%s-%d-%d", spool_directory, message_subdir,
1938 message_id, getpid(), return_count++);
1939 addr->return_file = open_msglog_file(addr->return_filename, 0400, &error);
1940 if (addr->return_file < 0)
1941 {
1942 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
1943 "to return message: %s", error, tp->name, strerror(errno));
1944 return;
1945 }
1946 }
1947
1948/* Create the pipe for inter-process communication. */
1949
1950if (pipe(pfd) != 0)
1951 {
1952 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
1953 strerror(errno));
1954 return;
1955 }
1956
1957/* Now fork the process to do the real work in the subprocess, but first
1958ensure that all cached resources are freed so that the subprocess starts with
1959a clean slate and doesn't interfere with the parent process. */
1960
1961search_tidyup();
1962
1963if ((pid = fork()) == 0)
1964 {
1965 BOOL replicate = TRUE;
1966
1967 /* Prevent core dumps, as we don't want them in users' home directories.
1968 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
1969 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
1970 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
a29e5231
PP
1971 complain if the error is "not supported".
1972
1973 There are two scenarios where changing the max limit has an effect. In one,
1974 the user is using a .forward and invoking a command of their choice via pipe;
1975 for these, we do need the max limit to be 0 unless the admin chooses to
1976 permit an increased limit. In the other, the command is invoked directly by
1977 the transport and is under administrator control, thus being able to raise
1978 the limit aids in debugging. So there's no general always-right answer.
1979
1980 Thus we inhibit core-dumps completely but let individual transports, while
1981 still root, re-raise the limits back up to aid debugging. We make the
1982 default be no core-dumps -- few enough people can use core dumps in
1983 diagnosis that it's reasonable to make them something that has to be explicitly requested.
1984 */
059ec3d9 1985
a7538db1 1986#ifdef RLIMIT_CORE
059ec3d9
PH
1987 struct rlimit rl;
1988 rl.rlim_cur = 0;
1989 rl.rlim_max = 0;
1990 if (setrlimit(RLIMIT_CORE, &rl) < 0)
1991 {
a7538db1 1992# ifdef SETRLIMIT_NOT_SUPPORTED
059ec3d9 1993 if (errno != ENOSYS && errno != ENOTSUP)
a7538db1 1994# endif
059ec3d9
PH
1995 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
1996 strerror(errno));
1997 }
a7538db1 1998#endif
059ec3d9
PH
1999
2000 /* Reset the random number generator, so different processes don't all
2001 have the same sequence. */
2002
2003 random_seed = 0;
2004
2005 /* If the transport has a setup entry, call this first, while still
2006 privileged. (Appendfile uses this to expand quota, for example, while
2007 able to read private files.) */
2008
d6c82d7b 2009 if (addr->transport->setup)
929ba01c 2010 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
059ec3d9
PH
2011 &(addr->message)))
2012 {
2013 case DEFER:
d6c82d7b
JH
2014 addr->transport_return = DEFER;
2015 goto PASS_BACK;
059ec3d9
PH
2016
2017 case FAIL:
d6c82d7b
JH
2018 addr->transport_return = PANIC;
2019 goto PASS_BACK;
059ec3d9 2020 }
059ec3d9
PH
2021
2022 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2023 when the process becomes unprivileged, it won't be able to write to the
2024 process log. SIGHUP is ignored throughout exim, except when it is being
2025 run as a daemon. */
2026
2027 signal(SIGINT, SIG_IGN);
2028 signal(SIGTERM, SIG_IGN);
2029 signal(SIGUSR1, SIG_IGN);
2030
2031 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2032 half - for transports that exec things (e.g. pipe). Then set the required
2033 gid/uid. */
2034
f1e894f3 2035 (void)close(pfd[pipe_read]);
ff790e47 2036 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
059ec3d9
PH
2037 FD_CLOEXEC);
2038 exim_setugid(uid, gid, use_initgroups,
2039 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2040 addr->address, addr->transport->name));
2041
2042 DEBUG(D_deliver)
2043 {
2044 address_item *batched;
2045 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
d6c82d7b 2046 for (batched = addr->next; batched; batched = batched->next)
059ec3d9
PH
2047 debug_printf("additional batched address: %s\n", batched->address);
2048 }
2049
2050 /* Set an appropriate working directory. */
2051
2052 if (Uchdir(working_directory) < 0)
2053 {
2054 addr->transport_return = DEFER;
2055 addr->basic_errno = errno;
2056 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2057 }
2058
2059 /* If successful, call the transport */
2060
2061 else
2062 {
2063 BOOL ok = TRUE;
2064 set_process_info("delivering %s to %s using %s", message_id,
2065 addr->local_part, addr->transport->name);
2066
2a47f028
JH
2067 /* Setting this global in the subprocess means we need never clear it */
2068 transport_name = addr->transport->name;
2069
059ec3d9
PH
2070 /* If a transport filter has been specified, set up its argument list.
2071 Any errors will get put into the address, and FALSE yielded. */
2072
d6c82d7b 2073 if (addr->transport->filter_command)
059ec3d9
PH
2074 {
2075 ok = transport_set_up_command(&transport_filter_argv,
2076 addr->transport->filter_command,
2077 TRUE, PANIC, addr, US"transport filter", NULL);
2078 transport_filter_timeout = addr->transport->filter_timeout;
2079 }
2080 else transport_filter_argv = NULL;
2081
2082 if (ok)
2083 {
2084 debug_print_string(addr->transport->debug_string);
2085 replicate = !(addr->transport->info->code)(addr->transport, addr);
2086 }
2087 }
2088
2089 /* Pass the results back down the pipe. If necessary, first replicate the
2090 status in the top address to the others in the batch. The label is the
2091 subject of a goto when a call to the transport's setup function fails. We
2092 pass the pointer to the transport back in case it got changed as a result of
2093 file_format in appendfile. */
2094
2095 PASS_BACK:
2096
2097 if (replicate) replicate_status(addr);
d6c82d7b 2098 for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9
PH
2099 {
2100 int i;
2101 int local_part_length = Ustrlen(addr2->local_part);
2102 uschar *s;
1ac6b2e7 2103 int ret;
059ec3d9 2104
d6c82d7b
JH
2105 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2106 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2107 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2108 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2109 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
2110 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2111 || (ret = write(pfd[pipe_write], &addr2->transport,
1ac6b2e7 2112 sizeof(transport_instance *))) != sizeof(transport_instance *)
059ec3d9
PH
2113
2114 /* For a file delivery, pass back the local part, in case the original
2115 was only part of the final delivery path. This gives more complete
2116 logging. */
2117
1ac6b2e7 2118 || (testflag(addr2, af_file)
d6c82d7b 2119 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
1ac6b2e7
JH
2120 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2121 )
2122 )
2123 )
2124 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2125 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2126
2127 /* Now any messages */
2128
2129 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2130 {
d6c82d7b
JH
2131 int message_length = s ? Ustrlen(s) + 1 : 0;
2132 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2133 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
1ac6b2e7
JH
2134 )
2135 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s\n",
2136 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2137 }
2138 }
2139
2140 /* OK, this process is now done. Free any cached resources that it opened,
2141 and close the pipe we were writing down before exiting. */
2142
f1e894f3 2143 (void)close(pfd[pipe_write]);
059ec3d9
PH
2144 search_tidyup();
2145 exit(EXIT_SUCCESS);
2146 }
2147
2148/* Back in the main process: panic if the fork did not succeed. This seems
2149better than returning an error - if forking is failing it is probably best
2150not to try other deliveries for this message. */
2151
2152if (pid < 0)
2153 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2154 addr->address);
2155
2156/* Read the pipe to get the delivery status codes and error messages. Our copy
2157of the writing end must be closed first, as otherwise read() won't return zero
2158on an empty pipe. We check that a status exists for each address before
2159overwriting the address structure. If data is missing, the default DEFER status
2160will remain. Afterwards, close the reading end. */
2161
f1e894f3 2162(void)close(pfd[pipe_write]);
059ec3d9 2163
d6c82d7b 2164for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9 2165 {
d6c82d7b 2166 len = read(pfd[pipe_read], &status, sizeof(int));
059ec3d9
PH
2167 if (len > 0)
2168 {
2169 int i;
2170 uschar **sptr;
2171
2172 addr2->transport_return = status;
d6c82d7b 2173 len = read(pfd[pipe_read], &transport_count,
059ec3d9 2174 sizeof(transport_count));
d6c82d7b
JH
2175 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2176 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2177 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
2178 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2179 len = read(pfd[pipe_read], &addr2->transport,
059ec3d9
PH
2180 sizeof(transport_instance *));
2181
2182 if (testflag(addr2, af_file))
2183 {
2184 int local_part_length;
d6c82d7b
JH
2185 len = read(pfd[pipe_read], &local_part_length, sizeof(int));
2186 len = read(pfd[pipe_read], big_buffer, local_part_length);
059ec3d9
PH
2187 big_buffer[local_part_length] = 0;
2188 addr2->local_part = string_copy(big_buffer);
2189 }
2190
d6c82d7b 2191 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
059ec3d9
PH
2192 {
2193 int message_length;
d6c82d7b 2194 len = read(pfd[pipe_read], &message_length, sizeof(int));
059ec3d9
PH
2195 if (message_length > 0)
2196 {
d6c82d7b 2197 len = read(pfd[pipe_read], big_buffer, message_length);
059ec3d9
PH
2198 if (len > 0) *sptr = string_copy(big_buffer);
2199 }
2200 }
2201 }
2202
2203 else
2204 {
2205 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2206 "from delivery subprocess", addr2->unique);
2207 break;
2208 }
2209 }
2210
f1e894f3 2211(void)close(pfd[pipe_read]);
059ec3d9
PH
2212
2213/* Unless shadowing, write all successful addresses immediately to the journal
2214file, to ensure they are recorded asap. For homonymic addresses, use the base
2215address plus the transport name. Failure to write the journal is panic-worthy,
2216but don't stop, as it may prove possible subsequently to update the spool file
2217in order to record the delivery. */
2218
2219if (!shadowing)
2220 {
d6c82d7b
JH
2221 for (addr2 = addr; addr2; addr2 = addr2->next)
2222 if (addr2->transport_return == OK)
2223 {
2224 if (testflag(addr2, af_homonym))
2225 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2226 else
2227 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
059ec3d9 2228
d6c82d7b
JH
2229 /* In the test harness, wait just a bit to let the subprocess finish off
2230 any debug output etc first. */
059ec3d9 2231
d6c82d7b 2232 if (running_in_test_harness) millisleep(300);
059ec3d9 2233
d6c82d7b
JH
2234 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2235 len = Ustrlen(big_buffer);
2236 if (write(journal_fd, big_buffer, len) != len)
2237 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2238 big_buffer, strerror(errno));
2239 }
059ec3d9
PH
2240
2241 /* Ensure the journal file is pushed out to disk. */
2242
54fc8428 2243 if (EXIMfsync(journal_fd) < 0)
059ec3d9
PH
2244 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2245 strerror(errno));
2246 }
2247
2248/* Wait for the process to finish. If it terminates with a non-zero code,
2249freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2250status values of all the addresses as they are. Take care to handle the case
2251when the subprocess doesn't seem to exist. This has been seen on one system
2252when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2253happens, wait() doesn't recognize the termination of child processes. Exim now
2254resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2255
2256while ((rc = wait(&status)) != pid)
059ec3d9
PH
2257 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2258 {
2259 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2260 addr->transport->driver_name);
2261 status = 0;
2262 break;
2263 }
059ec3d9
PH
2264
2265if ((status & 0xffff) != 0)
2266 {
2267 int msb = (status >> 8) & 255;
2268 int lsb = status & 255;
2269 int code = (msb == 0)? (lsb & 0x7f) : msb;
2270 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2271 addr->special_action = SPECIAL_FREEZE;
2272 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2273 "status 0x%04x: %s %d",
2274 addr->transport->driver_name,
2275 status,
d6c82d7b 2276 msb == 0 ? "terminated by signal" : "exit code",
059ec3d9
PH
2277 code);
2278 }
2279
2280/* If SPECIAL_WARN is set in the top address, send a warning message. */
2281
d6c82d7b 2282if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
059ec3d9
PH
2283 {
2284 int fd;
2285 uschar *warn_message;
d6c82d7b 2286 pid_t pid;
059ec3d9
PH
2287
2288 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2289
d6c82d7b 2290 if (!(warn_message = expand_string(addr->transport->warn_message)))
059ec3d9
PH
2291 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2292 "message for %s transport): %s", addr->transport->warn_message,
2293 addr->transport->name, expand_string_message);
d6c82d7b
JH
2294
2295 else if ((pid = child_open_exim(&fd)) > 0)
059ec3d9 2296 {
d6c82d7b
JH
2297 FILE *f = fdopen(fd, "wb");
2298 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2299 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2300 fprintf(f, "Auto-Submitted: auto-replied\n");
2301 if (!contains_header(US"From", warn_message))
2302 moan_write_from(f);
2303 fprintf(f, "%s", CS warn_message);
059ec3d9 2304
d6c82d7b 2305 /* Close and wait for child process to complete, without a timeout. */
059ec3d9 2306
d6c82d7b
JH
2307 (void)fclose(f);
2308 (void)child_close(pid, 0);
059ec3d9
PH
2309 }
2310
2311 addr->special_action = SPECIAL_NONE;
2312 }
2313}
2314
2315
2316
fa41615d
JH
2317
2318/* Put the chain of addrs on the defer list. Retry will happen
2319on the next queue run, earlier if triggered by a new message.
2320Loop for the next set of addresses. */
2321
2322static void
2323deferlist_chain(address_item * addr)
2324{
2325address_item * next;
2326for (next = addr; next->next; next = next->next) ;
2327next->next = addr_defer;
2328addr_defer = addr;
2329}
2330
2331
2332
059ec3d9
PH
2333/*************************************************
2334* Do local deliveries *
2335*************************************************/
2336
2337/* This function processes the list of addresses in addr_local. True local
2338deliveries are always done one address at a time. However, local deliveries can
2339be batched up in some cases. Typically this is when writing batched SMTP output
2340files for use by some external transport mechanism, or when running local
2341deliveries over LMTP.
2342
2343Arguments: None
2344Returns: Nothing
2345*/
2346
2347static void
2348do_local_deliveries(void)
2349{
2350open_db dbblock;
2351open_db *dbm_file = NULL;
2352time_t now = time(NULL);
2353
2354/* Loop until we have exhausted the supply of local deliveries */
2355
d6c82d7b 2356while (addr_local)
059ec3d9
PH
2357 {
2358 time_t delivery_start;
2359 int deliver_time;
2360 address_item *addr2, *addr3, *nextaddr;
2361 int logflags = LOG_MAIN;
2362 int logchar = dont_deliver? '*' : '=';
2363 transport_instance *tp;
fa41615d 2364 uschar * serialize_key = NULL;
059ec3d9
PH
2365
2366 /* Pick the first undelivered address off the chain */
2367
2368 address_item *addr = addr_local;
2369 addr_local = addr->next;
2370 addr->next = NULL;
2371
2372 DEBUG(D_deliver|D_transport)
2373 debug_printf("--------> %s <--------\n", addr->address);
2374
2375 /* An internal disaster if there is no transport. Should not occur! */
2376
d6c82d7b 2377 if (!(tp = addr->transport))
059ec3d9
PH
2378 {
2379 logflags |= LOG_PANIC;
2380 disable_logging = FALSE; /* Jic */
d6c82d7b
JH
2381 addr->message = addr->router
2382 ? string_sprintf("No transport set by %s router", addr->router->name)
2383 : string_sprintf("No transport set by system filter");
059ec3d9
PH
2384 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
2385 continue;
2386 }
2387
2388 /* Check that this base address hasn't previously been delivered to this
2389 transport. The check is necessary at this point to handle homonymic addresses
2390 correctly in cases where the pattern of redirection changes between delivery
2391 attempts. Non-homonymic previous delivery is detected earlier, at routing
2392 time. */
2393
c2c19e9d 2394 if (previously_transported(addr, FALSE)) continue;
059ec3d9
PH
2395
2396 /* There are weird cases where logging is disabled */
2397
2398 disable_logging = tp->disable_logging;
2399
f7fd3850
PH
2400 /* Check for batched addresses and possible amalgamation. Skip all the work
2401 if either batch_max <= 1 or there aren't any other addresses for local
2402 delivery. */
059ec3d9 2403
d6c82d7b 2404 if (tp->batch_max > 1 && addr_local)
059ec3d9
PH
2405 {
2406 int batch_count = 1;
2407 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
d6c82d7b
JH
2408 BOOL uses_lp = ( testflag(addr, af_pfr)
2409 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2410 )
2411 || readconf_depends((driver_instance *)tp, US"local_part");
059ec3d9
PH
2412 uschar *batch_id = NULL;
2413 address_item **anchor = &addr_local;
2414 address_item *last = addr;
2415 address_item *next;
2416
2417 /* Expand the batch_id string for comparison with other addresses.
2418 Expansion failure suppresses batching. */
2419
d6c82d7b 2420 if (tp->batch_id)
059ec3d9
PH
2421 {
2422 deliver_set_expansions(addr);
2423 batch_id = expand_string(tp->batch_id);
2424 deliver_set_expansions(NULL);
d6c82d7b 2425 if (!batch_id)
059ec3d9
PH
2426 {
2427 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2428 "in %s transport (%s): %s", tp->name, addr->address,
2429 expand_string_message);
2430 batch_count = tp->batch_max;
2431 }
2432 }
2433
2434 /* Until we reach the batch_max limit, pick off addresses which have the
2435 same characteristics. These are:
2436
2437 same transport
7816e254 2438 not previously delivered (see comment about 50 lines above)
059ec3d9 2439 same local part if the transport's configuration contains $local_part
f7fd3850 2440 or if this is a file or pipe delivery from a redirection
059ec3d9
PH
2441 same domain if the transport's configuration contains $domain
2442 same errors address
2443 same additional headers
2444 same headers to be removed
2445 same uid/gid for running the transport
2446 same first host if a host list is set
2447 */
2448
d6c82d7b 2449 while ((next = *anchor) && batch_count < tp->batch_max)
059ec3d9
PH
2450 {
2451 BOOL ok =
d6c82d7b
JH
2452 tp == next->transport
2453 && !previously_transported(next, TRUE)
2454 && (addr->flags & (af_pfr|af_file)) == (next->flags & (af_pfr|af_file))
2455 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2456 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2457 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2458 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2459 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2460 && same_ugid(tp, addr, next)
2461 && ( !addr->host_list && !next->host_list
2462 || addr->host_list
2463 && next->host_list
2464 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2465 );
059ec3d9
PH
2466
2467 /* If the transport has a batch_id setting, batch_id will be non-NULL
2468 from the expansion outside the loop. Expand for this address and compare.
2469 Expansion failure makes this address ineligible for batching. */
2470
d6c82d7b 2471 if (ok && batch_id)
059ec3d9
PH
2472 {
2473 uschar *bid;
2474 address_item *save_nextnext = next->next;
2475 next->next = NULL; /* Expansion for a single address */
2476 deliver_set_expansions(next);
2477 next->next = save_nextnext;
2478 bid = expand_string(tp->batch_id);
2479 deliver_set_expansions(NULL);
d6c82d7b 2480 if (!bid)
059ec3d9
PH
2481 {
2482 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2483 "in %s transport (%s): %s", tp->name, next->address,
2484 expand_string_message);
2485 ok = FALSE;
2486 }
2487 else ok = (Ustrcmp(batch_id, bid) == 0);
2488 }
2489
2490 /* Take address into batch if OK. */
2491
2492 if (ok)
2493 {
2494 *anchor = next->next; /* Include the address */
2495 next->next = NULL;
2496 last->next = next;
2497 last = next;
2498 batch_count++;
2499 }
fa41615d 2500 else anchor = &next->next; /* Skip the address */
059ec3d9
PH
2501 }
2502 }
2503
2504 /* We now have one or more addresses that can be delivered in a batch. Check
2505 whether the transport is prepared to accept a message of this size. If not,
2506 fail them all forthwith. If the expansion fails, or does not yield an
2507 integer, defer delivery. */
2508
d6c82d7b 2509 if (tp->message_size_limit)
059ec3d9
PH
2510 {
2511 int rc = check_message_size(tp, addr);
2512 if (rc != OK)
2513 {
2514 replicate_status(addr);
d6c82d7b 2515 while (addr)
059ec3d9
PH
2516 {
2517 addr2 = addr->next;
2518 post_process_one(addr, rc, logflags, DTYPE_TRANSPORT, 0);
2519 addr = addr2;
2520 }
2521 continue; /* With next batch of addresses */
2522 }
2523 }
2524
2525 /* If we are not running the queue, or if forcing, all deliveries will be
2526 attempted. Otherwise, we must respect the retry times for each address. Even
2527 when not doing this, we need to set up the retry key string, and determine
2528 whether a retry record exists, because after a successful delivery, a delete
2529 retry item must be set up. Keep the retry database open only for the duration
2530 of these checks, rather than for all local deliveries, because some local
2531 deliveries (e.g. to pipes) can take a substantial time. */
2532
d6c82d7b 2533 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
059ec3d9
PH
2534 {
2535 DEBUG(D_deliver|D_retry|D_hints_lookup)
2536 debug_printf("no retry data available\n");
2537 }
2538
2539 addr2 = addr;
2540 addr3 = NULL;
d6c82d7b 2541 while (addr2)
059ec3d9
PH
2542 {
2543 BOOL ok = TRUE; /* to deliver this address */
2544 uschar *retry_key;
2545
2546 /* Set up the retry key to include the domain or not, and change its
2547 leading character from "R" to "T". Must make a copy before doing this,
2548 because the old key may be pointed to from a "delete" retry item after
2549 a routing delay. */
2550
2551 retry_key = string_copy(
d6c82d7b 2552 tp->retry_use_local_part ? addr2->address_retry_key :
059ec3d9
PH
2553 addr2->domain_retry_key);
2554 *retry_key = 'T';
2555
2556 /* Inspect the retry data. If there is no hints file, delivery happens. */
2557
d6c82d7b 2558 if (dbm_file)
059ec3d9
PH
2559 {
2560 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2561
2562 /* If there is no retry record, delivery happens. If there is,
2563 remember it exists so it can be deleted after a successful delivery. */
2564
d6c82d7b 2565 if (retry_record)
059ec3d9
PH
2566 {
2567 setflag(addr2, af_lt_retry_exists);
2568
2569 /* A retry record exists for this address. If queue running and not
2570 forcing, inspect its contents. If the record is too old, or if its
2571 retry time has come, or if it has passed its cutoff time, delivery
2572 will go ahead. */
2573
2574 DEBUG(D_retry)
2575 {
ea49d0e1
PH
2576 debug_printf("retry record exists: age=%s ",
2577 readconf_printtime(now - retry_record->time_stamp));
2578 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2579 debug_printf(" time to retry = %s expired = %d\n",
2580 readconf_printtime(retry_record->next_try - now),
2581 retry_record->expired);
059ec3d9
PH
2582 }
2583
2584 if (queue_running && !deliver_force)
2585 {
d6c82d7b
JH
2586 ok = (now - retry_record->time_stamp > retry_data_expire)
2587 || (now >= retry_record->next_try)
2588 || retry_record->expired;
059ec3d9
PH
2589
2590 /* If we haven't reached the retry time, there is one more check
1ddeb334 2591 to do, which is for the ultimate address timeout. */
059ec3d9
PH
2592
2593 if (!ok)
ba9af0af
TF
2594 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2595 retry_record, now);
059ec3d9
PH
2596 }
2597 }
2598 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2599 }
2600
2601 /* This address is to be delivered. Leave it on the chain. */
2602
2603 if (ok)
2604 {
2605 addr3 = addr2;
2606 addr2 = addr2->next;
2607 }
2608
2609 /* This address is to be deferred. Take it out of the chain, and
2610 post-process it as complete. Must take it out of the chain first,
2611 because post processing puts it on another chain. */
2612
2613 else
2614 {
2615 address_item *this = addr2;
2616 this->message = US"Retry time not yet reached";
2617 this->basic_errno = ERRNO_LRETRY;
d6c82d7b
JH
2618 addr2 = addr3 ? (addr3->next = addr2->next)
2619 : (addr = addr2->next);
059ec3d9
PH
2620 post_process_one(this, DEFER, logflags, DTYPE_TRANSPORT, 0);
2621 }
2622 }
2623
d6c82d7b 2624 if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
2625
2626 /* If there are no addresses left on the chain, they all deferred. Loop
2627 for the next set of addresses. */
2628
d6c82d7b 2629 if (!addr) continue;
059ec3d9 2630
fa41615d
JH
2631 /* If the transport is limited for parallellism, enforce that here.
2632 We use a hints DB entry, incremented here and decremented after
2633 the transport (and any shadow transport) completes. */
2634
2635 if (tp->max_parallel)
2636 {
2637 int_eximarith_t max_parallel =
2638 expand_string_integer(tp->max_parallel, TRUE);
2639 if (expand_string_message)
2640 {
2641 logflags |= LOG_PANIC;
2642 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2643 "in %s transport (%s): %s", tp->name, addr->address,
2644 expand_string_message);
2645 for (addr2 = addr->next; addr; addr = addr2, addr2 = addr2->next)
2646 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
2647 continue;
2648 }
2649 if ( max_parallel > 0
2650 && !enq_start(
2651 serialize_key = string_sprintf("tpt-serialize-%s", tp->name),
2652 (unsigned) max_parallel)
2653 )
2654 {
2655 DEBUG(D_transport)
2656 debug_printf("skipping tpt %s because parallelism limit %u reached\n",
2657 tp->name, (unsigned) max_parallel);
2658
2659 deferlist_chain(addr);
2660 continue;
2661 }
2662 }
2663
2664
059ec3d9
PH
2665 /* So, finally, we do have some addresses that can be passed to the
2666 transport. Before doing so, set up variables that are relevant to a
2667 single delivery. */
2668
2669 deliver_set_expansions(addr);
2670 delivery_start = time(NULL);
2671 deliver_local(addr, FALSE);
2672 deliver_time = (int)(time(NULL) - delivery_start);
2673
2674 /* If a shadow transport (which must perforce be another local transport), is
2675 defined, and its condition is met, we must pass the message to the shadow
2676 too, but only those addresses that succeeded. We do this by making a new
2677 chain of addresses - also to keep the original chain uncontaminated. We must
2678 use a chain rather than doing it one by one, because the shadow transport may
2679 batch.
2680
2681 NOTE: if the condition fails because of a lookup defer, there is nothing we
2682 can do! */
2683
d6c82d7b
JH
2684 if ( tp->shadow
2685 && ( !tp->shadow_condition
2686 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
2687 ) )
059ec3d9
PH
2688 {
2689 transport_instance *stp;
2690 address_item *shadow_addr = NULL;
2691 address_item **last = &shadow_addr;
2692
d6c82d7b 2693 for (stp = transports; stp; stp = stp->next)
059ec3d9
PH
2694 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
2695
d6c82d7b 2696 if (!stp)
059ec3d9
PH
2697 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
2698 tp->shadow);
2699
2700 /* Pick off the addresses that have succeeded, and make clones. Put into
2701 the shadow_message field a pointer to the shadow_message field of the real
2702 address. */
2703
d6c82d7b
JH
2704 else for (addr2 = addr; addr2; addr2 = addr2->next)
2705 if (addr2->transport_return == OK)
2706 {
2707 addr3 = store_get(sizeof(address_item));
2708 *addr3 = *addr2;
2709 addr3->next = NULL;
2710 addr3->shadow_message = (uschar *) &(addr2->shadow_message);
2711 addr3->transport = stp;
2712 addr3->transport_return = DEFER;
2713 addr3->return_filename = NULL;
2714 addr3->return_file = -1;
2715 *last = addr3;
2716 last = &(addr3->next);
2717 }
059ec3d9
PH
2718
2719 /* If we found any addresses to shadow, run the delivery, and stick any
2720 message back into the shadow_message field in the original. */
2721
d6c82d7b 2722 if (shadow_addr)
059ec3d9
PH
2723 {
2724 int save_count = transport_count;
2725
2726 DEBUG(D_deliver|D_transport)
2727 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
2728 deliver_local(shadow_addr, TRUE);
2729
d6c82d7b 2730 for(; shadow_addr; shadow_addr = shadow_addr->next)
059ec3d9
PH
2731 {
2732 int sresult = shadow_addr->transport_return;
d6c82d7b
JH
2733 *(uschar **)shadow_addr->shadow_message =
2734 sresult == OK
2735 ? string_sprintf(" ST=%s", stp->name)
2736 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
2737 shadow_addr->basic_errno <= 0
2738 ? US""
2739 : US strerror(shadow_addr->basic_errno),
2740 shadow_addr->basic_errno <= 0 || !shadow_addr->message
2741 ? US""
2742 : US": ",
2743 shadow_addr->message
2744 ? shadow_addr->message
2745 : shadow_addr->basic_errno <= 0
2746 ? US"unknown error"
2747 : US"");
059ec3d9
PH
2748
2749 DEBUG(D_deliver|D_transport)
2750 debug_printf("%s shadow transport returned %s for %s\n",
2751 stp->name,
d6c82d7b
JH
2752 sresult == OK ? "OK" :
2753 sresult == DEFER ? "DEFER" :
2754 sresult == FAIL ? "FAIL" :
2755 sresult == PANIC ? "PANIC" : "?",
059ec3d9
PH
2756 shadow_addr->address);
2757 }
2758
2759 DEBUG(D_deliver|D_transport)
2760 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
2761
2762 transport_count = save_count; /* Restore original transport count */
2763 }
2764 }
2765
2766 /* Cancel the expansions that were set up for the delivery. */
2767
2768 deliver_set_expansions(NULL);
2769
fa41615d
JH
2770 /* If the transport was parallelism-limited, decrement the hints DB record. */
2771
2772 if (serialize_key) enq_end(serialize_key);
2773
059ec3d9
PH
2774 /* Now we can process the results of the real transport. We must take each
2775 address off the chain first, because post_process_one() puts it on another
2776 chain. */
2777
d6c82d7b 2778 for (addr2 = addr; addr2; addr2 = nextaddr)
059ec3d9
PH
2779 {
2780 int result = addr2->transport_return;
2781 nextaddr = addr2->next;
2782
2783 DEBUG(D_deliver|D_transport)
2784 debug_printf("%s transport returned %s for %s\n",
2785 tp->name,
d6c82d7b
JH
2786 result == OK ? "OK" :
2787 result == DEFER ? "DEFER" :
2788 result == FAIL ? "FAIL" :
2789 result == PANIC ? "PANIC" : "?",
059ec3d9
PH
2790 addr2->address);
2791
2792 /* If there is a retry_record, or if delivery is deferred, build a retry
2793 item for setting a new retry time or deleting the old retry record from
2794 the database. These items are handled all together after all addresses
2795 have been handled (so the database is open just for a short time for
2796 updating). */
2797
2798 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
2799 {
d6c82d7b
JH
2800 int flags = result == DEFER ? 0 : rf_delete;
2801 uschar *retry_key = string_copy(tp->retry_use_local_part
2802 ? addr2->address_retry_key : addr2->domain_retry_key);
059ec3d9
PH
2803 *retry_key = 'T';
2804 retry_add_item(addr2, retry_key, flags);
2805 }
2806
2807 /* Done with this address */
2808
2809 if (result == OK) addr2->more_errno = deliver_time;
2810 post_process_one(addr2, result, logflags, DTYPE_TRANSPORT, logchar);
2811
2812 /* If a pipe delivery generated text to be sent back, the result may be
2813 changed to FAIL, and we must copy this for subsequent addresses in the
2814 batch. */
2815
2816 if (addr2->transport_return != result)
2817 {
d6c82d7b 2818 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
059ec3d9
PH
2819 {
2820 addr3->transport_return = addr2->transport_return;
2821 addr3->basic_errno = addr2->basic_errno;
2822 addr3->message = addr2->message;
2823 }
2824 result = addr2->transport_return;
2825 }
2826
2827 /* Whether or not the result was changed to FAIL, we need to copy the
2828 return_file value from the first address into all the addresses of the
2829 batch, so they are all listed in the error message. */
2830
2831 addr2->return_file = addr->return_file;
2832
2833 /* Change log character for recording successful deliveries. */
2834
2835 if (result == OK) logchar = '-';
2836 }
2837 } /* Loop back for next batch of addresses */
2838}
2839
2840
2841
2842
2843/*************************************************
2844* Sort remote deliveries *
2845*************************************************/
2846
2847/* This function is called if remote_sort_domains is set. It arranges that the
2848chain of addresses for remote deliveries is ordered according to the strings
2849specified. Try to make this shuffling reasonably efficient by handling
2850sequences of addresses rather than just single ones.
2851
2852Arguments: None
2853Returns: Nothing
2854*/
2855
2856static void
2857sort_remote_deliveries(void)
2858{
2859int sep = 0;
2860address_item **aptr = &addr_remote;
55414b25 2861const uschar *listptr = remote_sort_domains;
059ec3d9
PH
2862uschar *pattern;
2863uschar patbuf[256];
2864
d6c82d7b
JH
2865while ( *aptr
2866 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
2867 )
059ec3d9
PH
2868 {
2869 address_item *moved = NULL;
2870 address_item **bptr = &moved;
2871
d6c82d7b 2872 while (*aptr)
059ec3d9
PH
2873 {
2874 address_item **next;
2875 deliver_domain = (*aptr)->domain; /* set $domain */
55414b25 2876 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
059ec3d9
PH
2877 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
2878 {
d6c82d7b 2879 aptr = &(*aptr)->next;
059ec3d9
PH
2880 continue;
2881 }
2882
d6c82d7b
JH
2883 next = &(*aptr)->next;
2884 while ( *next
2885 && (deliver_domain = (*next)->domain, /* Set $domain */
55414b25 2886 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
d6c82d7b
JH
2887 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
2888 )
2889 next = &(*next)->next;
059ec3d9
PH
2890
2891 /* If the batch of non-matchers is at the end, add on any that were
2892 extracted further up the chain, and end this iteration. Otherwise,
2893 extract them from the chain and hang on the moved chain. */
2894
d6c82d7b 2895 if (!*next)
059ec3d9
PH
2896 {
2897 *next = moved;
2898 break;
2899 }
2900
2901 *bptr = *aptr;
2902 *aptr = *next;
2903 *next = NULL;
2904 bptr = next;
d6c82d7b 2905 aptr = &(*aptr)->next;
059ec3d9
PH
2906 }
2907
2908 /* If the loop ended because the final address matched, *aptr will
2909 be NULL. Add on to the end any extracted non-matching addresses. If
2910 *aptr is not NULL, the loop ended via "break" when *next is null, that
2911 is, there was a string of non-matching addresses at the end. In this
2912 case the extracted addresses have already been added on the end. */
2913
d6c82d7b 2914 if (!*aptr) *aptr = moved;
059ec3d9
PH
2915 }
2916
2917DEBUG(D_deliver)
2918 {
2919 address_item *addr;
2920 debug_printf("remote addresses after sorting:\n");
d6c82d7b 2921 for (addr = addr_remote; addr; addr = addr->next)
059ec3d9
PH
2922 debug_printf(" %s\n", addr->address);
2923 }
2924}
2925
2926
2927
2928/*************************************************
2929* Read from pipe for remote delivery subprocess *
2930*************************************************/
2931
2932/* This function is called when the subprocess is complete, but can also be
2933called before it is complete, in order to empty a pipe that is full (to prevent
2934deadlock). It must therefore keep track of its progress in the parlist data
2935block.
2936
2937We read the pipe to get the delivery status codes and a possible error message
2938for each address, optionally preceded by unusability data for the hosts and
2939also by optional retry data.
2940
2941Read in large chunks into the big buffer and then scan through, interpreting
2942the data therein. In most cases, only a single read will be necessary. No
1c5466b9
PH
2943individual item will ever be anywhere near 2500 bytes in length, so by ensuring
2944that we read the next chunk when there is less than 2500 bytes left in the
2945non-final chunk, we can assume each item is complete in the buffer before
2946handling it. Each item is written using a single write(), which is atomic for
2947small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
2948often bigger) so even if we are reading while the subprocess is still going, we
2949should never have only a partial item in the buffer.
059ec3d9
PH
2950
2951Argument:
2952 poffset the offset of the parlist item
2953 eop TRUE if the process has completed
2954
2955Returns: TRUE if the terminating 'Z' item has been read,
2956 or there has been a disaster (i.e. no more data needed);
2957 FALSE otherwise
2958*/
2959
2960static BOOL
2961par_read_pipe(int poffset, BOOL eop)
2962{
2963host_item *h;
2964pardata *p = parlist + poffset;
2965address_item *addrlist = p->addrlist;
2966address_item *addr = p->addr;
2967pid_t pid = p->pid;
2968int fd = p->fd;
2969uschar *endptr = big_buffer;
2970uschar *ptr = endptr;
2971uschar *msg = p->msg;
2972BOOL done = p->done;
2973BOOL unfinished = TRUE;
bd21a787
WB
2974/* minimum size to read is header size including id, subid and length */
2975int required = PIPE_HEADER_SIZE;
059ec3d9
PH
2976
2977/* Loop through all items, reading from the pipe when necessary. The pipe
2978is set up to be non-blocking, but there are two different Unix mechanisms in
2979use. Exim uses O_NONBLOCK if it is defined. This returns 0 for end of file,
2980and EAGAIN for no more data. If O_NONBLOCK is not defined, Exim uses O_NDELAY,
2981which returns 0 for both end of file and no more data. We distinguish the
2982two cases by taking 0 as end of file only when we know the process has
2983completed.
2984
2985Each separate item is written to the pipe in a single write(), and as they are
2986all short items, the writes will all be atomic and we should never find
8e669ac1
PH
2987ourselves in the position of having read an incomplete item. "Short" in this
2988case can mean up to about 1K in the case when there is a long error message
1c5466b9 2989associated with an address. */
059ec3d9
PH
2990
2991DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
2992 (int)p->pid, eop? "ended" : "not ended");
2993
2994while (!done)
2995 {
2996 retry_item *r, **rp;
2997 int remaining = endptr - ptr;
bd21a787
WB
2998 uschar header[PIPE_HEADER_SIZE + 1];
2999 uschar id, subid;
3000 uschar *endc;
059ec3d9
PH
3001
3002 /* Read (first time) or top up the chars in the buffer if necessary.
3003 There will be only one read if we get all the available data (i.e. don't
3004 fill the buffer completely). */
3005
bd21a787 3006 if (remaining < required && unfinished)
059ec3d9
PH
3007 {
3008 int len;
3009 int available = big_buffer_size - remaining;
3010
3011 if (remaining > 0) memmove(big_buffer, ptr, remaining);
3012
3013 ptr = big_buffer;
3014 endptr = big_buffer + remaining;
3015 len = read(fd, endptr, available);
3016
3017 DEBUG(D_deliver) debug_printf("read() yielded %d\n", len);
3018
3019 /* If the result is EAGAIN and the process is not complete, just
3020 stop reading any more and process what we have already. */
3021
3022 if (len < 0)
3023 {
3024 if (!eop && errno == EAGAIN) len = 0; else
3025 {
3026 msg = string_sprintf("failed to read pipe from transport process "
3027 "%d for transport %s: %s", pid, addr->transport->driver_name,
3028 strerror(errno));
3029 break;
3030 }
3031 }
3032
3033 /* If the length is zero (eof or no-more-data), just process what we
3034 already have. Note that if the process is still running and we have
3035 read all the data in the pipe (but less that "available") then we
3036 won't read any more, as "unfinished" will get set FALSE. */
3037
3038 endptr += len;
bd21a787 3039 remaining += len;
059ec3d9
PH
3040 unfinished = len == available;
3041 }
3042
3043 /* If we are at the end of the available data, exit the loop. */
059ec3d9
PH
3044 if (ptr >= endptr) break;
3045
bd21a787
WB
3046 /* copy and read header */
3047 memcpy(header, ptr, PIPE_HEADER_SIZE);
3048 header[PIPE_HEADER_SIZE] = '\0';
3049 id = header[0];
3050 subid = header[1];
3051 required = Ustrtol(header + 2, &endc, 10) + PIPE_HEADER_SIZE; /* header + data */
3052 if (*endc)
3053 {
3054 msg = string_sprintf("failed to read pipe from transport process "
3055 "%d for transport %s: error reading size from header", pid, addr->transport->driver_name);
3056 done = TRUE;
3057 break;
3058 }
3059
3060 DEBUG(D_deliver)
3061 debug_printf("header read id:%c,subid:%c,size:%s,required:%d,remaining:%d,unfinished:%d\n",
3062 id, subid, header+2, required, remaining, unfinished);
3063
3064 /* is there room for the dataset we want to read ? */
3065 if (required > big_buffer_size - PIPE_HEADER_SIZE)
3066 {
3067 msg = string_sprintf("failed to read pipe from transport process "
3068 "%d for transport %s: big_buffer too small! required size=%d buffer size=%d", pid, addr->transport->driver_name,
3069 required, big_buffer_size - PIPE_HEADER_SIZE);
3070 done = TRUE;
3071 break;
3072 }
3073
3074 /* we wrote all datasets with atomic write() calls
3075 remaining < required only happens if big_buffer was too small
3076 to get all available data from pipe. unfinished has to be true
3077 as well. */
3078 if (remaining < required)
8fa0a4d4 3079 {
bd21a787
WB
3080 if (unfinished)
3081 continue;
8fa0a4d4
JH
3082 msg = string_sprintf("failed to read pipe from transport process "
3083 "%d for transport %s: required size=%d > remaining size=%d and unfinished=false",
3084 pid, addr->transport->driver_name, required, remaining);
3085 done = TRUE;
3086 break;
3087 }
bd21a787
WB
3088
3089 /* step behind the header */
3090 ptr += PIPE_HEADER_SIZE;
3091
059ec3d9
PH
3092 /* Handle each possible type of item, assuming the complete item is
3093 available in store. */
3094
bd21a787 3095 switch (id)
059ec3d9
PH
3096 {
3097 /* Host items exist only if any hosts were marked unusable. Match
3098 up by checking the IP address. */
3099
3100 case 'H':
d6c82d7b 3101 for (h = addrlist->host_list; h; h = h->next)
059ec3d9 3102 {
d6c82d7b 3103 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
059ec3d9
PH
3104 h->status = ptr[0];
3105 h->why = ptr[1];
3106 }
3107 ptr += 2;
3108 while (*ptr++);
3109 break;
3110
3111 /* Retry items are sent in a preceding R item for each address. This is
3112 kept separate to keep each message short enough to guarantee it won't
3113 be split in the pipe. Hopefully, in the majority of cases, there won't in
3114 fact be any retry items at all.
3115
3116 The complete set of retry items might include an item to delete a
3117 routing retry if there was a previous routing delay. However, routing
3118 retries are also used when a remote transport identifies an address error.
3119 In that case, there may also be an "add" item for the same key. Arrange
3120 that a "delete" item is dropped in favour of an "add" item. */
3121
3122 case 'R':
d6c82d7b 3123 if (!addr) goto ADDR_MISMATCH;
059ec3d9
PH
3124
3125 DEBUG(D_deliver|D_retry)
3126 debug_printf("reading retry information for %s from subprocess\n",
3127 ptr+1);
3128
3129 /* Cut out any "delete" items on the list. */
3130
d6c82d7b 3131 for (rp = &(addr->retries); (r = *rp); rp = &r->next)
059ec3d9
PH
3132 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3133 {
3134 if ((r->flags & rf_delete) == 0) break; /* It was not "delete" */
3135 *rp = r->next; /* Excise a delete item */
3136 DEBUG(D_deliver|D_retry)
3137 debug_printf(" existing delete item dropped\n");
3138 }
059ec3d9
PH
3139
3140 /* We want to add a delete item only if there is no non-delete item;
3141 however we still have to step ptr through the data. */
3142
d6c82d7b 3143 if (!r || (*ptr & rf_delete) == 0)
059ec3d9
PH
3144 {
3145 r = store_get(sizeof(retry_item));
3146 r->next = addr->retries;
3147 addr->retries = r;
3148 r->flags = *ptr++;
3149 r->key = string_copy(ptr);
3150 while (*ptr++);
3151 memcpy(&(r->basic_errno), ptr, sizeof(r->basic_errno));
3152 ptr += sizeof(r->basic_errno);
3153 memcpy(&(r->more_errno), ptr, sizeof(r->more_errno));
3154 ptr += sizeof(r->more_errno);
3155 r->message = (*ptr)? string_copy(ptr) : NULL;
3156 DEBUG(D_deliver|D_retry)
3157 debug_printf(" added %s item\n",
3158 ((r->flags & rf_delete) == 0)? "retry" : "delete");
3159 }
3160
3161 else
3162 {
3163 DEBUG(D_deliver|D_retry)
3164 debug_printf(" delete item not added: non-delete item exists\n");
3165 ptr++;
3166 while(*ptr++);
3167 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3168 }
3169
3170 while(*ptr++);
3171 break;
3172
3173 /* Put the amount of data written into the parlist block */
3174
3175 case 'S':
3176 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3177 ptr += sizeof(transport_count);
3178 break;
3179
3180 /* Address items are in the order of items on the address chain. We
3181 remember the current address value in case this function is called
3182 several times to empty the pipe in stages. Information about delivery
3183 over TLS is sent in a preceding X item for each address. We don't put
3184 it in with the other info, in order to keep each message short enough to
3185 guarantee it won't be split in the pipe. */
3186
a7538db1 3187#ifdef SUPPORT_TLS
059ec3d9 3188 case 'X':
d6c82d7b 3189 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
bd21a787 3190 switch (subid)
9d1c15ef
JH
3191 {
3192 case '1':
3193 addr->cipher = NULL;
3194 addr->peerdn = NULL;
3195
3196 if (*ptr)
3197 addr->cipher = string_copy(ptr);
3198 while (*ptr++);
3199 if (*ptr)
9d1c15ef 3200 addr->peerdn = string_copy(ptr);
9d1c15ef
JH
3201 break;
3202
3203 case '2':
9d1c15ef
JH
3204 if (*ptr)
3205 (void) tls_import_cert(ptr, &addr->peercert);
152e7604
JH
3206 else
3207 addr->peercert = NULL;
9d1c15ef
JH
3208 break;
3209
3210 case '3':
9d1c15ef
JH
3211 if (*ptr)
3212 (void) tls_import_cert(ptr, &addr->ourcert);
152e7604
JH
3213 else
3214 addr->ourcert = NULL;
9d1c15ef 3215 break;
018058b2 3216
a7538db1 3217# ifndef DISABLE_OCSP
018058b2
JH
3218 case '4':
3219 addr->ocsp = OCSP_NOT_REQ;
3220 if (*ptr)
3221 addr->ocsp = *ptr - '0';
3222 break;
a7538db1 3223# endif
9d1c15ef 3224 }
059ec3d9
PH
3225 while (*ptr++);
3226 break;
a7538db1 3227#endif /*SUPPORT_TLS*/
059ec3d9 3228
6f123593 3229 case 'C': /* client authenticator information */
bd21a787 3230 switch (subid)
9d1c15ef
JH
3231 {
3232 case '1':
3233 addr->authenticator = (*ptr)? string_copy(ptr) : NULL;
3234 break;
3235 case '2':
3236 addr->auth_id = (*ptr)? string_copy(ptr) : NULL;
3237 break;
3238 case '3':
3239 addr->auth_sndr = (*ptr)? string_copy(ptr) : NULL;
3240 break;
3241 }
6f123593
JH
3242 while (*ptr++);
3243 break;
3244
8ccd00b1 3245#ifndef DISABLE_PRDR
fd98a5c6 3246 case 'P':
8ccd00b1
JH
3247 addr->flags |= af_prdr_used;
3248 break;
fd98a5c6
JH
3249#endif
3250
6c1c3d1d 3251 case 'D':
d6c82d7b 3252 if (!addr) goto ADDR_MISMATCH;
6c1c3d1d
WB
3253 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3254 ptr += sizeof(addr->dsn_aware);
3255 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
3256 break;
6c1c3d1d 3257
059ec3d9 3258 case 'A':
d6c82d7b 3259 if (!addr)
059ec3d9
PH
3260 {
3261 ADDR_MISMATCH:
3262 msg = string_sprintf("address count mismatch for data read from pipe "
3263 "for transport process %d for transport %s", pid,
3264 addrlist->transport->driver_name);
3265 done = TRUE;
3266 break;
3267 }
3268
895fbaf2 3269 switch (subid)
059ec3d9 3270 {
895fbaf2
JH
3271#ifdef EXPERIMENTAL_DSN_INFO
3272 case '1': /* must arrive before A0, and applies to that addr */
3273 /* Two strings: smtp_greeting and helo_response */
3274 addr->smtp_greeting = string_copy(ptr);
3275 while(*ptr++);
3276 addr->helo_response = string_copy(ptr);
3277 while(*ptr++);
3278 break;
3279#endif
059ec3d9 3280
895fbaf2
JH
3281 case '0':
3282 addr->transport_return = *ptr++;
3283 addr->special_action = *ptr++;
3284 memcpy(&(addr->basic_errno), ptr, sizeof(addr->basic_errno));
3285 ptr += sizeof(addr->basic_errno);
3286 memcpy(&(addr->more_errno), ptr, sizeof(addr->more_errno));
3287 ptr += sizeof(addr->more_errno);
3288 memcpy(&(addr->flags), ptr, sizeof(addr->flags));
3289 ptr += sizeof(addr->flags);
3290 addr->message = (*ptr)? string_copy(ptr) : NULL;
3291 while(*ptr++);
3292 addr->user_message = (*ptr)? string_copy(ptr) : NULL;
3293 while(*ptr++);
059ec3d9 3294
895fbaf2
JH
3295 /* Always two strings for host information, followed by the port number and DNSSEC mark */
3296
3297 if (*ptr != 0)
3298 {
3299 h = store_get(sizeof(host_item));
3300 h->name = string_copy(ptr);
3301 while (*ptr++);
3302 h->address = string_copy(ptr);
3303 while(*ptr++);
3304 memcpy(&(h->port), ptr, sizeof(h->port));
3305 ptr += sizeof(h->port);
3306 h->dnssec = *ptr == '2' ? DS_YES
3307 : *ptr == '1' ? DS_NO
3308 : DS_UNK;
3309 ptr++;
3310 addr->host_used = h;
3311 }
3312 else ptr++;
3313
3314 /* Finished with this address */
3315
3316 addr = addr->next;
3317 break;
3318 }
059ec3d9
PH
3319 break;
3320
aeaf5db3
JH
3321 /* Local interface address/port */
3322 case 'I':
3323 if (*ptr) sending_ip_address = string_copy(ptr);
3324 while (*ptr++) ;
93a6fce2 3325 if (*ptr) sending_port = atoi(CS ptr);
aeaf5db3
JH
3326 while (*ptr++) ;
3327 break;
3328
059ec3d9
PH
3329 /* Z marks the logical end of the data. It is followed by '0' if
3330 continue_transport was NULL at the end of transporting, otherwise '1'.
3331 We need to know when it becomes NULL during a delivery down a passed SMTP
3332 channel so that we don't try to pass anything more down it. Of course, for
3333 most normal messages it will remain NULL all the time. */
3334
3335 case 'Z':
3336 if (*ptr == '0')
3337 {
3338 continue_transport = NULL;
3339 continue_hostname = NULL;
3340 }
3341 done = TRUE;
bd21a787 3342 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
059ec3d9
PH
3343 break;
3344
3345 /* Anything else is a disaster. */
3346
3347 default:
3348 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3349 "process %d for transport %s", ptr[-1], pid,
3350 addr->transport->driver_name);
3351 done = TRUE;
3352 break;
3353 }
3354 }
3355
3356/* The done flag is inspected externally, to determine whether or not to
3357call the function again when the process finishes. */
3358
3359p->done = done;
3360
3361/* If the process hadn't finished, and we haven't seen the end of the data
3362or suffered a disaster, update the rest of the state, and return FALSE to
3363indicate "not finished". */
3364
3365if (!eop && !done)
3366 {
3367 p->addr = addr;
3368 p->msg = msg;
3369 return FALSE;
3370 }
3371
3372/* Close our end of the pipe, to prevent deadlock if the far end is still
3373pushing stuff into it. */
3374
f1e894f3 3375(void)close(fd);
059ec3d9
PH
3376p->fd = -1;
3377
3378/* If we have finished without error, but haven't had data for every address,
3379something is wrong. */
3380
d6c82d7b 3381if (!msg && addr)
059ec3d9
PH
3382 msg = string_sprintf("insufficient address data read from pipe "
3383 "for transport process %d for transport %s", pid,
3384 addr->transport->driver_name);
3385
3386/* If an error message is set, something has gone wrong in getting back
3387the delivery data. Put the message into each address and freeze it. */
3388
d6c82d7b
JH
3389if (msg)
3390 for (addr = addrlist; addr; addr = addr->next)
059ec3d9
PH
3391 {
3392 addr->transport_return = DEFER;
3393 addr->special_action = SPECIAL_FREEZE;
3394 addr->message = msg;
3395 }
059ec3d9
PH
3396
3397/* Return TRUE to indicate we have got all we need from this process, even
3398if it hasn't actually finished yet. */
3399
3400return TRUE;
3401}
3402
3403
3404
3405/*************************************************
3406* Post-process a set of remote addresses *
3407*************************************************/
3408
3409/* Do what has to be done immediately after a remote delivery for each set of
3410addresses, then re-write the spool if necessary. Note that post_process_one
3411puts the address on an appropriate queue; hence we must fish off the next
3412one first. This function is also called if there is a problem with setting
3413up a subprocess to do a remote delivery in parallel. In this case, the final
3414argument contains a message, and the action must be forced to DEFER.
3415
3416Argument:
3417 addr pointer to chain of address items
3418 logflags flags for logging
3419 msg NULL for normal cases; -> error message for unexpected problems
3420 fallback TRUE if processing fallback hosts
3421
3422Returns: nothing
3423*/
3424
3425static void
3426remote_post_process(address_item *addr, int logflags, uschar *msg,
3427 BOOL fallback)
3428{
3429host_item *h;
3430
3431/* If any host addresses were found to be unusable, add them to the unusable
3432tree so that subsequent deliveries don't try them. */
3433
d6c82d7b
JH
3434for (h = addr->host_list; h; h = h->next)
3435 if (h->address)
3436 if (h->status >= hstatus_unusable) tree_add_unusable(h);
059ec3d9
PH
3437
3438/* Now handle each address on the chain. The transport has placed '=' or '-'
3439into the special_action field for each successful delivery. */
3440
d6c82d7b 3441while (addr)
059ec3d9
PH
3442 {
3443 address_item *next = addr->next;
3444
3445 /* If msg == NULL (normal processing) and the result is DEFER and we are
3446 processing the main hosts and there are fallback hosts available, put the
3447 address on the list for fallback delivery. */
3448
d6c82d7b
JH
3449 if ( addr->transport_return == DEFER
3450 && addr->fallback_hosts
3451 && !fallback
3452 && !msg
3453 )
059ec3d9
PH
3454 {
3455 addr->host_list = addr->fallback_hosts;
3456 addr->next = addr_fallback;
3457 addr_fallback = addr;
3458 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3459 }
3460
3461 /* If msg is set (=> unexpected problem), set it in the address before
3462 doing the ordinary post processing. */
3463
3464 else
3465 {
d6c82d7b 3466 if (msg)
059ec3d9
PH
3467 {
3468 addr->message = msg;
3469 addr->transport_return = DEFER;
3470 }
3471 (void)post_process_one(addr, addr->transport_return, logflags,
3472 DTYPE_TRANSPORT, addr->special_action);
3473 }
3474
3475 /* Next address */
3476
3477 addr = next;
3478 }
3479
3480/* If we have just delivered down a passed SMTP channel, and that was
3481the last address, the channel will have been closed down. Now that
3482we have logged that delivery, set continue_sequence to 1 so that
3483any subsequent deliveries don't get "*" incorrectly logged. */
3484
d6c82d7b 3485if (!continue_transport) continue_sequence = 1;
059ec3d9
PH
3486}
3487
3488
3489
3490/*************************************************
3491* Wait for one remote delivery subprocess *
3492*************************************************/
3493
3494/* This function is called while doing remote deliveries when either the
3495maximum number of processes exist and we need one to complete so that another
3496can be created, or when waiting for the last ones to complete. It must wait for
3497the completion of one subprocess, empty the control block slot, and return a
3498pointer to the address chain.
3499
3500Arguments: none
3501Returns: pointer to the chain of addresses handled by the process;
3502 NULL if no subprocess found - this is an unexpected error
3503*/
3504
3505static address_item *
3506par_wait(void)
3507{
3508int poffset, status;
3509address_item *addr, *addrlist;
3510pid_t pid;
3511
3512set_process_info("delivering %s: waiting for a remote delivery subprocess "
3513 "to finish", message_id);
3514
3515/* Loop until either a subprocess completes, or there are no subprocesses in
3516existence - in which case give an error return. We cannot proceed just by
3517waiting for a completion, because a subprocess may have filled up its pipe, and
3518be waiting for it to be emptied. Therefore, if no processes have finished, we
3519wait for one of the pipes to acquire some data by calling select(), with a
3520timeout just in case.
3521
3522The simple approach is just to iterate after reading data from a ready pipe.
3523This leads to non-ideal behaviour when the subprocess has written its final Z
3524item, closed the pipe, and is in the process of exiting (the common case). A
3525call to waitpid() yields nothing completed, but select() shows the pipe ready -
3526reading it yields EOF, so you end up with busy-waiting until the subprocess has
3527actually finished.
3528
3529To avoid this, if all the data that is needed has been read from a subprocess
3530after select(), an explicit wait() for it is done. We know that all it is doing
3531is writing to the pipe and then exiting, so the wait should not be long.
3532
3533The non-blocking waitpid() is to some extent just insurance; if we could
3534reliably detect end-of-file on the pipe, we could always know when to do a
3535blocking wait() for a completed process. However, because some systems use
3536NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3537use code that functions without the need to recognize EOF.
3538
3539There's a double loop here just in case we end up with a process that is not in
3540the list of remote delivery processes. Something has obviously gone wrong if
3541this is the case. (For example, a process that is incorrectly left over from
3542routing or local deliveries might be found.) The damage can be minimized by
3543looping back and looking for another process. If there aren't any, the error
3544return will happen. */
3545
3546for (;;) /* Normally we do not repeat this loop */
3547 {
3548 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3549 {
3550 struct timeval tv;
3551 fd_set select_pipes;
3552 int maxpipe, readycount;
3553
3554 /* A return value of -1 can mean several things. If errno != ECHILD, it
3555 either means invalid options (which we discount), or that this process was
3556 interrupted by a signal. Just loop to try the waitpid() again.
3557
3558 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3559 in existence. This should never happen, and is an unexpected error.
3560 However, there is a nasty complication when running under Linux. If "strace
3561 -f" is being used under Linux to trace this process and its children,
3562 subprocesses are "stolen" from their parents and become the children of the
3563 tracing process. A general wait such as the one we've just obeyed returns
3564 as if there are no children while subprocesses are running. Once a
3565 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3566 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3567 palliative.
3568
3569 This does not happen using "truss" on Solaris, nor (I think) with other
3570 tracing facilities on other OS. It seems to be specific to Linux.
3571
3572 What we do to get round this is to use kill() to see if any of our
3573 subprocesses are still in existence. If kill() gives an OK return, we know
3574 it must be for one of our processes - it can't be for a re-use of the pid,
3575 because if our process had finished, waitpid() would have found it. If any
3576 of our subprocesses are in existence, we proceed to use select() as if
3577 waitpid() had returned zero. I think this is safe. */
3578
3579 if (pid < 0)
3580 {
3581 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3582
3583 DEBUG(D_deliver)
3584 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3585 "for process existence\n");
3586
3587 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3588 {
3589 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3590 {
3591 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3592 "stolen by strace\n", (int)pid);
3593 break; /* With poffset set */
3594 }
3595 }
3596
3597 if (poffset >= remote_max_parallel)
3598 {
3599 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3600 return NULL; /* This is the error return */
3601 }
3602 }
3603
3604 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3605 been handled above. A return value of zero means that there is at least one
3606 subprocess, but there are no completed subprocesses. See if any pipes are
3607 ready with any data for reading. */
3608
3609 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3610
3611 maxpipe = 0;
3612 FD_ZERO(&select_pipes);
3613 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3614 {
3615 if (parlist[poffset].pid != 0)
3616 {
3617 int fd = parlist[poffset].fd;
3618 FD_SET(fd, &select_pipes);
3619 if (fd > maxpipe) maxpipe = fd;
3620 }
3621 }
3622
3623 /* Stick in a 60-second timeout, just in case. */
3624
3625 tv.tv_sec = 60;
3626 tv.tv_usec = 0;
3627
3628 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3629 NULL, NULL, &tv);
3630
3631 /* Scan through the pipes and read any that are ready; use the count
3632 returned by select() to stop when there are no more. Select() can return
3633 with no processes (e.g. if interrupted). This shouldn't matter.
3634
3635 If par_read_pipe() returns TRUE, it means that either the terminating Z was
3636 read, or there was a disaster. In either case, we are finished with this
3637 process. Do an explicit wait() for the process and break the main loop if
3638 it succeeds.
3639
3640 It turns out that we have to deal with the case of an interrupted system
3641 call, which can happen on some operating systems if the signal handling is
3642 set up to do that by default. */
3643
3644 for (poffset = 0;
3645 readycount > 0 && poffset < remote_max_parallel;
3646 poffset++)
3647 {
d6c82d7b
JH
3648 if ( (pid = parlist[poffset].pid) != 0
3649 && FD_ISSET(parlist[poffset].fd, &select_pipes)
3650 )
059ec3d9
PH
3651 {
3652 readycount--;
3653 if (par_read_pipe(poffset, FALSE)) /* Finished with this pipe */
3654 {
3655 for (;;) /* Loop for signals */
3656 {
3657 pid_t endedpid = waitpid(pid, &status, 0);
3658 if (endedpid == pid) goto PROCESS_DONE;
3659 if (endedpid != (pid_t)(-1) || errno != EINTR)
3660 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return "
3661 "%d (errno = %d) from waitpid() for process %d",
3662 (int)endedpid, errno, (int)pid);
3663 }
3664 }
3665 }
3666 }
3667
3668 /* Now go back and look for a completed subprocess again. */
3669 }
3670
3671 /* A completed process was detected by the non-blocking waitpid(). Find the
3672 data block that corresponds to this subprocess. */
3673
3674 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3675 if (pid == parlist[poffset].pid) break;
3676
3677 /* Found the data block; this is a known remote delivery process. We don't
3678 need to repeat the outer loop. This should be what normally happens. */
3679
3680 if (poffset < remote_max_parallel) break;
3681
3682 /* This situation is an error, but it's probably better to carry on looking
3683 for another process than to give up (as we used to do). */
3684
3685 log_write(0, LOG_MAIN|LOG_PANIC, "Process %d finished: not found in remote "
3686 "transport process list", pid);
3687 } /* End of the "for" loop */
3688
3689/* Come here when all the data was completely read after a select(), and
3690the process in pid has been wait()ed for. */
3691
3692PROCESS_DONE:
3693