string.c: do not interpret '\\' before '\0' (CVE-2019-15846)
[exim.git] / src / src / deliver.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* The main code for delivering a message. */
9
10
11#include "exim.h"
e39fdc85 12#include "transports/smtp.h"
c29da374 13#include <sys/uio.h>
dadff1d4 14#include <assert.h>
059ec3d9
PH
15
16
17/* Data block for keeping track of subprocesses for parallel remote
18delivery. */
19
20typedef struct pardata {
21 address_item *addrlist; /* chain of addresses */
22 address_item *addr; /* next address data expected for */
23 pid_t pid; /* subprocess pid */
24 int fd; /* pipe fd for getting result from subprocess */
25 int transport_count; /* returned transport count value */
26 BOOL done; /* no more data needed */
27 uschar *msg; /* error message */
28 uschar *return_path; /* return_path for these addresses */
29} pardata;
30
31/* Values for the process_recipients variable */
32
33enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
34 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
35 RECIP_FAIL_LOOP};
36
37/* Mutually recursive functions for marking addresses done. */
38
39static void child_done(address_item *, uschar *);
40static void address_done(address_item *, uschar *);
41
42/* Table for turning base-62 numbers into binary */
43
44static uschar tab62[] =
45 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
46 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
47 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
48 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
49 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
50 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
51 59,60,61}; /* x-z */
52
53
54/*************************************************
55* Local static variables *
56*************************************************/
57
58/* addr_duplicate is global because it needs to be seen from the Envelope-To
59writing code. */
60
61static address_item *addr_defer = NULL;
62static address_item *addr_failed = NULL;
63static address_item *addr_fallback = NULL;
64static address_item *addr_local = NULL;
65static address_item *addr_new = NULL;
66static address_item *addr_remote = NULL;
67static address_item *addr_route = NULL;
68static address_item *addr_succeed = NULL;
6c1c3d1d
WB
69static address_item *addr_dsntmp = NULL;
70static address_item *addr_senddsn = NULL;
059ec3d9
PH
71
72static FILE *message_log = NULL;
73static BOOL update_spool;
74static BOOL remove_journal;
75static int parcount = 0;
76static pardata *parlist = NULL;
77static int return_count;
78static uschar *frozen_info = US"";
79static uschar *used_return_path = NULL;
80
059ec3d9
PH
81
82
ee250936
JH
83/*************************************************
84* read as much as requested *
85*************************************************/
86
87/* The syscall read(2) doesn't always returns as much as we want. For
88several reasons it might get less. (Not talking about signals, as syscalls
89are restartable). When reading from a network or pipe connection the sender
90might send in smaller chunks, with delays between these chunks. The read(2)
91may return such a chunk.
92
93The more the writer writes and the smaller the pipe between write and read is,
94the more we get the chance of reading leass than requested. (See bug 2130)
95
96This function read(2)s until we got all the data we *requested*.
97
98Note: This function may block. Use it only if you're sure about the
99amount of data you will get.
100
101Argument:
102 fd the file descriptor to read from
103 buffer pointer to a buffer of size len
104 len the requested(!) amount of bytes
105
106Returns: the amount of bytes read
107*/
108static ssize_t
109readn(int fd, void * buffer, size_t len)
110{
111 void * next = buffer;
112 void * end = buffer + len;
113
114 while (next < end)
115 {
116 ssize_t got = read(fd, next, end - next);
117
118 /* I'm not sure if there are signals that can interrupt us,
119 for now I assume the worst */
120 if (got == -1 && errno == EINTR) continue;
121 if (got <= 0) return next - buffer;
122 next += got;
123 }
124
125 return len;
126}
127
128
059ec3d9
PH
129/*************************************************
130* Make a new address item *
131*************************************************/
132
133/* This function gets the store and initializes with default values. The
134transport_return value defaults to DEFER, so that any unexpected failure to
135deliver does not wipe out the message. The default unique string is set to a
136copy of the address, so that its domain can be lowercased.
137
138Argument:
139 address the RFC822 address string
140 copy force a copy of the address
141
142Returns: a pointer to an initialized address_item
143*/
144
145address_item *
146deliver_make_addr(uschar *address, BOOL copy)
147{
148address_item *addr = store_get(sizeof(address_item));
149*addr = address_defaults;
150if (copy) address = string_copy(address);
151addr->address = address;
152addr->unique = string_copy(address);
153return addr;
154}
155
156
157
158
159/*************************************************
160* Set expansion values for an address *
161*************************************************/
162
163/* Certain expansion variables are valid only when handling an address or
164address list. This function sets them up or clears the values, according to its
165argument.
166
167Arguments:
168 addr the address in question, or NULL to clear values
169Returns: nothing
170*/
171
172void
173deliver_set_expansions(address_item *addr)
174{
d6c82d7b 175if (!addr)
059ec3d9 176 {
55414b25 177 const uschar ***p = address_expansions;
d6c82d7b 178 while (*p) **p++ = NULL;
059ec3d9
PH
179 return;
180 }
181
182/* Exactly what gets set depends on whether there is one or more addresses, and
183what they contain. These first ones are always set, taking their values from
184the first address. */
185
d6c82d7b 186if (!addr->host_list)
059ec3d9
PH
187 {
188 deliver_host = deliver_host_address = US"";
a7538db1 189 deliver_host_port = 0;
059ec3d9
PH
190 }
191else
192 {
193 deliver_host = addr->host_list->name;
194 deliver_host_address = addr->host_list->address;
a7538db1 195 deliver_host_port = addr->host_list->port;
059ec3d9
PH
196 }
197
198deliver_recipients = addr;
d43cbe25
JH
199deliver_address_data = addr->prop.address_data;
200deliver_domain_data = addr->prop.domain_data;
201deliver_localpart_data = addr->prop.localpart_data;
059ec3d9
PH
202
203/* These may be unset for multiple addresses */
204
205deliver_domain = addr->domain;
206self_hostname = addr->self_hostname;
207
8523533c
TK
208#ifdef EXPERIMENTAL_BRIGHTMAIL
209bmi_deliver = 1; /* deliver by default */
210bmi_alt_location = NULL;
211bmi_base64_verdict = NULL;
212bmi_base64_tracker_verdict = NULL;
213#endif
214
059ec3d9
PH
215/* If there's only one address we can set everything. */
216
d6c82d7b 217if (!addr->next)
059ec3d9
PH
218 {
219 address_item *addr_orig;
220
221 deliver_localpart = addr->local_part;
222 deliver_localpart_prefix = addr->prefix;
223 deliver_localpart_suffix = addr->suffix;
224
d6c82d7b 225 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
059ec3d9
PH
226 deliver_domain_orig = addr_orig->domain;
227
228 /* Re-instate any prefix and suffix in the original local part. In all
229 normal cases, the address will have a router associated with it, and we can
230 choose the caseful or caseless version accordingly. However, when a system
231 filter sets up a pipe, file, or autoreply delivery, no router is involved.
232 In this case, though, there won't be any prefix or suffix to worry about. */
233
d6c82d7b
JH
234 deliver_localpart_orig = !addr_orig->router
235 ? addr_orig->local_part
236 : addr_orig->router->caseful_local_part
237 ? addr_orig->cc_local_part
238 : addr_orig->lc_local_part;
059ec3d9
PH
239
240 /* If there's a parent, make its domain and local part available, and if
241 delivering to a pipe or file, or sending an autoreply, get the local
242 part from the parent. For pipes and files, put the pipe or file string
243 into address_pipe and address_file. */
244
d6c82d7b 245 if (addr->parent)
059ec3d9
PH
246 {
247 deliver_domain_parent = addr->parent->domain;
d6c82d7b
JH
248 deliver_localpart_parent = !addr->parent->router
249 ? addr->parent->local_part
250 : addr->parent->router->caseful_local_part
251 ? addr->parent->cc_local_part
252 : addr->parent->lc_local_part;
059ec3d9
PH
253
254 /* File deliveries have their own flag because they need to be picked out
255 as special more often. */
256
257 if (testflag(addr, af_pfr))
258 {
d6c82d7b
JH
259 if (testflag(addr, af_file)) address_file = addr->local_part;
260 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
059ec3d9
PH
261 deliver_localpart = addr->parent->local_part;
262 deliver_localpart_prefix = addr->parent->prefix;
263 deliver_localpart_suffix = addr->parent->suffix;
264 }
265 }
8523533c
TK
266
267#ifdef EXPERIMENTAL_BRIGHTMAIL
268 /* Set expansion variables related to Brightmail AntiSpam */
269 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
270 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
271 /* get message delivery status (0 - don't deliver | 1 - deliver) */
272 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
273 /* if message is to be delivered, get eventual alternate location */
d6c82d7b 274 if (bmi_deliver == 1)
8523533c 275 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
8523533c
TK
276#endif
277
059ec3d9
PH
278 }
279
280/* For multiple addresses, don't set local part, and leave the domain and
f7fd3850
PH
281self_hostname set only if it is the same for all of them. It is possible to
282have multiple pipe and file addresses, but only when all addresses have routed
283to the same pipe or file. */
059ec3d9
PH
284
285else
286 {
287 address_item *addr2;
f7fd3850
PH
288 if (testflag(addr, af_pfr))
289 {
d6c82d7b
JH
290 if (testflag(addr, af_file)) address_file = addr->local_part;
291 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
f7fd3850 292 }
d6c82d7b 293 for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 294 {
d6c82d7b 295 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
059ec3d9 296 deliver_domain = NULL;
d6c82d7b
JH
297 if ( self_hostname
298 && ( !addr2->self_hostname
299 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
300 ) )
059ec3d9 301 self_hostname = NULL;
d6c82d7b 302 if (!deliver_domain && !self_hostname) break;
059ec3d9
PH
303 }
304 }
305}
306
307
308
309
310/*************************************************
311* Open a msglog file *
312*************************************************/
313
314/* This function is used both for normal message logs, and for files in the
315msglog directory that are used to catch output from pipes. Try to create the
316directory if it does not exist. From release 4.21, normal message logs should
317be created when the message is received.
318
92b0827a
JH
319Called from deliver_message(), can be operating as root.
320
059ec3d9
PH
321Argument:
322 filename the file name
323 mode the mode required
324 error used for saying what failed
325
326Returns: a file descriptor, or -1 (with errno set)
327*/
328
329static int
330open_msglog_file(uschar *filename, int mode, uschar **error)
331{
92b0827a 332int fd, i;
059ec3d9 333
92b0827a 334for (i = 2; i > 0; i--)
059ec3d9 335 {
92b0827a
JH
336 fd = Uopen(filename,
337#ifdef O_CLOEXEC
338 O_CLOEXEC |
339#endif
340#ifdef O_NOFOLLOW
341 O_NOFOLLOW |
342#endif
343 O_WRONLY|O_APPEND|O_CREAT, mode);
344 if (fd >= 0)
345 {
346 /* Set the close-on-exec flag and change the owner to the exim uid/gid (this
347 function is called as root). Double check the mode, because the group setting
348 doesn't always get set automatically. */
349
350#ifndef O_CLOEXEC
351 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
352#endif
353 if (fchown(fd, exim_uid, exim_gid) < 0)
354 {
355 *error = US"chown";
356 return -1;
357 }
358 if (fchmod(fd, mode) < 0)
359 {
360 *error = US"chmod";
361 return -1;
362 }
363 return fd;
364 }
365 if (errno != ENOENT)
366 break;
367
41313d92
JH
368 (void)directory_make(spool_directory,
369 spool_sname(US"msglog", message_subdir),
370 MSGLOG_DIRECTORY_MODE, TRUE);
059ec3d9 371 }
059ec3d9 372
92b0827a
JH
373*error = US"create";
374return -1;
059ec3d9
PH
375}
376
377
378
379
380/*************************************************
381* Write to msglog if required *
382*************************************************/
383
384/* Write to the message log, if configured. This function may also be called
385from transports.
386
387Arguments:
388 format a string format
389
390Returns: nothing
391*/
392
393void
394deliver_msglog(const char *format, ...)
395{
396va_list ap;
397if (!message_logs) return;
398va_start(ap, format);
399vfprintf(message_log, format, ap);
400fflush(message_log);
401va_end(ap);
402}
403
404
405
406
407/*************************************************
408* Replicate status for batch *
409*************************************************/
410
411/* When a transport handles a batch of addresses, it may treat them
412individually, or it may just put the status in the first one, and return FALSE,
413requesting that the status be copied to all the others externally. This is the
414replication function. As well as the status, it copies the transport pointer,
415which may have changed if appendfile passed the addresses on to a different
416transport.
417
418Argument: pointer to the first address in a chain
419Returns: nothing
420*/
421
422static void
423replicate_status(address_item *addr)
424{
425address_item *addr2;
d6c82d7b 426for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9 427 {
d6c82d7b 428 addr2->transport = addr->transport;
059ec3d9 429 addr2->transport_return = addr->transport_return;
d6c82d7b
JH
430 addr2->basic_errno = addr->basic_errno;
431 addr2->more_errno = addr->more_errno;
32dfdf8b 432 addr2->delivery_usec = addr->delivery_usec;
d6c82d7b
JH
433 addr2->special_action = addr->special_action;
434 addr2->message = addr->message;
435 addr2->user_message = addr->user_message;
059ec3d9
PH
436 }
437}
438
439
440
441/*************************************************
442* Compare lists of hosts *
443*************************************************/
444
445/* This function is given two pointers to chains of host items, and it yields
446TRUE if the lists refer to the same hosts in the same order, except that
447
448(1) Multiple hosts with the same non-negative MX values are permitted to appear
449 in different orders. Round-robinning nameservers can cause this to happen.
450
451(2) Multiple hosts with the same negative MX values less than MX_NONE are also
452 permitted to appear in different orders. This is caused by randomizing
453 hosts lists.
454
455This enables Exim to use a single SMTP transaction for sending to two entirely
456different domains that happen to end up pointing at the same hosts.
457
458Arguments:
459 one points to the first host list
460 two points to the second host list
461
462Returns: TRUE if the lists refer to the same host set
463*/
464
465static BOOL
466same_hosts(host_item *one, host_item *two)
467{
d6c82d7b 468while (one && two)
059ec3d9
PH
469 {
470 if (Ustrcmp(one->name, two->name) != 0)
471 {
472 int mx = one->mx;
473 host_item *end_one = one;
474 host_item *end_two = two;
475
476 /* Batch up only if there was no MX and the list was not randomized */
477
478 if (mx == MX_NONE) return FALSE;
479
480 /* Find the ends of the shortest sequence of identical MX values */
481
d6c82d7b
JH
482 while ( end_one->next && end_one->next->mx == mx
483 && end_two->next && end_two->next->mx == mx)
059ec3d9
PH
484 {
485 end_one = end_one->next;
486 end_two = end_two->next;
487 }
488
489 /* If there aren't any duplicates, there's no match. */
490
491 if (end_one == one) return FALSE;
492
493 /* For each host in the 'one' sequence, check that it appears in the 'two'
494 sequence, returning FALSE if not. */
495
496 for (;;)
497 {
498 host_item *hi;
499 for (hi = two; hi != end_two->next; hi = hi->next)
500 if (Ustrcmp(one->name, hi->name) == 0) break;
501 if (hi == end_two->next) return FALSE;
502 if (one == end_one) break;
503 one = one->next;
504 }
505
506 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
507 Ensure both are pointing at the last host, and carry on as for equality. */
508
509 two = end_two;
510 }
511
1c788856
JH
512 /* if the names matched but ports do not, mismatch */
513 else if (one->port != two->port)
514 return FALSE;
515
059ec3d9
PH
516 /* Hosts matched */
517
518 one = one->next;
519 two = two->next;
520 }
521
522/* True if both are NULL */
523
524return (one == two);
525}
526
527
528
529/*************************************************
530* Compare header lines *
531*************************************************/
532
533/* This function is given two pointers to chains of header items, and it yields
534TRUE if they are the same header texts in the same order.
535
536Arguments:
537 one points to the first header list
538 two points to the second header list
539
540Returns: TRUE if the lists refer to the same header set
541*/
542
543static BOOL
544same_headers(header_line *one, header_line *two)
545{
d6c82d7b 546for (;; one = one->next, two = two->next)
059ec3d9
PH
547 {
548 if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 549 if (!one || !two) return FALSE;
059ec3d9 550 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
059ec3d9
PH
551 }
552}
553
554
555
556/*************************************************
557* Compare string settings *
558*************************************************/
559
560/* This function is given two pointers to strings, and it returns
561TRUE if they are the same pointer, or if the two strings are the same.
562
563Arguments:
564 one points to the first string
565 two points to the second string
566
567Returns: TRUE or FALSE
568*/
569
570static BOOL
571same_strings(uschar *one, uschar *two)
572{
573if (one == two) return TRUE; /* Includes the case where both NULL */
d6c82d7b 574if (!one || !two) return FALSE;
059ec3d9
PH
575return (Ustrcmp(one, two) == 0);
576}
577
578
579
580/*************************************************
581* Compare uid/gid for addresses *
582*************************************************/
583
584/* This function is given a transport and two addresses. It yields TRUE if the
585uid/gid/initgroups settings for the two addresses are going to be the same when
586they are delivered.
587
588Arguments:
589 tp the transort
590 addr1 the first address
591 addr2 the second address
592
593Returns: TRUE or FALSE
594*/
595
596static BOOL
597same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
598{
d6c82d7b
JH
599if ( !tp->uid_set && !tp->expand_uid
600 && !tp->deliver_as_creator
601 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
602 || ( testflag(addr1, af_uid_set)
603 && ( addr1->uid != addr2->uid
604 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
605 ) ) ) )
606 return FALSE;
059ec3d9 607
d6c82d7b
JH
608if ( !tp->gid_set && !tp->expand_gid
609 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
610 || ( testflag(addr1, af_gid_set)
611 && addr1->gid != addr2->gid
612 ) ) )
613 return FALSE;
059ec3d9
PH
614
615return TRUE;
616}
617
618
619
620
621/*************************************************
622* Record that an address is complete *
623*************************************************/
624
625/* This function records that an address is complete. This is straightforward
626for most addresses, where the unique address is just the full address with the
627domain lower cased. For homonyms (addresses that are the same as one of their
628ancestors) their are complications. Their unique addresses have \x\ prepended
629(where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
630cousins.
631
632Exim used to record the unique addresses of homonyms as "complete". This,
633however, fails when the pattern of redirection varies over time (e.g. if taking
634unseen copies at only some times of day) because the prepended numbers may vary
635from one delivery run to the next. This problem is solved by never recording
636prepended unique addresses as complete. Instead, when a homonymic address has
637actually been delivered via a transport, we record its basic unique address
638followed by the name of the transport. This is checked in subsequent delivery
639runs whenever an address is routed to a transport.
640
641If the completed address is a top-level one (has no parent, which means it
642cannot be homonymic) we also add the original address to the non-recipients
643tree, so that it gets recorded in the spool file and therefore appears as
644"done" in any spool listings. The original address may differ from the unique
645address in the case of the domain.
646
647Finally, this function scans the list of duplicates, marks as done any that
648match this address, and calls child_done() for their ancestors.
649
650Arguments:
651 addr address item that has been completed
652 now current time as a string
653
654Returns: nothing
655*/
656
657static void
658address_done(address_item *addr, uschar *now)
659{
660address_item *dup;
661
662update_spool = TRUE; /* Ensure spool gets updated */
663
664/* Top-level address */
665
d6c82d7b 666if (!addr->parent)
059ec3d9
PH
667 {
668 tree_add_nonrecipient(addr->unique);
669 tree_add_nonrecipient(addr->address);
670 }
671
672/* Homonymous child address */
673
674else if (testflag(addr, af_homonym))
675 {
d6c82d7b 676 if (addr->transport)
059ec3d9
PH
677 tree_add_nonrecipient(
678 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
059ec3d9
PH
679 }
680
681/* Non-homonymous child address */
682
683else tree_add_nonrecipient(addr->unique);
684
685/* Check the list of duplicate addresses and ensure they are now marked
686done as well. */
687
d6c82d7b 688for (dup = addr_duplicate; dup; dup = dup->next)
059ec3d9
PH
689 if (Ustrcmp(addr->unique, dup->unique) == 0)
690 {
57730b52 691 tree_add_nonrecipient(dup->unique);
059ec3d9
PH
692 child_done(dup, now);
693 }
059ec3d9
PH
694}
695
696
697
698
699/*************************************************
700* Decrease counts in parents and mark done *
701*************************************************/
702
703/* This function is called when an address is complete. If there is a parent
704address, its count of children is decremented. If there are still other
705children outstanding, the function exits. Otherwise, if the count has become
706zero, address_done() is called to mark the parent and its duplicates complete.
707Then loop for any earlier ancestors.
708
709Arguments:
710 addr points to the completed address item
711 now the current time as a string, for writing to the message log
712
713Returns: nothing
714*/
715
716static void
717child_done(address_item *addr, uschar *now)
718{
719address_item *aa;
d6c82d7b 720while (addr->parent)
059ec3d9
PH
721 {
722 addr = addr->parent;
82f90600 723 if (--addr->child_count > 0) return; /* Incomplete parent */
059ec3d9
PH
724 address_done(addr, now);
725
726 /* Log the completion of all descendents only when there is no ancestor with
727 the same original address. */
728
d6c82d7b 729 for (aa = addr->parent; aa; aa = aa->parent)
059ec3d9 730 if (Ustrcmp(aa->address, addr->address) == 0) break;
d6c82d7b 731 if (aa) continue;
059ec3d9
PH
732
733 deliver_msglog("%s %s: children all complete\n", now, addr->address);
734 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
735 }
736}
737
738
739
ac881e27
TF
740/*************************************************
741* Delivery logging support functions *
742*************************************************/
743
744/* The LOGGING() checks in d_log_interface() are complicated for backwards
745compatibility. When outgoing interface logging was originally added, it was
746conditional on just incoming_interface (which is off by default). The
747outgoing_interface option is on by default to preserve this behaviour, but
748you can enable incoming_interface and disable outgoing_interface to get I=
749fields on incoming lines only.
750
751Arguments:
acec9514 752 g The log line
ac881e27
TF
753 addr The address to be logged
754
755Returns: New value for s
756*/
059ec3d9 757
acec9514
JH
758static gstring *
759d_log_interface(gstring * g)
67d81c10 760{
ac881e27 761if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
d6c82d7b 762 && sending_ip_address)
ac881e27 763 {
d12746bc
JH
764 g = string_fmt_append(g, " I=[%s]", sending_ip_address);
765 if (LOGGING(outgoing_port))
766 g = string_fmt_append(g, "%d", sending_port);
ac881e27 767 }
acec9514 768return g;
67d81c10
JH
769}
770
ac881e27
TF
771
772
acec9514
JH
773static gstring *
774d_hostlog(gstring * g, address_item * addr)
ac881e27 775{
fc16abb4
JH
776host_item * h = addr->host_used;
777
acec9514 778g = string_append(g, 2, US" H=", h->name);
fc16abb4
JH
779
780if (LOGGING(dnssec) && h->dnssec == DS_YES)
acec9514 781 g = string_catn(g, US" DS", 3);
fc16abb4 782
acec9514 783g = string_append(g, 3, US" [", h->address, US"]");
fc16abb4 784
ac881e27 785if (LOGGING(outgoing_port))
d12746bc 786 g = string_fmt_append(g, ":%d", h->port);
e6d2a989
JH
787
788#ifdef SUPPORT_SOCKS
789if (LOGGING(proxy) && proxy_local_address)
790 {
acec9514 791 g = string_append(g, 3, US" PRX=[", proxy_local_address, US"]");
e6d2a989 792 if (LOGGING(outgoing_port))
d12746bc 793 g = string_fmt_append(g, ":%d", proxy_local_port);
e6d2a989
JH
794 }
795#endif
796
acec9514 797g = d_log_interface(g);
a2673768
JH
798
799if (testflag(addr, af_tcp_fastopen))
ee8b8090 800 g = string_catn(g, US" TFO*", testflag(addr, af_tcp_fastopen_data) ? 5 : 4);
a2673768 801
acec9514 802return g;
ac881e27
TF
803}
804
805
806
e6d2a989
JH
807
808
67d81c10 809#ifdef SUPPORT_TLS
acec9514
JH
810static gstring *
811d_tlslog(gstring * s, address_item * addr)
67d81c10 812{
d6c82d7b 813if (LOGGING(tls_cipher) && addr->cipher)
acec9514 814 s = string_append(s, 2, US" X=", addr->cipher);
d6c82d7b 815if (LOGGING(tls_certificate_verified) && addr->cipher)
acec9514 816 s = string_append(s, 2, US" CV=",
ac881e27
TF
817 testflag(addr, af_cert_verified)
818 ?
c0635b6d 819#ifdef SUPPORT_DANE
ac881e27
TF
820 testflag(addr, af_dane_verified)
821 ? "dane"
822 :
53a7196b 823#endif
ac881e27
TF
824 "yes"
825 : "no");
d6c82d7b 826if (LOGGING(tls_peerdn) && addr->peerdn)
acec9514 827 s = string_append(s, 3, US" DN=\"", string_printing(addr->peerdn), US"\"");
ac881e27 828return s;
67d81c10
JH
829}
830#endif
831
a7538db1 832
14a465c3
JH
833
834
0cbf2b82 835#ifndef DISABLE_EVENT
b30275b8 836uschar *
55414b25 837event_raise(uschar * action, const uschar * event, uschar * ev_data)
a7538db1
JH
838{
839uschar * s;
840if (action)
841 {
842 DEBUG(D_deliver)
774ef2d7 843 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
a7538db1
JH
844 event,
845 action, deliver_host_address);
846
774ef2d7
JH
847 event_name = event;
848 event_data = ev_data;
a7538db1
JH
849
850 if (!(s = expand_string(action)) && *expand_string_message)
851 log_write(0, LOG_MAIN|LOG_PANIC,
774ef2d7 852 "failed to expand event_action %s in %s: %s\n",
7ea1237c 853 event, transport_name ? transport_name : US"main", expand_string_message);
a7538db1 854
774ef2d7 855 event_name = event_data = NULL;
a7538db1
JH
856
857 /* If the expansion returns anything but an empty string, flag for
858 the caller to modify his normal processing
859 */
860 if (s && *s)
861 {
862 DEBUG(D_deliver)
774ef2d7 863 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
b30275b8 864 return s;
a7538db1
JH
865 }
866 }
b30275b8 867return NULL;
a7538db1 868}
14a465c3 869
5ef5dd52 870void
55414b25 871msg_event_raise(const uschar * event, const address_item * addr)
14a465c3 872{
55414b25 873const uschar * save_domain = deliver_domain;
14a465c3 874uschar * save_local = deliver_localpart;
55414b25 875const uschar * save_host = deliver_host;
5ef5dd52
JB
876const uschar * save_address = deliver_host_address;
877const int save_port = deliver_host_port;
14a465c3 878
14a465c3 879router_name = addr->router ? addr->router->name : NULL;
14a465c3
JH
880deliver_domain = addr->domain;
881deliver_localpart = addr->local_part;
c562fd30 882deliver_host = addr->host_used ? addr->host_used->name : NULL;
14a465c3 883
4e928780
MK
884if (!addr->transport)
885 {
886 if (Ustrcmp(event, "msg:fail:delivery") == 0)
887 {
888 /* An address failed with no transport involved. This happens when
889 a filter was used which triggered a fail command (in such a case
890 a transport isn't needed). Convert it to an internal fail event. */
891
892 (void) event_raise(event_action, US"msg:fail:internal", addr->message);
893 }
894 }
895else
896 {
897 transport_name = addr->transport->name;
898
899 (void) event_raise(addr->transport->event_action, event,
900 addr->host_used
901 || Ustrcmp(addr->transport->driver_name, "smtp") == 0
902 || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
903 || Ustrcmp(addr->transport->driver_name, "autoreply") == 0
904 ? addr->message : NULL);
905 }
14a465c3 906
5ef5dd52
JB
907deliver_host_port = save_port;
908deliver_host_address = save_address;
c562fd30 909deliver_host = save_host;
14a465c3
JH
910deliver_localpart = save_local;
911deliver_domain = save_domain;
912router_name = transport_name = NULL;
913}
0cbf2b82 914#endif /*DISABLE_EVENT*/
14a465c3
JH
915
916
a7538db1 917
5dac72a7
JH
918/******************************************************************************/
919
920
921/*************************************************
922* Generate local prt for logging *
923*************************************************/
924
925/* This function is a subroutine for use in string_log_address() below.
926
927Arguments:
928 addr the address being logged
929 yield the current dynamic buffer pointer
5dac72a7
JH
930
931Returns: the new value of the buffer pointer
932*/
933
acec9514
JH
934static gstring *
935string_get_localpart(address_item * addr, gstring * yield)
5dac72a7
JH
936{
937uschar * s;
938
939s = addr->prefix;
940if (testflag(addr, af_include_affixes) && s)
941 {
942#ifdef SUPPORT_I18N
943 if (testflag(addr, af_utf8_downcvt))
944 s = string_localpart_utf8_to_alabel(s, NULL);
945#endif
acec9514 946 yield = string_cat(yield, s);
5dac72a7
JH
947 }
948
949s = addr->local_part;
950#ifdef SUPPORT_I18N
951if (testflag(addr, af_utf8_downcvt))
952 s = string_localpart_utf8_to_alabel(s, NULL);
953#endif
acec9514 954yield = string_cat(yield, s);
5dac72a7
JH
955
956s = addr->suffix;
957if (testflag(addr, af_include_affixes) && s)
958 {
959#ifdef SUPPORT_I18N
960 if (testflag(addr, af_utf8_downcvt))
961 s = string_localpart_utf8_to_alabel(s, NULL);
962#endif
acec9514 963 yield = string_cat(yield, s);
5dac72a7
JH
964 }
965
966return yield;
967}
968
969
970/*************************************************
971* Generate log address list *
972*************************************************/
973
974/* This function generates a list consisting of an address and its parents, for
975use in logging lines. For saved onetime aliased addresses, the onetime parent
976field is used. If the address was delivered by a transport with rcpt_include_
977affixes set, the af_include_affixes bit will be set in the address. In that
978case, we include the affixes here too.
979
980Arguments:
acec9514 981 g points to growing-string struct
5dac72a7
JH
982 addr bottom (ultimate) address
983 all_parents if TRUE, include all parents
984 success TRUE for successful delivery
985
986Returns: a growable string in dynamic store
987*/
988
acec9514
JH
989static gstring *
990string_log_address(gstring * g,
5dac72a7
JH
991 address_item *addr, BOOL all_parents, BOOL success)
992{
993BOOL add_topaddr = TRUE;
994address_item *topaddr;
995
996/* Find the ultimate parent */
997
998for (topaddr = addr; topaddr->parent; topaddr = topaddr->parent) ;
999
1000/* We start with just the local part for pipe, file, and reply deliveries, and
1001for successful local deliveries from routers that have the log_as_local flag
1002set. File deliveries from filters can be specified as non-absolute paths in
4c04137d 1003cases where the transport is going to complete the path. If there is an error
5dac72a7
JH
1004before this happens (expansion failure) the local part will not be updated, and
1005so won't necessarily look like a path. Add extra text for this case. */
1006
1007if ( testflag(addr, af_pfr)
1008 || ( success
1009 && addr->router && addr->router->log_as_local
1010 && addr->transport && addr->transport->info->local
1011 ) )
1012 {
1013 if (testflag(addr, af_file) && addr->local_part[0] != '/')
acec9514
JH
1014 g = string_catn(g, CUS"save ", 5);
1015 g = string_get_localpart(addr, g);
5dac72a7
JH
1016 }
1017
1018/* Other deliveries start with the full address. It we have split it into local
1019part and domain, use those fields. Some early failures can happen before the
1020splitting is done; in those cases use the original field. */
1021
1022else
1023 {
acec9514 1024 uschar * cmp = g->s + g->ptr;
5dac72a7
JH
1025
1026 if (addr->local_part)
1027 {
1028 const uschar * s;
acec9514
JH
1029 g = string_get_localpart(addr, g);
1030 g = string_catn(g, US"@", 1);
5dac72a7
JH
1031 s = addr->domain;
1032#ifdef SUPPORT_I18N
1033 if (testflag(addr, af_utf8_downcvt))
1034 s = string_localpart_utf8_to_alabel(s, NULL);
1035#endif
acec9514 1036 g = string_cat(g, s);
5dac72a7
JH
1037 }
1038 else
acec9514 1039 g = string_cat(g, addr->address);
5dac72a7
JH
1040
1041 /* If the address we are going to print is the same as the top address,
1042 and all parents are not being included, don't add on the top address. First
1043 of all, do a caseless comparison; if this succeeds, do a caseful comparison
1044 on the local parts. */
1045
306c6c77 1046 string_from_gstring(g); /* ensure nul-terminated */
5dac72a7
JH
1047 if ( strcmpic(cmp, topaddr->address) == 0
1048 && Ustrncmp(cmp, topaddr->address, Ustrchr(cmp, '@') - cmp) == 0
1049 && !addr->onetime_parent
1050 && (!all_parents || !addr->parent || addr->parent == topaddr)
1051 )
1052 add_topaddr = FALSE;
1053 }
1054
1055/* If all parents are requested, or this is a local pipe/file/reply, and
1056there is at least one intermediate parent, show it in brackets, and continue
1057with all of them if all are wanted. */
1058
1059if ( (all_parents || testflag(addr, af_pfr))
1060 && addr->parent
1061 && addr->parent != topaddr)
1062 {
1063 uschar *s = US" (";
1064 address_item *addr2;
1065 for (addr2 = addr->parent; addr2 != topaddr; addr2 = addr2->parent)
1066 {
acec9514
JH
1067 g = string_catn(g, s, 2);
1068 g = string_cat (g, addr2->address);
5dac72a7
JH
1069 if (!all_parents) break;
1070 s = US", ";
1071 }
acec9514 1072 g = string_catn(g, US")", 1);
5dac72a7
JH
1073 }
1074
1075/* Add the top address if it is required */
1076
1077if (add_topaddr)
acec9514 1078 g = string_append(g, 3,
5dac72a7
JH
1079 US" <",
1080 addr->onetime_parent ? addr->onetime_parent : topaddr->address,
1081 US">");
1082
acec9514 1083return g;
5dac72a7
JH
1084}
1085
1086
32dfdf8b
JH
1087
1088void
1089timesince(struct timeval * diff, struct timeval * then)
1090{
1091gettimeofday(diff, NULL);
1092diff->tv_sec -= then->tv_sec;
1093if ((diff->tv_usec -= then->tv_usec) < 0)
1094 {
1095 diff->tv_sec--;
1096 diff->tv_usec += 1000*1000;
1097 }
1098}
1099
1100
1101
306c6c77 1102uschar *
32dfdf8b
JH
1103string_timediff(struct timeval * diff)
1104{
1105static uschar buf[sizeof("0.000s")];
1106
1107if (diff->tv_sec >= 5 || !LOGGING(millisec))
1108 return readconf_printtime((int)diff->tv_sec);
1109
13c7874e 1110sprintf(CS buf, "%u.%03us", (uint)diff->tv_sec, (uint)diff->tv_usec/1000);
32dfdf8b
JH
1111return buf;
1112}
1113
1114
0f1a8658 1115uschar *
32dfdf8b
JH
1116string_timesince(struct timeval * then)
1117{
1118struct timeval diff;
1119
1120timesince(&diff, then);
1121return string_timediff(&diff);
1122}
1123
5dac72a7
JH
1124/******************************************************************************/
1125
1126
1127
817d9f57 1128/* If msg is NULL this is a delivery log and logchar is used. Otherwise
67d81c10 1129this is a nonstandard call; no two-character delivery flag is written
817d9f57
JH
1130but sender-host and sender are prefixed and "msg" is inserted in the log line.
1131
1132Arguments:
1133 flags passed to log_write()
1134*/
e4bdf652 1135void
817d9f57 1136delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
e4bdf652 1137{
acec9514 1138gstring * g; /* Used for a temporary, expanding buffer, for building log lines */
5dac72a7 1139void * reset_point; /* released afterwards. */
e4bdf652 1140
e4bdf652
JH
1141/* Log the delivery on the main log. We use an extensible string to build up
1142the log line, and reset the store afterwards. Remote deliveries should always
1143have a pointer to the host item that succeeded; local deliveries can have a
1144pointer to a single host item in their host list, for use by the transport. */
1145
0cbf2b82 1146#ifndef DISABLE_EVENT
a7538db1 1147 /* presume no successful remote delivery */
783b385f 1148 lookup_dnssec_authenticated = NULL;
d68218c7
JH
1149#endif
1150
acec9514 1151g = reset_point = string_get(256);
e4bdf652 1152
817d9f57 1153if (msg)
acec9514 1154 g = string_append(g, 2, host_and_ident(TRUE), US" ");
817d9f57
JH
1155else
1156 {
acec9514
JH
1157 g->s[0] = logchar; g->ptr = 1;
1158 g = string_catn(g, US"> ", 2);
817d9f57 1159 }
acec9514 1160g = string_log_address(g, addr, LOGGING(all_parents), TRUE);
e4bdf652 1161
6c6d6e48 1162if (LOGGING(sender_on_delivery) || msg)
acec9514 1163 g = string_append(g, 3, US" F=<",
8c5d388a 1164#ifdef SUPPORT_I18N
3c8b3577
JH
1165 testflag(addr, af_utf8_downcvt)
1166 ? string_address_utf8_to_alabel(sender_address, NULL)
1167 :
1168#endif
1169 sender_address,
1170 US">");
e4bdf652 1171
afa6d3ad 1172if (*queue_name)
acec9514 1173 g = string_append(g, 2, US" Q=", queue_name);
afa6d3ad 1174
e4bdf652 1175#ifdef EXPERIMENTAL_SRS
d43cbe25 1176if(addr->prop.srs_sender)
acec9514 1177 g = string_append(g, 3, US" SRS=<", addr->prop.srs_sender, US">");
e4bdf652
JH
1178#endif
1179
1180/* You might think that the return path must always be set for a successful
1181delivery; indeed, I did for some time, until this statement crashed. The case
1182when it is not set is for a delivery to /dev/null which is optimised by not
1183being run at all. */
1184
d6c82d7b 1185if (used_return_path && LOGGING(return_path_on_delivery))
acec9514 1186 g = string_append(g, 3, US" P=<", used_return_path, US">");
e4bdf652 1187
817d9f57 1188if (msg)
acec9514 1189 g = string_append(g, 2, US" ", msg);
e4bdf652 1190
817d9f57 1191/* For a delivery from a system filter, there may not be a router */
d6c82d7b 1192if (addr->router)
acec9514 1193 g = string_append(g, 2, US" R=", addr->router->name);
e4bdf652 1194
acec9514 1195g = string_append(g, 2, US" T=", addr->transport->name);
e4bdf652 1196
6c6d6e48 1197if (LOGGING(delivery_size))
d12746bc 1198 g = string_fmt_append(g, " S=%d", transport_count);
e4bdf652
JH
1199
1200/* Local delivery */
1201
1202if (addr->transport->info->local)
1203 {
a7538db1 1204 if (addr->host_list)
acec9514
JH
1205 g = string_append(g, 2, US" H=", addr->host_list->name);
1206 g = d_log_interface(g);
d6c82d7b 1207 if (addr->shadow_message)
acec9514 1208 g = string_cat(g, addr->shadow_message);
e4bdf652
JH
1209 }
1210
1211/* Remote delivery */
1212
1213else
1214 {
783b385f 1215 if (addr->host_used)
e4bdf652 1216 {
acec9514 1217 g = d_hostlog(g, addr);
e4bdf652 1218 if (continue_sequence > 1)
acec9514 1219 g = string_catn(g, US"*", 1);
d68218c7 1220
0cbf2b82 1221#ifndef DISABLE_EVENT
a7538db1
JH
1222 deliver_host_address = addr->host_used->address;
1223 deliver_host_port = addr->host_used->port;
029f4192 1224 deliver_host = addr->host_used->name;
783b385f
JH
1225
1226 /* DNS lookup status */
1227 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
1228 : addr->host_used->dnssec==DS_NO ? US"no"
1229 : NULL;
a7538db1 1230#endif
e4bdf652
JH
1231 }
1232
a7538db1 1233#ifdef SUPPORT_TLS
acec9514 1234 g = d_tlslog(g, addr);
a7538db1 1235#endif
e4bdf652 1236
018c60d7 1237 if (addr->authenticator)
6f123593 1238 {
acec9514 1239 g = string_append(g, 2, US" A=", addr->authenticator);
018c60d7 1240 if (addr->auth_id)
c8e2fc1e 1241 {
acec9514 1242 g = string_append(g, 2, US":", addr->auth_id);
6c6d6e48 1243 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
acec9514 1244 g = string_append(g, 2, US":", addr->auth_sndr);
c8e2fc1e 1245 }
6f123593
JH
1246 }
1247
ee8b8090
JH
1248 if (LOGGING(pipelining))
1249 {
1250 if (testflag(addr, af_pipelining))
1251 g = string_catn(g, US" L", 2);
1252#ifdef EXPERIMENTAL_PIPE_CONNECT
1253 if (testflag(addr, af_early_pipe))
1254 g = string_catn(g, US"*", 1);
1255#endif
1256 }
7c576fca 1257
a7538db1 1258#ifndef DISABLE_PRDR
7eb0e5d2 1259 if (testflag(addr, af_prdr_used))
acec9514 1260 g = string_catn(g, US" PRDR", 5);
a7538db1 1261#endif
e0cc6cda 1262
7eb0e5d2 1263 if (testflag(addr, af_chunking_used))
acec9514 1264 g = string_catn(g, US" K", 2);
76f44207
WB
1265 }
1266
1267/* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
fd98a5c6 1268
d6c82d7b
JH
1269if ( LOGGING(smtp_confirmation)
1270 && addr->message
1271 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
1272 )
76f44207 1273 {
b980ed83 1274 unsigned i;
1c5b1499 1275 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
76f44207
WB
1276 uschar *p = big_buffer;
1277 uschar *ss = addr->message;
1278 *p++ = '\"';
b980ed83 1279 for (i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
e4bdf652 1280 {
27520a68 1281 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
76f44207 1282 *p++ = ss[i];
e4bdf652 1283 }
76f44207
WB
1284 *p++ = '\"';
1285 *p = 0;
acec9514 1286 g = string_append(g, 2, US" C=", big_buffer);
e4bdf652
JH
1287 }
1288
1289/* Time on queue and actual time taken to deliver */
1290
6c6d6e48 1291if (LOGGING(queue_time))
acec9514 1292 g = string_append(g, 2, US" QT=",
32dfdf8b 1293 string_timesince(&received_time));
e4bdf652 1294
6c6d6e48 1295if (LOGGING(deliver_time))
32dfdf8b 1296 {
f2ed27cf 1297 struct timeval diff = {.tv_sec = addr->more_errno, .tv_usec = addr->delivery_usec};
acec9514 1298 g = string_append(g, 2, US" DT=", string_timediff(&diff));
32dfdf8b 1299 }
e4bdf652
JH
1300
1301/* string_cat() always leaves room for the terminator. Release the
1302store we used to build the line after writing it. */
1303
acec9514 1304log_write(0, flags, "%s", string_from_gstring(g));
d68218c7 1305
0cbf2b82 1306#ifndef DISABLE_EVENT
774ef2d7 1307if (!msg) msg_event_raise(US"msg:delivery", addr);
d68218c7 1308#endif
14a465c3 1309
e4bdf652
JH
1310store_reset(reset_point);
1311return;
1312}
1313
1314
1315
5dac72a7
JH
1316static void
1317deferral_log(address_item * addr, uschar * now,
1318 int logflags, uschar * driver_name, uschar * driver_kind)
1319{
acec9514
JH
1320gstring * g;
1321void * reset_point;
5dac72a7
JH
1322
1323/* Build up the line that is used for both the message log and the main
1324log. */
1325
acec9514 1326g = reset_point = string_get(256);
5dac72a7
JH
1327
1328/* Create the address string for logging. Must not do this earlier, because
1329an OK result may be changed to FAIL when a pipe returns text. */
1330
acec9514 1331g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
5dac72a7
JH
1332
1333if (*queue_name)
acec9514 1334 g = string_append(g, 2, US" Q=", queue_name);
5dac72a7
JH
1335
1336/* Either driver_name contains something and driver_kind contains
1337" router" or " transport" (note the leading space), or driver_name is
1338a null string and driver_kind contains "routing" without the leading
1339space, if all routing has been deferred. When a domain has been held,
1340so nothing has been done at all, both variables contain null strings. */
1341
1342if (driver_name)
1343 {
1344 if (driver_kind[1] == 't' && addr->router)
acec9514 1345 g = string_append(g, 2, US" R=", addr->router->name);
d12746bc 1346 g = string_fmt_append(g, " %c=%s", toupper(driver_kind[1]), driver_name);
5dac72a7
JH
1347 }
1348else if (driver_kind)
acec9514 1349 g = string_append(g, 2, US" ", driver_kind);
5dac72a7 1350
d12746bc 1351g = string_fmt_append(g, " defer (%d)", addr->basic_errno);
5dac72a7
JH
1352
1353if (addr->basic_errno > 0)
acec9514 1354 g = string_append(g, 2, US": ",
5dac72a7
JH
1355 US strerror(addr->basic_errno));
1356
1357if (addr->host_used)
1358 {
acec9514 1359 g = string_append(g, 5,
5dac72a7
JH
1360 US" H=", addr->host_used->name,
1361 US" [", addr->host_used->address, US"]");
1362 if (LOGGING(outgoing_port))
1363 {
1364 int port = addr->host_used->port;
d12746bc 1365 g = string_fmt_append(g, ":%d", port == PORT_NONE ? 25 : port);
5dac72a7
JH
1366 }
1367 }
1368
1369if (addr->message)
acec9514 1370 g = string_append(g, 2, US": ", addr->message);
5dac72a7 1371
acec9514 1372(void) string_from_gstring(g);
5dac72a7
JH
1373
1374/* Log the deferment in the message log, but don't clutter it
1375up with retry-time defers after the first delivery attempt. */
1376
8768d548 1377if (f.deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
acec9514 1378 deliver_msglog("%s %s\n", now, g->s);
5dac72a7
JH
1379
1380/* Write the main log and reset the store.
1381For errors of the type "retry time not reached" (also remotes skipped
1382on queue run), logging is controlled by L_retry_defer. Note that this kind
1383of error number is negative, and all the retry ones are less than any
1384others. */
1385
1386
1387log_write(addr->basic_errno <= ERRNO_RETRY_BASE ? L_retry_defer : 0, logflags,
acec9514 1388 "== %s", g->s);
5dac72a7
JH
1389
1390store_reset(reset_point);
1391return;
1392}
1393
1394
1395
1396static void
1397failure_log(address_item * addr, uschar * driver_kind, uschar * now)
1398{
acec9514
JH
1399void * reset_point;
1400gstring * g = reset_point = string_get(256);
5dac72a7 1401
ffbc20ed
MK
1402#ifndef DISABLE_EVENT
1403/* Message failures for which we will send a DSN get their event raised
1404later so avoid doing it here. */
1405
1406if ( !addr->prop.ignore_error
1407 && !(addr->dsn_flags & (rf_dsnflags & ~rf_notify_failure))
1408 )
1409 msg_event_raise(US"msg:fail:delivery", addr);
1410#endif
1411
5dac72a7
JH
1412/* Build up the log line for the message and main logs */
1413
5dac72a7
JH
1414/* Create the address string for logging. Must not do this earlier, because
1415an OK result may be changed to FAIL when a pipe returns text. */
1416
acec9514 1417g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
5dac72a7
JH
1418
1419if (LOGGING(sender_on_delivery))
acec9514 1420 g = string_append(g, 3, US" F=<", sender_address, US">");
5dac72a7
JH
1421
1422if (*queue_name)
acec9514 1423 g = string_append(g, 2, US" Q=", queue_name);
5dac72a7
JH
1424
1425/* Return path may not be set if no delivery actually happened */
1426
1427if (used_return_path && LOGGING(return_path_on_delivery))
acec9514 1428 g = string_append(g, 3, US" P=<", used_return_path, US">");
5dac72a7
JH
1429
1430if (addr->router)
acec9514 1431 g = string_append(g, 2, US" R=", addr->router->name);
5dac72a7 1432if (addr->transport)
acec9514 1433 g = string_append(g, 2, US" T=", addr->transport->name);
5dac72a7
JH
1434
1435if (addr->host_used)
acec9514 1436 g = d_hostlog(g, addr);
5dac72a7
JH
1437
1438#ifdef SUPPORT_TLS
acec9514 1439g = d_tlslog(g, addr);
5dac72a7
JH
1440#endif
1441
1442if (addr->basic_errno > 0)
acec9514 1443 g = string_append(g, 2, US": ", US strerror(addr->basic_errno));
5dac72a7
JH
1444
1445if (addr->message)
acec9514 1446 g = string_append(g, 2, US": ", addr->message);
5dac72a7 1447
acec9514 1448(void) string_from_gstring(g);
5dac72a7
JH
1449
1450/* Do the logging. For the message log, "routing failed" for those cases,
1451just to make it clearer. */
1452
1453if (driver_kind)
acec9514 1454 deliver_msglog("%s %s failed for %s\n", now, driver_kind, g->s);
5dac72a7 1455else
acec9514 1456 deliver_msglog("%s %s\n", now, g->s);
5dac72a7 1457
acec9514 1458log_write(0, LOG_MAIN, "** %s", g->s);
5dac72a7 1459
5dac72a7
JH
1460store_reset(reset_point);
1461return;
1462}
1463
1464
1465
059ec3d9
PH
1466/*************************************************
1467* Actions at the end of handling an address *
1468*************************************************/
1469
1470/* This is a function for processing a single address when all that can be done
1471with it has been done.
1472
1473Arguments:
1474 addr points to the address block
1475 result the result of the delivery attempt
1476 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1477 driver_type indicates which type of driver (transport, or router) was last
1478 to process the address
1479 logchar '=' or '-' for use when logging deliveries with => or ->
1480
1481Returns: nothing
1482*/
1483
1484static void
1485post_process_one(address_item *addr, int result, int logflags, int driver_type,
1486 int logchar)
1487{
1488uschar *now = tod_stamp(tod_log);
1489uschar *driver_kind = NULL;
1490uschar *driver_name = NULL;
059ec3d9 1491
059ec3d9
PH
1492DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1493
1494/* Set up driver kind and name for logging. Disable logging if the router or
1495transport has disabled it. */
1496
ba936fb8 1497if (driver_type == EXIM_DTYPE_TRANSPORT)
059ec3d9 1498 {
d6c82d7b 1499 if (addr->transport)
059ec3d9
PH
1500 {
1501 driver_name = addr->transport->name;
1502 driver_kind = US" transport";
8768d548 1503 f.disable_logging = addr->transport->disable_logging;
059ec3d9
PH
1504 }
1505 else driver_kind = US"transporting";
1506 }
ba936fb8 1507else if (driver_type == EXIM_DTYPE_ROUTER)
059ec3d9 1508 {
d6c82d7b 1509 if (addr->router)
059ec3d9
PH
1510 {
1511 driver_name = addr->router->name;
1512 driver_kind = US" router";
8768d548 1513 f.disable_logging = addr->router->disable_logging;
059ec3d9
PH
1514 }
1515 else driver_kind = US"routing";
1516 }
1517
1518/* If there's an error message set, ensure that it contains only printing
1519characters - it should, but occasionally things slip in and this at least
49c2d5ea
PH
1520stops the log format from getting wrecked. We also scan the message for an LDAP
1521expansion item that has a password setting, and flatten the password. This is a
1522fudge, but I don't know a cleaner way of doing this. (If the item is badly
1523malformed, it won't ever have gone near LDAP.) */
059ec3d9 1524
d6c82d7b 1525if (addr->message)
49c2d5ea 1526 {
55414b25 1527 const uschar * s = string_printing(addr->message);
f42deca9
JH
1528
1529 /* deconst cast ok as string_printing known to have alloc'n'copied */
1530 addr->message = expand_hide_passwords(US s);
49c2d5ea 1531 }
059ec3d9
PH
1532
1533/* If we used a transport that has one of the "return_output" options set, and
1534if it did in fact generate some output, then for return_output we treat the
1535message as failed if it was not already set that way, so that the output gets
1536returned to the sender, provided there is a sender to send it to. For
1537return_fail_output, do this only if the delivery failed. Otherwise we just
1538unlink the file, and remove the name so that if the delivery failed, we don't
1539try to send back an empty or unwanted file. The log_output options operate only
1540on a non-empty file.
1541
1542In any case, we close the message file, because we cannot afford to leave a
1543file-descriptor for one address while processing (maybe very many) others. */
1544
d6c82d7b 1545if (addr->return_file >= 0 && addr->return_filename)
059ec3d9
PH
1546 {
1547 BOOL return_output = FALSE;
1548 struct stat statbuf;
54fc8428 1549 (void)EXIMfsync(addr->return_file);
059ec3d9
PH
1550
1551 /* If there is no output, do nothing. */
1552
1553 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1554 {
1555 transport_instance *tb = addr->transport;
1556
1557 /* Handle logging options */
1558
d6c82d7b
JH
1559 if ( tb->log_output
1560 || result == FAIL && tb->log_fail_output
1561 || result == DEFER && tb->log_defer_output
1562 )
059ec3d9
PH
1563 {
1564 uschar *s;
1565 FILE *f = Ufopen(addr->return_filename, "rb");
d6c82d7b 1566 if (!f)
059ec3d9
PH
1567 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1568 "from %s transport: %s", addr->return_filename, tb->name,
1569 strerror(errno));
1570 else
d6c82d7b 1571 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
059ec3d9
PH
1572 {
1573 uschar *p = big_buffer + Ustrlen(big_buffer);
55414b25 1574 const uschar * sp;
059ec3d9
PH
1575 while (p > big_buffer && isspace(p[-1])) p--;
1576 *p = 0;
55414b25 1577 sp = string_printing(big_buffer);
059ec3d9 1578 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
55414b25 1579 addr->address, tb->name, sp);
059ec3d9 1580 }
0e8aed8a 1581 (void)fclose(f);
059ec3d9
PH
1582 }
1583
1584 /* Handle returning options, but only if there is an address to return
1585 the text to. */
1586
d6c82d7b 1587 if (sender_address[0] != 0 || addr->prop.errors_address)
059ec3d9
PH
1588 if (tb->return_output)
1589 {
1590 addr->transport_return = result = FAIL;
d6c82d7b 1591 if (addr->basic_errno == 0 && !addr->message)
059ec3d9
PH
1592 addr->message = US"return message generated";
1593 return_output = TRUE;
1594 }
1595 else
1596 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
059ec3d9
PH
1597 }
1598
1599 /* Get rid of the file unless it might be returned, but close it in
1600 all cases. */
1601
1602 if (!return_output)
1603 {
1604 Uunlink(addr->return_filename);
1605 addr->return_filename = NULL;
1606 addr->return_file = -1;
1607 }
1608
f1e894f3 1609 (void)close(addr->return_file);
059ec3d9
PH
1610 }
1611
9d1c15ef 1612/* The success case happens only after delivery by a transport. */
059ec3d9
PH
1613
1614if (result == OK)
1615 {
1616 addr->next = addr_succeed;
1617 addr_succeed = addr;
1618
1619 /* Call address_done() to ensure that we don't deliver to this address again,
1620 and write appropriate things to the message log. If it is a child address, we
1621 call child_done() to scan the ancestors and mark them complete if this is the
1622 last child to complete. */
1623
1624 address_done(addr, now);
1625 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1626
d6c82d7b 1627 if (!addr->parent)
059ec3d9
PH
1628 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1629 driver_name, driver_kind);
059ec3d9
PH
1630 else
1631 {
1632 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1633 addr->parent->address, driver_name, driver_kind);
1634 child_done(addr, now);
1635 }
1636
774ef2d7 1637 /* Certificates for logging (via events) */
a7538db1 1638#ifdef SUPPORT_TLS
9d1c15ef
JH
1639 tls_out.ourcert = addr->ourcert;
1640 addr->ourcert = NULL;
1641 tls_out.peercert = addr->peercert;
1642 addr->peercert = NULL;
018058b2 1643
018058b2
JH
1644 tls_out.cipher = addr->cipher;
1645 tls_out.peerdn = addr->peerdn;
1646 tls_out.ocsp = addr->ocsp;
c0635b6d 1647# ifdef SUPPORT_DANE
83b27293
JH
1648 tls_out.dane_verified = testflag(addr, af_dane_verified);
1649# endif
a7538db1 1650#endif
9d1c15ef 1651
817d9f57 1652 delivery_log(LOG_MAIN, addr, logchar, NULL);
9d1c15ef 1653
a7538db1 1654#ifdef SUPPORT_TLS
790fbb71
JH
1655 tls_free_cert(&tls_out.ourcert);
1656 tls_free_cert(&tls_out.peercert);
018058b2
JH
1657 tls_out.cipher = NULL;
1658 tls_out.peerdn = NULL;
1659 tls_out.ocsp = OCSP_NOT_REQ;
c0635b6d 1660# ifdef SUPPORT_DANE
83b27293
JH
1661 tls_out.dane_verified = FALSE;
1662# endif
a7538db1 1663#endif
059ec3d9
PH
1664 }
1665
1666
1667/* Soft failure, or local delivery process failed; freezing may be
1668requested. */
1669
1670else if (result == DEFER || result == PANIC)
1671 {
1672 if (result == PANIC) logflags |= LOG_PANIC;
1673
1674 /* This puts them on the chain in reverse order. Do not change this, because
1675 the code for handling retries assumes that the one with the retry
1676 information is last. */
1677
1678 addr->next = addr_defer;
1679 addr_defer = addr;
1680
1681 /* The only currently implemented special action is to freeze the
1682 message. Logging of this is done later, just before the -H file is
1683 updated. */
1684
1685 if (addr->special_action == SPECIAL_FREEZE)
1686 {
8768d548 1687 f.deliver_freeze = TRUE;
059ec3d9
PH
1688 deliver_frozen_at = time(NULL);
1689 update_spool = TRUE;
1690 }
1691
1692 /* If doing a 2-stage queue run, we skip writing to either the message
1693 log or the main log for SMTP defers. */
1694
8768d548 1695 if (!f.queue_2stage || addr->basic_errno != 0)
5dac72a7 1696 deferral_log(addr, now, logflags, driver_name, driver_kind);
059ec3d9
PH
1697 }
1698
1699
1700/* Hard failure. If there is an address to which an error message can be sent,
1701put this address on the failed list. If not, put it on the deferred list and
1702freeze the mail message for human attention. The latter action can also be
1703explicitly requested by a router or transport. */
1704
1705else
1706 {
1707 /* If this is a delivery error, or a message for which no replies are
1708 wanted, and the message's age is greater than ignore_bounce_errors_after,
1709 force the af_ignore_error flag. This will cause the address to be discarded
1710 later (with a log entry). */
1711
10c50704 1712 if (!*sender_address && message_age >= ignore_bounce_errors_after)
7eb0e5d2 1713 addr->prop.ignore_error = TRUE;
059ec3d9
PH
1714
1715 /* Freeze the message if requested, or if this is a bounce message (or other
652e1b65
PH
1716 message with null sender) and this address does not have its own errors
1717 address. However, don't freeze if errors are being ignored. The actual code
1718 to ignore occurs later, instead of sending a message. Logging of freezing
1719 occurs later, just before writing the -H file. */
059ec3d9 1720
7eb0e5d2 1721 if ( !addr->prop.ignore_error
d6c82d7b
JH
1722 && ( addr->special_action == SPECIAL_FREEZE
1723 || (sender_address[0] == 0 && !addr->prop.errors_address)
1724 ) )
059ec3d9 1725 {
d6c82d7b
JH
1726 frozen_info = addr->special_action == SPECIAL_FREEZE
1727 ? US""
8768d548 1728 : f.sender_local && !f.local_error_message
d6c82d7b
JH
1729 ? US" (message created with -f <>)"
1730 : US" (delivery error message)";
8768d548 1731 f.deliver_freeze = TRUE;
059ec3d9
PH
1732 deliver_frozen_at = time(NULL);
1733 update_spool = TRUE;
1734
1735 /* The address is put on the defer rather than the failed queue, because
1736 the message is being retained. */
1737
1738 addr->next = addr_defer;
1739 addr_defer = addr;
1740 }
1741
1742 /* Don't put the address on the nonrecipients tree yet; wait until an
1743 error message has been successfully sent. */
1744
1745 else
1746 {
1747 addr->next = addr_failed;
1748 addr_failed = addr;
1749 }
1750
5dac72a7 1751 failure_log(addr, driver_name ? NULL : driver_kind, now);
059ec3d9
PH
1752 }
1753
1754/* Ensure logging is turned on again in all cases */
1755
8768d548 1756f.disable_logging = FALSE;
059ec3d9
PH
1757}
1758
1759
1760
1761
1762/*************************************************
1763* Address-independent error *
1764*************************************************/
1765
1766/* This function is called when there's an error that is not dependent on a
1767particular address, such as an expansion string failure. It puts the error into
1768all the addresses in a batch, logs the incident on the main and panic logs, and
1769clears the expansions. It is mostly called from local_deliver(), but can be
1770called for a remote delivery via findugid().
1771
1772Arguments:
1773 logit TRUE if (MAIN+PANIC) logging required
1774 addr the first of the chain of addresses
1775 code the error code
1776 format format string for error message, or NULL if already set in addr
1777 ... arguments for the format
1778
1779Returns: nothing
1780*/
1781
1782static void
1783common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1784{
1785address_item *addr2;
1786addr->basic_errno = code;
1787
d6c82d7b 1788if (format)
059ec3d9
PH
1789 {
1790 va_list ap;
d12746bc
JH
1791 gstring * g;
1792
059ec3d9 1793 va_start(ap, format);
d12746bc 1794 g = string_vformat(NULL, TRUE, CS format, ap);
059ec3d9 1795 va_end(ap);
d12746bc 1796 addr->message = string_from_gstring(g);
059ec3d9
PH
1797 }
1798
d6c82d7b 1799for (addr2 = addr->next; addr2; addr2 = addr2->next)
059ec3d9
PH
1800 {
1801 addr2->basic_errno = code;
1802 addr2->message = addr->message;
1803 }
1804
1805if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1806deliver_set_expansions(NULL);
1807}
1808
1809
1810
1811
1812/*************************************************
1813* Check a "never users" list *
1814*************************************************/
1815
1816/* This function is called to check whether a uid is on one of the two "never
1817users" lists.
1818
1819Arguments:
1820 uid the uid to be checked
1821 nusers the list to be scanned; the first item in the list is the count
1822
1823Returns: TRUE if the uid is on the list
1824*/
1825
1826static BOOL
1827check_never_users(uid_t uid, uid_t *nusers)
1828{
1829int i;
d6c82d7b 1830if (!nusers) return FALSE;
059ec3d9
PH
1831for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1832return FALSE;
1833}
1834
1835
1836
1837/*************************************************
1838* Find uid and gid for a transport *
1839*************************************************/
1840
1841/* This function is called for both local and remote deliveries, to find the
1842uid/gid under which to run the delivery. The values are taken preferentially
1843from the transport (either explicit or deliver_as_creator), then from the
1844address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1845the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1846panic error is logged, and the function fails (which normally leads to delivery
1847deferral).
1848
1849Arguments:
1850 addr the address (possibly a chain)
1851 tp the transport
1852 uidp pointer to uid field
1853 gidp pointer to gid field
1854 igfp pointer to the use_initgroups field
1855
1856Returns: FALSE if failed - error has been set in address(es)
1857*/
1858
1859static BOOL
1860findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1861 BOOL *igfp)
1862{
d6c82d7b 1863uschar *nuname;
059ec3d9
PH
1864BOOL gid_set = FALSE;
1865
1866/* Default initgroups flag comes from the transport */
1867
1868*igfp = tp->initgroups;
1869
1870/* First see if there's a gid on the transport, either fixed or expandable.
1871The expanding function always logs failure itself. */
1872
1873if (tp->gid_set)
1874 {
1875 *gidp = tp->gid;
1876 gid_set = TRUE;
1877 }
d6c82d7b 1878else if (tp->expand_gid)
059ec3d9 1879 {
d6c82d7b
JH
1880 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1881 &(addr->message)))
059ec3d9
PH
1882 {
1883 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1884 return FALSE;
1885 }
d6c82d7b 1886 gid_set = TRUE;
059ec3d9
PH
1887 }
1888
911f6fde
PH
1889/* If the transport did not set a group, see if the router did. */
1890
1891if (!gid_set && testflag(addr, af_gid_set))
1892 {
1893 *gidp = addr->gid;
1894 gid_set = TRUE;
1895 }
1896
059ec3d9
PH
1897/* Pick up a uid from the transport if one is set. */
1898
1899if (tp->uid_set) *uidp = tp->uid;
1900
1901/* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1902it does not provide a passwd value from which a gid can be taken. */
1903
d6c82d7b 1904else if (tp->expand_uid)
059ec3d9
PH
1905 {
1906 struct passwd *pw;
1907 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1908 uidp, &(addr->message)))
1909 {
1910 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1911 return FALSE;
1912 }
d6c82d7b 1913 if (!gid_set && pw)
059ec3d9
PH
1914 {
1915 *gidp = pw->pw_gid;
1916 gid_set = TRUE;
1917 }
1918 }
1919
1920/* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1921
1922else if (tp->deliver_as_creator)
1923 {
1924 *uidp = originator_uid;
1925 if (!gid_set)
1926 {
1927 *gidp = originator_gid;
1928 gid_set = TRUE;
1929 }
1930 }
1931
911f6fde
PH
1932/* Otherwise see if the address specifies the uid and if so, take it and its
1933initgroups flag. */
059ec3d9
PH
1934
1935else if (testflag(addr, af_uid_set))
1936 {
1937 *uidp = addr->uid;
1938 *igfp = testflag(addr, af_initgroups);
059ec3d9
PH
1939 }
1940
1941/* Nothing has specified the uid - default to the Exim user, and group if the
1942gid is not set. */
1943
1944else
1945 {
1946 *uidp = exim_uid;
1947 if (!gid_set)
1948 {
1949 *gidp = exim_gid;
1950 gid_set = TRUE;
1951 }
1952 }
1953
911f6fde
PH
1954/* If no gid is set, it is a disaster. We default to the Exim gid only if
1955defaulting to the Exim uid. In other words, if the configuration has specified
1956a uid, it must also provide a gid. */
059ec3d9
PH
1957
1958if (!gid_set)
1959 {
1960 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1961 "%s transport", tp->name);
1962 return FALSE;
1963 }
1964
1965/* Check that the uid is not on the lists of banned uids that may not be used
1966for delivery processes. */
1967
d6c82d7b
JH
1968nuname = check_never_users(*uidp, never_users)
1969 ? US"never_users"
1970 : check_never_users(*uidp, fixed_never_users)
1971 ? US"fixed_never_users"
1972 : NULL;
1973if (nuname)
059ec3d9
PH
1974 {
1975 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1976 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1977 return FALSE;
1978 }
1979
1980/* All is well */
1981
1982return TRUE;
1983}
1984
1985
1986
1987
1988/*************************************************
1989* Check the size of a message for a transport *
1990*************************************************/
1991
1992/* Checks that the message isn't too big for the selected transport.
1993This is called only when it is known that the limit is set.
1994
1995Arguments:
1996 tp the transport
1997 addr the (first) address being delivered
1998
1999Returns: OK
2000 DEFER expansion failed or did not yield an integer
2001 FAIL message too big
2002*/
2003
2004int
2005check_message_size(transport_instance *tp, address_item *addr)
2006{
2007int rc = OK;
2008int size_limit;
2009
2010deliver_set_expansions(addr);
d45b1de8 2011size_limit = expand_string_integer(tp->message_size_limit, TRUE);
059ec3d9
PH
2012deliver_set_expansions(NULL);
2013
d6c82d7b 2014if (expand_string_message)
059ec3d9
PH
2015 {
2016 rc = DEFER;
d6c82d7b
JH
2017 addr->message = size_limit == -1
2018 ? string_sprintf("failed to expand message_size_limit "
2019 "in %s transport: %s", tp->name, expand_string_message)
2020 : string_sprintf("invalid message_size_limit "
059ec3d9
PH
2021 "in %s transport: %s", tp->name, expand_string_message);
2022 }
2023else if (size_limit > 0 && message_size > size_limit)
2024 {
2025 rc = FAIL;
2026 addr->message =
2027 string_sprintf("message is too big (transport limit = %d)",
2028 size_limit);
2029 }
2030
2031return rc;
2032}
2033
2034
2035
2036/*************************************************
2037* Transport-time check for a previous delivery *
2038*************************************************/
2039
2040/* Check that this base address hasn't previously been delivered to its routed
c2c19e9d
PH
2041transport. If it has been delivered, mark it done. The check is necessary at
2042delivery time in order to handle homonymic addresses correctly in cases where
2043the pattern of redirection changes between delivery attempts (so the unique
2044fields change). Non-homonymic previous delivery is detected earlier, at routing
2045time (which saves unnecessary routing).
2046
2047Arguments:
2048 addr the address item
2049 testing TRUE if testing wanted only, without side effects
059ec3d9 2050
059ec3d9
PH
2051Returns: TRUE if previously delivered by the transport
2052*/
2053
2054static BOOL
c2c19e9d 2055previously_transported(address_item *addr, BOOL testing)
059ec3d9
PH
2056{
2057(void)string_format(big_buffer, big_buffer_size, "%s/%s",
2058 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
2059
2060if (tree_search(tree_nonrecipients, big_buffer) != 0)
2061 {
2062 DEBUG(D_deliver|D_route|D_transport)
2063 debug_printf("%s was previously delivered (%s transport): discarded\n",
2064 addr->address, addr->transport->name);
c2c19e9d 2065 if (!testing) child_done(addr, tod_stamp(tod_log));
059ec3d9
PH
2066 return TRUE;
2067 }
2068
2069return FALSE;
2070}
2071
2072
2073
064a94c9
PH
2074/******************************************************
2075* Check for a given header in a header string *
2076******************************************************/
2077
2078/* This function is used when generating quota warnings. The configuration may
2079specify any header lines it likes in quota_warn_message. If certain of them are
2080missing, defaults are inserted, so we need to be able to test for the presence
2081of a given header.
2082
2083Arguments:
2084 hdr the required header name
2085 hstring the header string
2086
2087Returns: TRUE the header is in the string
2088 FALSE the header is not in the string
2089*/
2090
2091static BOOL
2092contains_header(uschar *hdr, uschar *hstring)
2093{
2094int len = Ustrlen(hdr);
2095uschar *p = hstring;
2096while (*p != 0)
2097 {
2098 if (strncmpic(p, hdr, len) == 0)
2099 {
2100 p += len;
2101 while (*p == ' ' || *p == '\t') p++;
2102 if (*p == ':') return TRUE;
2103 }
2104 while (*p != 0 && *p != '\n') p++;
2105 if (*p == '\n') p++;
2106 }
2107return FALSE;
2108}
2109
2110
2111
059ec3d9
PH
2112
2113/*************************************************
2114* Perform a local delivery *
2115*************************************************/
2116
2117/* Each local delivery is performed in a separate process which sets its
2118uid and gid as specified. This is a safer way than simply changing and
2119restoring using seteuid(); there is a body of opinion that seteuid() cannot be
2120used safely. From release 4, Exim no longer makes any use of it. Besides, not
2121all systems have seteuid().
2122
2123If the uid/gid are specified in the transport_instance, they are used; the
2124transport initialization must ensure that either both or neither are set.
2125Otherwise, the values associated with the address are used. If neither are set,
2126it is a configuration error.
2127
2128The transport or the address may specify a home directory (transport over-
2129rides), and if they do, this is set as $home. If neither have set a working
2130directory, this value is used for that as well. Otherwise $home is left unset
2131and the cwd is set to "/" - a directory that should be accessible to all users.
2132
2133Using a separate process makes it more complicated to get error information
2134back. We use a pipe to pass the return code and also an error code and error
2135text string back to the parent process.
2136
2137Arguments:
2138 addr points to an address block for this delivery; for "normal" local
2139 deliveries this is the only address to be delivered, but for
2140 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
2141 a number of addresses can be handled simultaneously, and in this
2142 case addr will point to a chain of addresses with the same
2143 characteristics.
2144
2145 shadowing TRUE if running a shadow transport; this causes output from pipes
2146 to be ignored.
2147
2148Returns: nothing
2149*/
2150
2151static void
2152deliver_local(address_item *addr, BOOL shadowing)
2153{
2154BOOL use_initgroups;
2155uid_t uid;
2156gid_t gid;
2157int status, len, rc;
2158int pfd[2];
2159pid_t pid;
2160uschar *working_directory;
2161address_item *addr2;
2162transport_instance *tp = addr->transport;
2163
2164/* Set up the return path from the errors or sender address. If the transport
2165has its own return path setting, expand it and replace the existing value. */
2166
d6c82d7b 2167if(addr->prop.errors_address)
d43cbe25 2168 return_path = addr->prop.errors_address;
384152a6 2169#ifdef EXPERIMENTAL_SRS
d6c82d7b 2170else if (addr->prop.srs_sender)
d43cbe25 2171 return_path = addr->prop.srs_sender;
384152a6
TK
2172#endif
2173else
2174 return_path = sender_address;
059ec3d9 2175
d6c82d7b 2176if (tp->return_path)
059ec3d9
PH
2177 {
2178 uschar *new_return_path = expand_string(tp->return_path);
d6c82d7b 2179 if (!new_return_path)
059ec3d9 2180 {
8768d548 2181 if (!f.expand_string_forcedfail)
059ec3d9
PH
2182 {
2183 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
2184 US"Failed to expand return path \"%s\" in %s transport: %s",
2185 tp->return_path, tp->name, expand_string_message);
2186 return;
2187 }
2188 }
2189 else return_path = new_return_path;
2190 }
2191
2192/* For local deliveries, one at a time, the value used for logging can just be
2193set directly, once and for all. */
2194
2195used_return_path = return_path;
2196
2197/* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
2198gets put into the address(es), and the expansions are unset, so we can just
2199return. */
2200
2201if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
2202
5418e93b
PH
2203/* See if either the transport or the address specifies a home directory. A
2204home directory set in the address may already be expanded; a flag is set to
2205indicate that. In other cases we must expand it. */
059ec3d9 2206
d6c82d7b
JH
2207if ( (deliver_home = tp->home_dir) /* Set in transport, or */
2208 || ( (deliver_home = addr->home_dir) /* Set in address and */
2209 && !testflag(addr, af_home_expanded) /* not expanded */
2210 ) )
059ec3d9
PH
2211 {
2212 uschar *rawhome = deliver_home;
2213 deliver_home = NULL; /* in case it contains $home */
d6c82d7b 2214 if (!(deliver_home = expand_string(rawhome)))
059ec3d9
PH
2215 {
2216 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
2217 "to expand for %s transport: %s", rawhome, tp->name,
2218 expand_string_message);
2219 return;
2220 }
2221 if (*deliver_home != '/')
2222 {
2223 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
2224 "is not absolute for %s transport", deliver_home, tp->name);
2225 return;
2226 }
2227 }
2228
5418e93b
PH
2229/* See if either the transport or the address specifies a current directory,
2230and if so, expand it. If nothing is set, use the home directory, unless it is
2231also unset in which case use "/", which is assumed to be a directory to which
2232all users have access. It is necessary to be in a visible directory for some
2233operating systems when running pipes, as some commands (e.g. "rm" under Solaris
22342.5) require this. */
2235
d6c82d7b
JH
2236working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
2237if (working_directory)
059ec3d9
PH
2238 {
2239 uschar *raw = working_directory;
d6c82d7b 2240 if (!(working_directory = expand_string(raw)))
059ec3d9
PH
2241 {
2242 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
2243 "failed to expand for %s transport: %s", raw, tp->name,
2244 expand_string_message);
2245 return;
2246 }
2247 if (*working_directory != '/')
2248 {
2249 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
2250 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
2251 return;
2252 }
2253 }
d6c82d7b 2254else working_directory = deliver_home ? deliver_home : US"/";
059ec3d9
PH
2255
2256/* If one of the return_output flags is set on the transport, create and open a
2257file in the message log directory for the transport to write its output onto.
2258This is mainly used by pipe transports. The file needs to be unique to the
2259address. This feature is not available for shadow transports. */
2260
d6c82d7b
JH
2261if ( !shadowing
2262 && ( tp->return_output || tp->return_fail_output
ec0eb1a3 2263 || tp->log_output || tp->log_fail_output || tp->log_defer_output
d6c82d7b 2264 ) )
059ec3d9 2265 {
41313d92
JH
2266 uschar * error;
2267
059ec3d9 2268 addr->return_filename =
41313d92
JH
2269 spool_fname(US"msglog", message_subdir, message_id,
2270 string_sprintf("-%d-%d", getpid(), return_count++));
4c0a7a9c 2271
41313d92 2272 if ((addr->return_file = open_msglog_file(addr->return_filename, 0400, &error)) < 0)
059ec3d9
PH
2273 {
2274 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
2275 "to return message: %s", error, tp->name, strerror(errno));
2276 return;
2277 }
2278 }
2279
2280/* Create the pipe for inter-process communication. */
2281
2282if (pipe(pfd) != 0)
2283 {
2284 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
2285 strerror(errno));
2286 return;
2287 }
2288
2289/* Now fork the process to do the real work in the subprocess, but first
2290ensure that all cached resources are freed so that the subprocess starts with
2291a clean slate and doesn't interfere with the parent process. */
2292
2293search_tidyup();
2294
2295if ((pid = fork()) == 0)
2296 {
2297 BOOL replicate = TRUE;
2298
2299 /* Prevent core dumps, as we don't want them in users' home directories.
2300 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
2301 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
2302 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
a29e5231
PP
2303 complain if the error is "not supported".
2304
2305 There are two scenarios where changing the max limit has an effect. In one,
2306 the user is using a .forward and invoking a command of their choice via pipe;
2307 for these, we do need the max limit to be 0 unless the admin chooses to
2308 permit an increased limit. In the other, the command is invoked directly by
2309 the transport and is under administrator control, thus being able to raise
2310 the limit aids in debugging. So there's no general always-right answer.
2311
2312 Thus we inhibit core-dumps completely but let individual transports, while
2313 still root, re-raise the limits back up to aid debugging. We make the
2314 default be no core-dumps -- few enough people can use core dumps in
2315 diagnosis that it's reasonable to make them something that has to be explicitly requested.
2316 */
059ec3d9 2317
a7538db1 2318#ifdef RLIMIT_CORE
059ec3d9
PH
2319 struct rlimit rl;
2320 rl.rlim_cur = 0;
2321 rl.rlim_max = 0;
2322 if (setrlimit(RLIMIT_CORE, &rl) < 0)
2323 {
a7538db1 2324# ifdef SETRLIMIT_NOT_SUPPORTED
059ec3d9 2325 if (errno != ENOSYS && errno != ENOTSUP)
a7538db1 2326# endif
059ec3d9
PH
2327 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
2328 strerror(errno));
2329 }
a7538db1 2330#endif
059ec3d9
PH
2331
2332 /* Reset the random number generator, so different processes don't all
2333 have the same sequence. */
2334
2335 random_seed = 0;
2336
2337 /* If the transport has a setup entry, call this first, while still
2338 privileged. (Appendfile uses this to expand quota, for example, while
2339 able to read private files.) */
2340
d6c82d7b 2341 if (addr->transport->setup)
929ba01c 2342 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
059ec3d9
PH
2343 &(addr->message)))
2344 {
2345 case DEFER:
d6c82d7b
JH
2346 addr->transport_return = DEFER;
2347 goto PASS_BACK;
059ec3d9
PH
2348
2349 case FAIL:
d6c82d7b
JH
2350 addr->transport_return = PANIC;
2351 goto PASS_BACK;
059ec3d9 2352 }
059ec3d9
PH
2353
2354 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2355 when the process becomes unprivileged, it won't be able to write to the
2356 process log. SIGHUP is ignored throughout exim, except when it is being
2357 run as a daemon. */
2358
2359 signal(SIGINT, SIG_IGN);
2360 signal(SIGTERM, SIG_IGN);
2361 signal(SIGUSR1, SIG_IGN);
2362
2363 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2364 half - for transports that exec things (e.g. pipe). Then set the required
2365 gid/uid. */
2366
f1e894f3 2367 (void)close(pfd[pipe_read]);
ff790e47 2368 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
059ec3d9
PH
2369 FD_CLOEXEC);
2370 exim_setugid(uid, gid, use_initgroups,
2371 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2372 addr->address, addr->transport->name));
2373
2374 DEBUG(D_deliver)
2375 {
2376 address_item *batched;
2377 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
d6c82d7b 2378 for (batched = addr->next; batched; batched = batched->next)
059ec3d9
PH
2379 debug_printf("additional batched address: %s\n", batched->address);
2380 }
2381
2382 /* Set an appropriate working directory. */
2383
2384 if (Uchdir(working_directory) < 0)
2385 {
2386 addr->transport_return = DEFER;
2387 addr->basic_errno = errno;
2388 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2389 }
2390
2391 /* If successful, call the transport */
2392
2393 else
2394 {
2395 BOOL ok = TRUE;
2396 set_process_info("delivering %s to %s using %s", message_id,
2397 addr->local_part, addr->transport->name);
2398
2a47f028
JH
2399 /* Setting this global in the subprocess means we need never clear it */
2400 transport_name = addr->transport->name;
2401
059ec3d9
PH
2402 /* If a transport filter has been specified, set up its argument list.
2403 Any errors will get put into the address, and FALSE yielded. */
2404
d6c82d7b 2405 if (addr->transport->filter_command)
059ec3d9
PH
2406 {
2407 ok = transport_set_up_command(&transport_filter_argv,
2408 addr->transport->filter_command,
2409 TRUE, PANIC, addr, US"transport filter", NULL);
2410 transport_filter_timeout = addr->transport->filter_timeout;
2411 }
2412 else transport_filter_argv = NULL;
2413
2414 if (ok)
2415 {
2416 debug_print_string(addr->transport->debug_string);
2417 replicate = !(addr->transport->info->code)(addr->transport, addr);
2418 }
2419 }
2420
2421 /* Pass the results back down the pipe. If necessary, first replicate the
2422 status in the top address to the others in the batch. The label is the
2423 subject of a goto when a call to the transport's setup function fails. We
2424 pass the pointer to the transport back in case it got changed as a result of
2425 file_format in appendfile. */
2426
2427 PASS_BACK:
2428
2429 if (replicate) replicate_status(addr);
d6c82d7b 2430 for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9
PH
2431 {
2432 int i;
2433 int local_part_length = Ustrlen(addr2->local_part);
2434 uschar *s;
1ac6b2e7 2435 int ret;
059ec3d9 2436
d6c82d7b
JH
2437 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2438 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2439 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2440 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2441 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
32dfdf8b 2442 || (ret = write(pfd[pipe_write], &addr2->delivery_usec, sizeof(int))) != sizeof(int)
d6c82d7b
JH
2443 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2444 || (ret = write(pfd[pipe_write], &addr2->transport,
1ac6b2e7 2445 sizeof(transport_instance *))) != sizeof(transport_instance *)
059ec3d9
PH
2446
2447 /* For a file delivery, pass back the local part, in case the original
2448 was only part of the final delivery path. This gives more complete
2449 logging. */
2450
1ac6b2e7 2451 || (testflag(addr2, af_file)
d6c82d7b 2452 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
1ac6b2e7
JH
2453 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2454 )
2455 )
2456 )
a23ff3b4 2457 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
1ac6b2e7 2458 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2459
2460 /* Now any messages */
2461
2462 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2463 {
d6c82d7b
JH
2464 int message_length = s ? Ustrlen(s) + 1 : 0;
2465 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2466 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
1ac6b2e7 2467 )
a23ff3b4 2468 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
1ac6b2e7 2469 ret == -1 ? strerror(errno) : "short write");
059ec3d9
PH
2470 }
2471 }
2472
2473 /* OK, this process is now done. Free any cached resources that it opened,
2474 and close the pipe we were writing down before exiting. */
2475
f1e894f3 2476 (void)close(pfd[pipe_write]);
059ec3d9
PH
2477 search_tidyup();
2478 exit(EXIT_SUCCESS);
2479 }
2480
2481/* Back in the main process: panic if the fork did not succeed. This seems
2482better than returning an error - if forking is failing it is probably best
2483not to try other deliveries for this message. */
2484
2485if (pid < 0)
2486 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2487 addr->address);
2488
2489/* Read the pipe to get the delivery status codes and error messages. Our copy
2490of the writing end must be closed first, as otherwise read() won't return zero
2491on an empty pipe. We check that a status exists for each address before
2492overwriting the address structure. If data is missing, the default DEFER status
2493will remain. Afterwards, close the reading end. */
2494
f1e894f3 2495(void)close(pfd[pipe_write]);
059ec3d9 2496
d6c82d7b 2497for (addr2 = addr; addr2; addr2 = addr2->next)
059ec3d9 2498 {
d88f0784 2499 if ((len = read(pfd[pipe_read], &status, sizeof(int))) > 0)
059ec3d9
PH
2500 {
2501 int i;
2502 uschar **sptr;
2503
2504 addr2->transport_return = status;
d6c82d7b 2505 len = read(pfd[pipe_read], &transport_count,
059ec3d9 2506 sizeof(transport_count));
d6c82d7b
JH
2507 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2508 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2509 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
32dfdf8b 2510 len = read(pfd[pipe_read], &addr2->delivery_usec, sizeof(int));
d6c82d7b
JH
2511 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2512 len = read(pfd[pipe_read], &addr2->transport,
059ec3d9
PH
2513 sizeof(transport_instance *));
2514
2515 if (testflag(addr2, af_file))
2516 {
d88f0784
JH
2517 int llen;
2518 if ( read(pfd[pipe_read], &llen, sizeof(int)) != sizeof(int)
2519 || llen > 64*4 /* limit from rfc 5821, times I18N factor */
2520 )
2521 {
2522 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part length read"
2523 " from delivery subprocess");
2524 break;
2525 }
2526 /* sanity-checked llen so disable the Coverity error */
2527 /* coverity[tainted_data] */
2528 if (read(pfd[pipe_read], big_buffer, llen) != llen)
2529 {
2530 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part read"
2531 " from delivery subprocess");
2532 break;
2533 }
2534 big_buffer[llen] = 0;
059ec3d9
PH
2535 addr2->local_part = string_copy(big_buffer);
2536 }
2537
d6c82d7b 2538 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
059ec3d9
PH
2539 {
2540 int message_length;
d6c82d7b 2541 len = read(pfd[pipe_read], &message_length, sizeof(int));
059ec3d9
PH
2542 if (message_length > 0)
2543 {
d6c82d7b 2544 len = read(pfd[pipe_read], big_buffer, message_length);
dc8091e7 2545 big_buffer[big_buffer_size-1] = '\0'; /* guard byte */
059ec3d9
PH
2546 if (len > 0) *sptr = string_copy(big_buffer);
2547 }
2548 }
2549 }
2550
2551 else
2552 {
2553 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2554 "from delivery subprocess", addr2->unique);
2555 break;
2556 }
2557 }
2558
f1e894f3 2559(void)close(pfd[pipe_read]);
059ec3d9
PH
2560
2561/* Unless shadowing, write all successful addresses immediately to the journal
2562file, to ensure they are recorded asap. For homonymic addresses, use the base
2563address plus the transport name. Failure to write the journal is panic-worthy,
2564but don't stop, as it may prove possible subsequently to update the spool file
2565in order to record the delivery. */
2566
2567if (!shadowing)
2568 {
d6c82d7b
JH
2569 for (addr2 = addr; addr2; addr2 = addr2->next)
2570 if (addr2->transport_return == OK)
2571 {
2572 if (testflag(addr2, af_homonym))
2573 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2574 else
2575 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
059ec3d9 2576
d6c82d7b
JH
2577 /* In the test harness, wait just a bit to let the subprocess finish off
2578 any debug output etc first. */
059ec3d9 2579
8768d548 2580 if (f.running_in_test_harness) millisleep(300);
059ec3d9 2581
d6c82d7b
JH
2582 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2583 len = Ustrlen(big_buffer);
2584 if (write(journal_fd, big_buffer, len) != len)
2585 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2586 big_buffer, strerror(errno));
2587 }
059ec3d9
PH
2588
2589 /* Ensure the journal file is pushed out to disk. */
2590
54fc8428 2591 if (EXIMfsync(journal_fd) < 0)
059ec3d9
PH
2592 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2593 strerror(errno));
2594 }
2595
2596/* Wait for the process to finish. If it terminates with a non-zero code,
2597freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2598status values of all the addresses as they are. Take care to handle the case
2599when the subprocess doesn't seem to exist. This has been seen on one system
2600when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2601happens, wait() doesn't recognize the termination of child processes. Exim now
2602resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2603
2604while ((rc = wait(&status)) != pid)
059ec3d9
PH
2605 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2606 {
2607 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2608 addr->transport->driver_name);
2609 status = 0;
2610 break;
2611 }
059ec3d9
PH
2612
2613if ((status & 0xffff) != 0)
2614 {
2615 int msb = (status >> 8) & 255;
2616 int lsb = status & 255;
2617 int code = (msb == 0)? (lsb & 0x7f) : msb;
2618 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2619 addr->special_action = SPECIAL_FREEZE;
2620 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2621 "status 0x%04x: %s %d",
2622 addr->transport->driver_name,
2623 status,
d6c82d7b 2624 msb == 0 ? "terminated by signal" : "exit code",
059ec3d9
PH
2625 code);
2626 }
2627
2628/* If SPECIAL_WARN is set in the top address, send a warning message. */
2629
d6c82d7b 2630if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
059ec3d9
PH
2631 {
2632 int fd;
2633 uschar *warn_message;
d6c82d7b 2634 pid_t pid;
059ec3d9
PH
2635
2636 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2637
d6c82d7b 2638 if (!(warn_message = expand_string(addr->transport->warn_message)))
059ec3d9
PH
2639 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2640 "message for %s transport): %s", addr->transport->warn_message,
2641 addr->transport->name, expand_string_message);
d6c82d7b
JH
2642
2643 else if ((pid = child_open_exim(&fd)) > 0)
059ec3d9 2644 {
d6c82d7b
JH
2645 FILE *f = fdopen(fd, "wb");
2646 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2647 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2648 fprintf(f, "Auto-Submitted: auto-replied\n");
2649 if (!contains_header(US"From", warn_message))
2650 moan_write_from(f);
2651 fprintf(f, "%s", CS warn_message);
059ec3d9 2652
d6c82d7b 2653 /* Close and wait for child process to complete, without a timeout. */
059ec3d9 2654
d6c82d7b
JH
2655 (void)fclose(f);
2656 (void)child_close(pid, 0);
059ec3d9
PH
2657 }
2658
2659 addr->special_action = SPECIAL_NONE;
2660 }
2661}
2662
2663
2664
fa41615d 2665
3070ceee
JH
2666/* Check transport for the given concurrency limit. Return TRUE if over
2667the limit (or an expansion failure), else FALSE and if there was a limit,
2668the key for the hints database used for the concurrency count. */
fa41615d 2669
3070ceee
JH
2670static BOOL
2671tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key)
fa41615d 2672{
3070ceee
JH
2673unsigned max_parallel;
2674
2675if (!tp->max_parallel) return FALSE;
2676
2677max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE);
2678if (expand_string_message)
2679 {
2680 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2681 "in %s transport (%s): %s", tp->name, addr->address,
2682 expand_string_message);
2683 return TRUE;
2684 }
2685
2686if (max_parallel > 0)
2687 {
2688 uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name);
2689 if (!enq_start(serialize_key, max_parallel))
2690 {
2691 address_item * next;
2692 DEBUG(D_transport)
2693 debug_printf("skipping tpt %s because concurrency limit %u reached\n",
2694 tp->name, max_parallel);
2695 do
2696 {
2697 next = addr->next;
2698 addr->message = US"concurrency limit reached for transport";
2699 addr->basic_errno = ERRNO_TRETRY;
ba936fb8 2700 post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_TRANSPORT, 0);
3070ceee
JH
2701 } while ((addr = next));
2702 return TRUE;
2703 }
2704 *key = serialize_key;
2705 }
2706return FALSE;
fa41615d
JH
2707}
2708
2709
2710
059ec3d9
PH
2711/*************************************************
2712* Do local deliveries *
2713*************************************************/
2714
2715/* This function processes the list of addresses in addr_local. True local
2716deliveries are always done one address at a time. However, local deliveries can
2717be batched up in some cases. Typically this is when writing batched SMTP output
2718files for use by some external transport mechanism, or when running local
2719deliveries over LMTP.
2720
2721Arguments: None
2722Returns: Nothing
2723*/
2724
2725static void
2726do_local_deliveries(void)
2727{
2728open_db dbblock;
2729open_db *dbm_file = NULL;
2730time_t now = time(NULL);
2731
2732/* Loop until we have exhausted the supply of local deliveries */
2733
d6c82d7b 2734while (addr_local)
059ec3d9 2735 {
32dfdf8b
JH
2736 struct timeval delivery_start;
2737 struct timeval deliver_time;
059ec3d9
PH
2738 address_item *addr2, *addr3, *nextaddr;
2739 int logflags = LOG_MAIN;
8768d548 2740 int logchar = f.dont_deliver? '*' : '=';
059ec3d9 2741 transport_instance *tp;
fa41615d 2742 uschar * serialize_key = NULL;
059ec3d9
PH
2743
2744 /* Pick the first undelivered address off the chain */
2745
2746 address_item *addr = addr_local;
2747 addr_local = addr->next;
2748 addr->next = NULL;
2749
2750 DEBUG(D_deliver|D_transport)
2751 debug_printf("--------> %s <--------\n", addr->address);
2752
2753 /* An internal disaster if there is no transport. Should not occur! */
2754
d6c82d7b 2755 if (!(tp = addr->transport))
059ec3d9
PH
2756 {
2757 logflags |= LOG_PANIC;
8768d548 2758 f.disable_logging = FALSE; /* Jic */
d6c82d7b
JH
2759 addr->message = addr->router
2760 ? string_sprintf("No transport set by %s router", addr->router->name)
2761 : string_sprintf("No transport set by system filter");
ba936fb8 2762 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
059ec3d9
PH
2763 continue;
2764 }
2765
2766 /* Check that this base address hasn't previously been delivered to this
2767 transport. The check is necessary at this point to handle homonymic addresses
2768 correctly in cases where the pattern of redirection changes between delivery
2769 attempts. Non-homonymic previous delivery is detected earlier, at routing
2770 time. */
2771
c2c19e9d 2772 if (previously_transported(addr, FALSE)) continue;
059ec3d9
PH
2773
2774 /* There are weird cases where logging is disabled */
2775
8768d548 2776 f.disable_logging = tp->disable_logging;
059ec3d9 2777
f7fd3850
PH
2778 /* Check for batched addresses and possible amalgamation. Skip all the work
2779 if either batch_max <= 1 or there aren't any other addresses for local
2780 delivery. */
059ec3d9 2781
d6c82d7b 2782 if (tp->batch_max > 1 && addr_local)
059ec3d9
PH
2783 {
2784 int batch_count = 1;
2785 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
d6c82d7b
JH
2786 BOOL uses_lp = ( testflag(addr, af_pfr)
2787 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2788 )
2789 || readconf_depends((driver_instance *)tp, US"local_part");
059ec3d9
PH
2790 uschar *batch_id = NULL;
2791 address_item **anchor = &addr_local;
2792 address_item *last = addr;
2793 address_item *next;
2794
2795 /* Expand the batch_id string for comparison with other addresses.
2796 Expansion failure suppresses batching. */
2797
d6c82d7b 2798 if (tp->batch_id)
059ec3d9
PH
2799 {
2800 deliver_set_expansions(addr);
2801 batch_id = expand_string(tp->batch_id);
2802 deliver_set_expansions(NULL);
d6c82d7b 2803 if (!batch_id)
059ec3d9
PH
2804 {
2805 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2806 "in %s transport (%s): %s", tp->name, addr->address,
2807 expand_string_message);
2808 batch_count = tp->batch_max;
2809 }
2810 }
2811
2812 /* Until we reach the batch_max limit, pick off addresses which have the
2813 same characteristics. These are:
2814
2815 same transport
7816e254 2816 not previously delivered (see comment about 50 lines above)
059ec3d9 2817 same local part if the transport's configuration contains $local_part
f7fd3850 2818 or if this is a file or pipe delivery from a redirection
059ec3d9
PH
2819 same domain if the transport's configuration contains $domain
2820 same errors address
2821 same additional headers
2822 same headers to be removed
2823 same uid/gid for running the transport
2824 same first host if a host list is set
2825 */
2826
d6c82d7b 2827 while ((next = *anchor) && batch_count < tp->batch_max)
059ec3d9
PH
2828 {
2829 BOOL ok =
d6c82d7b
JH
2830 tp == next->transport
2831 && !previously_transported(next, TRUE)
7eb0e5d2
JH
2832 && testflag(addr, af_pfr) == testflag(next, af_pfr)
2833 && testflag(addr, af_file) == testflag(next, af_file)
d6c82d7b
JH
2834 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2835 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2836 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2837 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2838 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2839 && same_ugid(tp, addr, next)
2840 && ( !addr->host_list && !next->host_list
2841 || addr->host_list
2842 && next->host_list
2843 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2844 );
059ec3d9
PH
2845
2846 /* If the transport has a batch_id setting, batch_id will be non-NULL
2847 from the expansion outside the loop. Expand for this address and compare.
2848 Expansion failure makes this address ineligible for batching. */
2849
d6c82d7b 2850 if (ok && batch_id)
059ec3d9
PH
2851 {
2852 uschar *bid;
2853 address_item *save_nextnext = next->next;
2854 next->next = NULL; /* Expansion for a single address */
2855 deliver_set_expansions(next);
2856 next->next = save_nextnext;
2857 bid = expand_string(tp->batch_id);
2858 deliver_set_expansions(NULL);
d6c82d7b 2859 if (!bid)
059ec3d9
PH
2860 {
2861 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2862 "in %s transport (%s): %s", tp->name, next->address,
2863 expand_string_message);
2864 ok = FALSE;
2865 }
2866 else ok = (Ustrcmp(batch_id, bid) == 0);
2867 }
2868
2869 /* Take address into batch if OK. */
2870
2871 if (ok)
2872 {
2873 *anchor = next->next; /* Include the address */
2874 next->next = NULL;
2875 last->next = next;
2876 last = next;
2877 batch_count++;
2878 }
fa41615d 2879 else anchor = &next->next; /* Skip the address */
059ec3d9
PH
2880 }
2881 }
2882
2883 /* We now have one or more addresses that can be delivered in a batch. Check
2884 whether the transport is prepared to accept a message of this size. If not,
2885 fail them all forthwith. If the expansion fails, or does not yield an
2886 integer, defer delivery. */
2887
d6c82d7b 2888 if (tp->message_size_limit)
059ec3d9
PH
2889 {
2890 int rc = check_message_size(tp, addr);
2891 if (rc != OK)
2892 {
2893 replicate_status(addr);
d6c82d7b 2894 while (addr)
059ec3d9
PH
2895 {
2896 addr2 = addr->next;
ba936fb8 2897 post_process_one(addr, rc, logflags, EXIM_DTYPE_TRANSPORT, 0);
059ec3d9
PH
2898 addr = addr2;
2899 }
2900 continue; /* With next batch of addresses */
2901 }
2902 }
2903
2904 /* If we are not running the queue, or if forcing, all deliveries will be
2905 attempted. Otherwise, we must respect the retry times for each address. Even
2906 when not doing this, we need to set up the retry key string, and determine
2907 whether a retry record exists, because after a successful delivery, a delete
2908 retry item must be set up. Keep the retry database open only for the duration
2909 of these checks, rather than for all local deliveries, because some local
2910 deliveries (e.g. to pipes) can take a substantial time. */
2911
d6c82d7b 2912 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
059ec3d9
PH
2913 {
2914 DEBUG(D_deliver|D_retry|D_hints_lookup)
2915 debug_printf("no retry data available\n");
2916 }
2917
2918 addr2 = addr;
2919 addr3 = NULL;
d6c82d7b 2920 while (addr2)
059ec3d9
PH
2921 {
2922 BOOL ok = TRUE; /* to deliver this address */
2923 uschar *retry_key;
2924
2925 /* Set up the retry key to include the domain or not, and change its
2926 leading character from "R" to "T". Must make a copy before doing this,
2927 because the old key may be pointed to from a "delete" retry item after
2928 a routing delay. */
2929
2930 retry_key = string_copy(
d6c82d7b 2931 tp->retry_use_local_part ? addr2->address_retry_key :
059ec3d9
PH
2932 addr2->domain_retry_key);
2933 *retry_key = 'T';
2934
2935 /* Inspect the retry data. If there is no hints file, delivery happens. */
2936
d6c82d7b 2937 if (dbm_file)
059ec3d9
PH
2938 {
2939 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2940
2941 /* If there is no retry record, delivery happens. If there is,
2942 remember it exists so it can be deleted after a successful delivery. */
2943
d6c82d7b 2944 if (retry_record)
059ec3d9
PH
2945 {
2946 setflag(addr2, af_lt_retry_exists);
2947
2948 /* A retry record exists for this address. If queue running and not
2949 forcing, inspect its contents. If the record is too old, or if its
2950 retry time has come, or if it has passed its cutoff time, delivery
2951 will go ahead. */
2952
2953 DEBUG(D_retry)
2954 {
ea49d0e1
PH
2955 debug_printf("retry record exists: age=%s ",
2956 readconf_printtime(now - retry_record->time_stamp));
2957 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2958 debug_printf(" time to retry = %s expired = %d\n",
2959 readconf_printtime(retry_record->next_try - now),
2960 retry_record->expired);
059ec3d9
PH
2961 }
2962
8768d548 2963 if (f.queue_running && !f.deliver_force)
059ec3d9 2964 {
d6c82d7b
JH
2965 ok = (now - retry_record->time_stamp > retry_data_expire)
2966 || (now >= retry_record->next_try)
2967 || retry_record->expired;
059ec3d9
PH
2968
2969 /* If we haven't reached the retry time, there is one more check
1ddeb334 2970 to do, which is for the ultimate address timeout. */
059ec3d9
PH
2971
2972 if (!ok)
ba9af0af
TF
2973 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2974 retry_record, now);
059ec3d9
PH
2975 }
2976 }
2977 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2978 }
2979
2980 /* This address is to be delivered. Leave it on the chain. */
2981
2982 if (ok)
2983 {
2984 addr3 = addr2;
2985 addr2 = addr2->next;
2986 }
2987
2988 /* This address is to be deferred. Take it out of the chain, and
2989 post-process it as complete. Must take it out of the chain first,
2990 because post processing puts it on another chain. */
2991
2992 else
2993 {
2994 address_item *this = addr2;
2995 this->message = US"Retry time not yet reached";
2996 this->basic_errno = ERRNO_LRETRY;
d6c82d7b
JH
2997 addr2 = addr3 ? (addr3->next = addr2->next)
2998 : (addr = addr2->next);
ba936fb8 2999 post_process_one(this, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
059ec3d9
PH
3000 }
3001 }
3002
d6c82d7b 3003 if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
3004
3005 /* If there are no addresses left on the chain, they all deferred. Loop
3006 for the next set of addresses. */
3007
d6c82d7b 3008 if (!addr) continue;
059ec3d9 3009
fa41615d
JH
3010 /* If the transport is limited for parallellism, enforce that here.
3011 We use a hints DB entry, incremented here and decremented after
3012 the transport (and any shadow transport) completes. */
3013
3070ceee 3014 if (tpt_parallel_check(tp, addr, &serialize_key))
fa41615d 3015 {
fa41615d
JH
3016 if (expand_string_message)
3017 {
3018 logflags |= LOG_PANIC;
3070ceee
JH
3019 do
3020 {
3021 addr = addr->next;
ba936fb8 3022 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
3070ceee 3023 } while ((addr = addr2));
fa41615d 3024 }
3070ceee 3025 continue; /* Loop for the next set of addresses. */
fa41615d
JH
3026 }
3027
3028
059ec3d9
PH
3029 /* So, finally, we do have some addresses that can be passed to the
3030 transport. Before doing so, set up variables that are relevant to a
3031 single delivery. */
3032
3033 deliver_set_expansions(addr);
32dfdf8b
JH
3034
3035 gettimeofday(&delivery_start, NULL);
059ec3d9 3036 deliver_local(addr, FALSE);
32dfdf8b 3037 timesince(&deliver_time, &delivery_start);
059ec3d9
PH
3038
3039 /* If a shadow transport (which must perforce be another local transport), is
3040 defined, and its condition is met, we must pass the message to the shadow
3041 too, but only those addresses that succeeded. We do this by making a new
3042 chain of addresses - also to keep the original chain uncontaminated. We must
3043 use a chain rather than doing it one by one, because the shadow transport may
3044 batch.
3045
3046 NOTE: if the condition fails because of a lookup defer, there is nothing we
3047 can do! */
3048
d6c82d7b
JH
3049 if ( tp->shadow
3050 && ( !tp->shadow_condition
3051 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
3052 ) )
059ec3d9
PH
3053 {
3054 transport_instance *stp;
3055 address_item *shadow_addr = NULL;
3056 address_item **last = &shadow_addr;
3057
d6c82d7b 3058 for (stp = transports; stp; stp = stp->next)
059ec3d9
PH
3059 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
3060
d6c82d7b 3061 if (!stp)
059ec3d9
PH
3062 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
3063 tp->shadow);
3064
3065 /* Pick off the addresses that have succeeded, and make clones. Put into
3066 the shadow_message field a pointer to the shadow_message field of the real
3067 address. */
3068
d6c82d7b
JH
3069 else for (addr2 = addr; addr2; addr2 = addr2->next)
3070 if (addr2->transport_return == OK)
3071 {
3072 addr3 = store_get(sizeof(address_item));
3073 *addr3 = *addr2;
3074 addr3->next = NULL;
82f90600 3075 addr3->shadow_message = US &addr2->shadow_message;
d6c82d7b
JH
3076 addr3->transport = stp;
3077 addr3->transport_return = DEFER;
3078 addr3->return_filename = NULL;
3079 addr3->return_file = -1;
3080 *last = addr3;
82f90600 3081 last = &addr3->next;
d6c82d7b 3082 }
059ec3d9
PH
3083
3084 /* If we found any addresses to shadow, run the delivery, and stick any
3085 message back into the shadow_message field in the original. */
3086
d6c82d7b 3087 if (shadow_addr)
059ec3d9
PH
3088 {
3089 int save_count = transport_count;
3090
3091 DEBUG(D_deliver|D_transport)
3092 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
3093 deliver_local(shadow_addr, TRUE);
3094
d6c82d7b 3095 for(; shadow_addr; shadow_addr = shadow_addr->next)
059ec3d9
PH
3096 {
3097 int sresult = shadow_addr->transport_return;
d6c82d7b
JH
3098 *(uschar **)shadow_addr->shadow_message =
3099 sresult == OK
3100 ? string_sprintf(" ST=%s", stp->name)
3101 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
3102 shadow_addr->basic_errno <= 0
3103 ? US""
3104 : US strerror(shadow_addr->basic_errno),
3105 shadow_addr->basic_errno <= 0 || !shadow_addr->message
3106 ? US""
3107 : US": ",
3108 shadow_addr->message
3109 ? shadow_addr->message
3110 : shadow_addr->basic_errno <= 0
3111 ? US"unknown error"
3112 : US"");
059ec3d9
PH
3113
3114 DEBUG(D_deliver|D_transport)
3115 debug_printf("%s shadow transport returned %s for %s\n",
3116 stp->name,
d6c82d7b
JH
3117 sresult == OK ? "OK" :
3118 sresult == DEFER ? "DEFER" :
3119 sresult == FAIL ? "FAIL" :
3120 sresult == PANIC ? "PANIC" : "?",
059ec3d9
PH
3121 shadow_addr->address);
3122 }
3123
3124 DEBUG(D_deliver|D_transport)
3125 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
3126
3127 transport_count = save_count; /* Restore original transport count */
3128 }
3129 }
3130
3131 /* Cancel the expansions that were set up for the delivery. */
3132
3133 deliver_set_expansions(NULL);
3134
fa41615d
JH
3135 /* If the transport was parallelism-limited, decrement the hints DB record. */
3136
3137 if (serialize_key) enq_end(serialize_key);
3138
059ec3d9
PH
3139 /* Now we can process the results of the real transport. We must take each
3140 address off the chain first, because post_process_one() puts it on another
3141 chain. */
3142
d6c82d7b 3143 for (addr2 = addr; addr2; addr2 = nextaddr)
059ec3d9
PH
3144 {
3145 int result = addr2->transport_return;
3146 nextaddr = addr2->next;
3147
3148 DEBUG(D_deliver|D_transport)
3149 debug_printf("%s transport returned %s for %s\n",
3150 tp->name,
d6c82d7b
JH
3151 result == OK ? "OK" :
3152 result == DEFER ? "DEFER" :
3153 result == FAIL ? "FAIL" :
3154 result == PANIC ? "PANIC" : "?",
059ec3d9
PH
3155 addr2->address);
3156
3157 /* If there is a retry_record, or if delivery is deferred, build a retry
3158 item for setting a new retry time or deleting the old retry record from
3159 the database. These items are handled all together after all addresses
3160 have been handled (so the database is open just for a short time for
3161 updating). */
3162
3163 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
3164 {
d6c82d7b
JH
3165 int flags = result == DEFER ? 0 : rf_delete;
3166 uschar *retry_key = string_copy(tp->retry_use_local_part
3167 ? addr2->address_retry_key : addr2->domain_retry_key);
059ec3d9
PH
3168 *retry_key = 'T';
3169 retry_add_item(addr2, retry_key, flags);
3170 }
3171
3172 /* Done with this address */
3173
32dfdf8b
JH
3174 if (result == OK)
3175 {
3176 addr2->more_errno = deliver_time.tv_sec;
3177 addr2->delivery_usec = deliver_time.tv_usec;
3178 }
ba936fb8 3179 post_process_one(addr2, result, logflags, EXIM_DTYPE_TRANSPORT, logchar);
059ec3d9
PH
3180
3181 /* If a pipe delivery generated text to be sent back, the result may be
3182 changed to FAIL, and we must copy this for subsequent addresses in the
3183 batch. */
3184
3185 if (addr2->transport_return != result)
3186 {
d6c82d7b 3187 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
059ec3d9
PH
3188 {
3189 addr3->transport_return = addr2->transport_return;
3190 addr3->basic_errno = addr2->basic_errno;
3191 addr3->message = addr2->message;
3192 }
3193 result = addr2->transport_return;
3194 }
3195
3196 /* Whether or not the result was changed to FAIL, we need to copy the
3197 return_file value from the first address into all the addresses of the
3198 batch, so they are all listed in the error message. */
3199
3200 addr2->return_file = addr->return_file;
3201
3202 /* Change log character for recording successful deliveries. */
3203
3204 if (result == OK) logchar = '-';
3205 }
3206 } /* Loop back for next batch of addresses */
3207}
3208
3209
3210
3211
3212/*************************************************
3213* Sort remote deliveries *
3214*************************************************/
3215
3216/* This function is called if remote_sort_domains is set. It arranges that the
3217chain of addresses for remote deliveries is ordered according to the strings
3218specified. Try to make this shuffling reasonably efficient by handling
3219sequences of addresses rather than just single ones.
3220
3221Arguments: None
3222Returns: Nothing
3223*/
3224
3225static void
3226sort_remote_deliveries(void)
3227{
3228int sep = 0;
3229address_item **aptr = &addr_remote;
55414b25 3230const uschar *listptr = remote_sort_domains;
059ec3d9
PH
3231uschar *pattern;
3232uschar patbuf[256];
3233
d6c82d7b
JH
3234while ( *aptr
3235 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
3236 )
059ec3d9
PH
3237 {
3238 address_item *moved = NULL;
3239 address_item **bptr = &moved;
3240
d6c82d7b 3241 while (*aptr)
059ec3d9
PH
3242 {
3243 address_item **next;
3244 deliver_domain = (*aptr)->domain; /* set $domain */
55414b25 3245 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
059ec3d9
PH
3246 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
3247 {
d6c82d7b 3248 aptr = &(*aptr)->next;
059ec3d9
PH
3249 continue;
3250 }
3251
d6c82d7b
JH
3252 next = &(*aptr)->next;
3253 while ( *next
3254 && (deliver_domain = (*next)->domain, /* Set $domain */
55414b25 3255 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
d6c82d7b
JH
3256 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
3257 )
3258 next = &(*next)->next;
059ec3d9
PH
3259
3260 /* If the batch of non-matchers is at the end, add on any that were
3261 extracted further up the chain, and end this iteration. Otherwise,
3262 extract them from the chain and hang on the moved chain. */
3263
d6c82d7b 3264 if (!*next)
059ec3d9
PH
3265 {
3266 *next = moved;
3267 break;
3268 }
3269
3270 *bptr = *aptr;
3271 *aptr = *next;
3272 *next = NULL;
3273 bptr = next;
d6c82d7b 3274 aptr = &(*aptr)->next;
059ec3d9
PH
3275 }
3276
3277 /* If the loop ended because the final address matched, *aptr will
3278 be NULL. Add on to the end any extracted non-matching addresses. If
3279 *aptr is not NULL, the loop ended via "break" when *next is null, that
3280 is, there was a string of non-matching addresses at the end. In this
3281 case the extracted addresses have already been added on the end. */
3282
d6c82d7b 3283 if (!*aptr) *aptr = moved;
059ec3d9
PH
3284 }
3285
3286DEBUG(D_deliver)
3287 {
3288 address_item *addr;
3289 debug_printf("remote addresses after sorting:\n");
d6c82d7b 3290 for (addr = addr_remote; addr; addr = addr->next)
059ec3d9
PH
3291 debug_printf(" %s\n", addr->address);
3292 }
3293}
3294
3295
3296
3297/*************************************************
3298* Read from pipe for remote delivery subprocess *
3299*************************************************/
3300
3301/* This function is called when the subprocess is complete, but can also be
3302called before it is complete, in order to empty a pipe that is full (to prevent
3303deadlock). It must therefore keep track of its progress in the parlist data
3304block.
3305
3306We read the pipe to get the delivery status codes and a possible error message
3307for each address, optionally preceded by unusability data for the hosts and
3308also by optional retry data.
3309
3310Read in large chunks into the big buffer and then scan through, interpreting
3311the data therein. In most cases, only a single read will be necessary. No
1c5466b9
PH
3312individual item will ever be anywhere near 2500 bytes in length, so by ensuring
3313that we read the next chunk when there is less than 2500 bytes left in the
3314non-final chunk, we can assume each item is complete in the buffer before
3315handling it. Each item is written using a single write(), which is atomic for
3316small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
3317often bigger) so even if we are reading while the subprocess is still going, we
3318should never have only a partial item in the buffer.
059ec3d9 3319
0768462d
JH
3320hs12: This assumption is not true anymore, since we get quite large items (certificate
3321information and such).
2cee425a 3322
059ec3d9
PH
3323Argument:
3324 poffset the offset of the parlist item
3325 eop TRUE if the process has completed
3326
3327Returns: TRUE if the terminating 'Z' item has been read,
3328 or there has been a disaster (i.e. no more data needed);
3329 FALSE otherwise
3330*/
3331
3332static BOOL
3333par_read_pipe(int poffset, BOOL eop)
3334{
3335host_item *h;
3336pardata *p = parlist + poffset;
3337address_item *addrlist = p->addrlist;
3338address_item *addr = p->addr;
3339pid_t pid = p->pid;
3340int fd = p->fd;
2cee425a 3341
059ec3d9
PH
3342uschar *msg = p->msg;
3343BOOL done = p->done;
059ec3d9
PH
3344
3345/* Loop through all items, reading from the pipe when necessary. The pipe
2cee425a
HSHR
3346used to be non-blocking. But I do not see a reason for using non-blocking I/O
3347here, as the preceding select() tells us, if data is available for reading.
3348
3349A read() on a "selected" handle should never block, but(!) it may return
3350less data then we expected. (The buffer size we pass to read() shouldn't be
3351understood as a "request", but as a "limit".)
3352
3353Each separate item is written to the pipe in a timely manner. But, especially for
3354larger items, the read(2) may already return partial data from the write(2).
3355
3356The write is atomic mostly (depending on the amount written), but atomic does
3357not imply "all or noting", it just is "not intermixed" with other writes on the
3358same channel (pipe).
6c454556
HSHR
3359
3360*/
059ec3d9
PH
3361
3362DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
2cee425a 3363 (int)p->pid, eop? "ended" : "not ended yet");
059ec3d9
PH
3364
3365while (!done)
3366 {
3367 retry_item *r, **rp;
2cee425a
HSHR
3368 uschar pipeheader[PIPE_HEADER_SIZE+1];
3369 uschar *id = &pipeheader[0];
3370 uschar *subid = &pipeheader[1];
3371 uschar *ptr = big_buffer;
3372 size_t required = PIPE_HEADER_SIZE; /* first the pipehaeder, later the data */
3373 ssize_t got;
3374
7845dbb3 3375 DEBUG(D_deliver) debug_printf(
0d68983b 3376 "expect %lu bytes (pipeheader) from tpt process %d\n", (u_long)required, pid);
2cee425a
HSHR
3377
3378 /* We require(!) all the PIPE_HEADER_SIZE bytes here, as we know,
3379 they're written in a timely manner, so waiting for the write shouldn't hurt a lot.
3380 If we get less, we can assume the subprocess do be done and do not expect any further
3381 information from it. */
7845dbb3 3382
ee250936 3383 if ((got = readn(fd, pipeheader, required)) != required)
059ec3d9 3384 {
bb07bcd3
JH
3385 msg = string_sprintf("got " SSIZE_T_FMT " of %d bytes (pipeheader) "
3386 "from transport process %d for transport %s",
3387 got, PIPE_HEADER_SIZE, pid, addr->transport->driver_name);
3388 done = TRUE;
3389 break;
059ec3d9
PH
3390 }
3391
2cee425a
HSHR
3392 pipeheader[PIPE_HEADER_SIZE] = '\0';
3393 DEBUG(D_deliver)
7845dbb3
JH
3394 debug_printf("got %ld bytes (pipeheader) from transport process %d\n",
3395 (long) got, pid);
059ec3d9 3396
2cee425a
HSHR
3397 {
3398 /* If we can't decode the pipeheader, the subprocess seems to have a
3399 problem, we do not expect any furher information from it. */
3400 char *endc;
7845dbb3 3401 required = Ustrtol(pipeheader+2, &endc, 10);
bd21a787
WB
3402 if (*endc)
3403 {
2cee425a
HSHR
3404 msg = string_sprintf("failed to read pipe "
3405 "from transport process %d for transport %s: error decoding size from header",
3406 pid, addr->transport->driver_name);
bd21a787
WB
3407 done = TRUE;
3408 break;
3409 }
2cee425a 3410 }
bd21a787
WB
3411
3412 DEBUG(D_deliver)
7845dbb3 3413 debug_printf("expect %lu bytes (pipedata) from transport process %d\n",
0d68983b 3414 (u_long)required, pid);
2c0f3ea1 3415
2cee425a
HSHR
3416 /* Same as above, the transport process will write the bytes announced
3417 in a timely manner, so we can just wait for the bytes, getting less than expected
3418 is considered a problem of the subprocess, we do not expect anything else from it. */
ee250936 3419 if ((got = readn(fd, big_buffer, required)) != required)
8fa0a4d4 3420 {
bb07bcd3
JH
3421 msg = string_sprintf("got only " SSIZE_T_FMT " of " SIZE_T_FMT
3422 " bytes (pipedata) from transport process %d for transport %s",
3423 got, required, pid, addr->transport->driver_name);
3424 done = TRUE;
3425 break;
8fa0a4d4 3426 }
bd21a787 3427
059ec3d9
PH
3428 /* Handle each possible type of item, assuming the complete item is
3429 available in store. */
3430
2cee425a 3431 switch (*id)
059ec3d9
PH
3432 {
3433 /* Host items exist only if any hosts were marked unusable. Match
3434 up by checking the IP address. */
3435
3436 case 'H':
57cc2785
JH
3437 for (h = addrlist->host_list; h; h = h->next)
3438 {
3439 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
3440 h->status = ptr[0];
3441 h->why = ptr[1];
3442 }
3443 ptr += 2;
3444 while (*ptr++);
3445 break;
059ec3d9
PH
3446
3447 /* Retry items are sent in a preceding R item for each address. This is
3448 kept separate to keep each message short enough to guarantee it won't
3449 be split in the pipe. Hopefully, in the majority of cases, there won't in
3450 fact be any retry items at all.
3451
3452 The complete set of retry items might include an item to delete a
3453 routing retry if there was a previous routing delay. However, routing
3454 retries are also used when a remote transport identifies an address error.
3455 In that case, there may also be an "add" item for the same key. Arrange
3456 that a "delete" item is dropped in favour of an "add" item. */
3457
3458 case 'R':
57cc2785 3459 if (!addr) goto ADDR_MISMATCH;
059ec3d9 3460
57cc2785
JH
3461 DEBUG(D_deliver|D_retry)
3462 debug_printf("reading retry information for %s from subprocess\n",
3463 ptr+1);
059ec3d9 3464
57cc2785 3465 /* Cut out any "delete" items on the list. */
059ec3d9 3466
57cc2785
JH
3467 for (rp = &addr->retries; (r = *rp); rp = &r->next)
3468 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3469 {
3470 if (!(r->flags & rf_delete)) break; /* It was not "delete" */
3471 *rp = r->next; /* Excise a delete item */
3472 DEBUG(D_deliver|D_retry)
3473 debug_printf(" existing delete item dropped\n");
3474 }
059ec3d9 3475
57cc2785
JH
3476 /* We want to add a delete item only if there is no non-delete item;
3477 however we still have to step ptr through the data. */
059ec3d9 3478
57cc2785
JH
3479 if (!r || !(*ptr & rf_delete))
3480 {
3481 r = store_get(sizeof(retry_item));
3482 r->next = addr->retries;
3483 addr->retries = r;
3484 r->flags = *ptr++;
3485 r->key = string_copy(ptr);
3486 while (*ptr++);
32dfdf8b 3487 memcpy(&r->basic_errno, ptr, sizeof(r->basic_errno));
57cc2785 3488 ptr += sizeof(r->basic_errno);
32dfdf8b 3489 memcpy(&r->more_errno, ptr, sizeof(r->more_errno));
57cc2785
JH
3490 ptr += sizeof(r->more_errno);
3491 r->message = *ptr ? string_copy(ptr) : NULL;
3492 DEBUG(D_deliver|D_retry) debug_printf(" added %s item\n",
3493 r->flags & rf_delete ? "delete" : "retry");
3494 }
059ec3d9 3495
57cc2785
JH
3496 else
3497 {
3498 DEBUG(D_deliver|D_retry)
3499 debug_printf(" delete item not added: non-delete item exists\n");
3500 ptr++;
3501 while(*ptr++);
3502 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3503 }
059ec3d9 3504
57cc2785
JH
3505 while(*ptr++);
3506 break;
059ec3d9
PH
3507
3508 /* Put the amount of data written into the parlist block */
3509
3510 case 'S':
57cc2785
JH
3511 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3512 ptr += sizeof(transport_count);
3513 break;
059ec3d9
PH
3514
3515 /* Address items are in the order of items on the address chain. We
3516 remember the current address value in case this function is called
3517 several times to empty the pipe in stages. Information about delivery
3518 over TLS is sent in a preceding X item for each address. We don't put
3519 it in with the other info, in order to keep each message short enough to
3520 guarantee it won't be split in the pipe. */
3521
a7538db1 3522#ifdef SUPPORT_TLS
059ec3d9 3523 case 'X':
57cc2785 3524 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
2cee425a 3525 switch (*subid)
57cc2785
JH
3526 {
3527 case '1':
3528 addr->cipher = NULL;
3529 addr->peerdn = NULL;
9d1c15ef 3530
57cc2785
JH
3531 if (*ptr)
3532 addr->cipher = string_copy(ptr);
3533 while (*ptr++);
3534 if (*ptr)
3535 addr->peerdn = string_copy(ptr);
3536 break;
3537
3538 case '2':
3539 if (*ptr)
3540 (void) tls_import_cert(ptr, &addr->peercert);
3541 else
3542 addr->peercert = NULL;
3543 break;
3544
3545 case '3':
3546 if (*ptr)
3547 (void) tls_import_cert(ptr, &addr->ourcert);
3548 else
3549 addr->ourcert = NULL;
3550 break;
018058b2 3551
a7538db1 3552# ifndef DISABLE_OCSP
57cc2785
JH
3553 case '4':
3554 addr->ocsp = *ptr ? *ptr - '0' : OCSP_NOT_REQ;
3555 break;
a7538db1 3556# endif
57cc2785
JH
3557 }
3558 while (*ptr++);
3559 break;
a7538db1 3560#endif /*SUPPORT_TLS*/
059ec3d9 3561
6f123593 3562 case 'C': /* client authenticator information */
2cee425a 3563 switch (*subid)
57cc2785
JH
3564 {
3565 case '1': addr->authenticator = *ptr ? string_copy(ptr) : NULL; break;
3566 case '2': addr->auth_id = *ptr ? string_copy(ptr) : NULL; break;
3567 case '3': addr->auth_sndr = *ptr ? string_copy(ptr) : NULL; break;
3568 }
3569 while (*ptr++);
3570 break;
6f123593 3571
8ccd00b1 3572#ifndef DISABLE_PRDR
fd98a5c6 3573 case 'P':
7eb0e5d2 3574 setflag(addr, af_prdr_used);
57cc2785 3575 break;
fd98a5c6
JH
3576#endif
3577
7c576fca
JH
3578 case 'L':
3579 switch (*subid)
3580 {
ee8b8090
JH
3581#ifdef EXPERIMENTAL_PIPE_CONNECT
3582 case 2: setflag(addr, af_early_pipe); /*FALLTHROUGH*/
3583#endif
7c576fca
JH
3584 case 1: setflag(addr, af_pipelining); break;
3585 }
3586 break;
3587
e0cc6cda 3588 case 'K':
7eb0e5d2 3589 setflag(addr, af_chunking_used);
57cc2785 3590 break;
e0cc6cda 3591
a2673768 3592 case 'T':
1ccd5f67
JH
3593 setflag(addr, af_tcp_fastopen_conn);
3594 if (*subid > '0') setflag(addr, af_tcp_fastopen);
ee8b8090 3595 if (*subid > '1') setflag(addr, af_tcp_fastopen_data);
a2673768
JH
3596 break;
3597
6c1c3d1d 3598 case 'D':
57cc2785
JH
3599 if (!addr) goto ADDR_MISMATCH;
3600 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3601 ptr += sizeof(addr->dsn_aware);
3602 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
059ec3d9 3603 break;
059ec3d9 3604
57cc2785
JH
3605 case 'A':
3606 if (!addr)
3607 {
3608 ADDR_MISMATCH:
3609 msg = string_sprintf("address count mismatch for data read from pipe "
3610 "for transport process %d for transport %s", pid,
3611 addrlist->transport->driver_name);
3612 done = TRUE;
e6d2a989 3613 break;
57cc2785 3614 }
e6d2a989 3615
2cee425a 3616 switch (*subid)
57cc2785
JH
3617 {
3618 #ifdef SUPPORT_SOCKS
3619 case '2': /* proxy information; must arrive before A0 and applies to that addr XXX oops*/
3620 proxy_session = TRUE; /*XXX should this be cleared somewhere? */
3621 if (*ptr == 0)
3622 ptr++;
3623 else
3624 {
3625 proxy_local_address = string_copy(ptr);
3626 while(*ptr++);
3627 memcpy(&proxy_local_port, ptr, sizeof(proxy_local_port));
3628 ptr += sizeof(proxy_local_port);
3629 }
3630 break;
3631 #endif
059ec3d9 3632
57cc2785
JH
3633 #ifdef EXPERIMENTAL_DSN_INFO
3634 case '1': /* must arrive before A0, and applies to that addr */
3635 /* Two strings: smtp_greeting and helo_response */
3636 addr->smtp_greeting = string_copy(ptr);
3637 while(*ptr++);
3638 addr->helo_response = string_copy(ptr);
3639 while(*ptr++);
3640 break;
3641 #endif
3642
3643 case '0':
3644 DEBUG(D_deliver) debug_printf("A0 %s tret %d\n", addr->address, *ptr);
3645 addr->transport_return = *ptr++;
3646 addr->special_action = *ptr++;
32dfdf8b 3647 memcpy(&addr->basic_errno, ptr, sizeof(addr->basic_errno));
57cc2785 3648 ptr += sizeof(addr->basic_errno);
32dfdf8b 3649 memcpy(&addr->more_errno, ptr, sizeof(addr->more_errno));
57cc2785 3650 ptr += sizeof(addr->more_errno);
32dfdf8b
JH
3651 memcpy(&addr->delivery_usec, ptr, sizeof(addr->delivery_usec));
3652 ptr += sizeof(addr->delivery_usec);
3653 memcpy(&addr->flags, ptr, sizeof(addr->flags));
57cc2785
JH
3654 ptr += sizeof(addr->flags);
3655 addr->message = *ptr ? string_copy(ptr) : NULL;
3656 while(*ptr++);
3657 addr->user_message = *ptr ? string_copy(ptr) : NULL;
3658 while(*ptr++);
059ec3d9 3659
57cc2785 3660 /* Always two strings for host information, followed by the port number and DNSSEC mark */
895fbaf2 3661
57cc2785
JH
3662 if (*ptr)
3663 {
3664 h = store_get(sizeof(host_item));
3665 h->name = string_copy(ptr);
3666 while (*ptr++);
3667 h->address = string_copy(ptr);
3668 while(*ptr++);
3669 memcpy(&h->port, ptr, sizeof(h->port));
3670 ptr += sizeof(h->port);
3671 h->dnssec = *ptr == '2' ? DS_YES
3672 : *ptr == '1' ? DS_NO
3673 : DS_UNK;
3674 ptr++;
3675 addr->host_used = h;
3676 }
3677 else ptr++;
895fbaf2 3678
57cc2785 3679 /* Finished with this address */
895fbaf2 3680
57cc2785
JH
3681 addr = addr->next;
3682 break;
3683 }
3684 break;
059ec3d9 3685
aeaf5db3
JH
3686 /* Local interface address/port */
3687 case 'I':
57cc2785
JH
3688 if (*ptr) sending_ip_address = string_copy(ptr);
3689 while (*ptr++) ;
3690 if (*ptr) sending_port = atoi(CS ptr);
3691 while (*ptr++) ;
3692 break;
aeaf5db3 3693
059ec3d9
PH
3694 /* Z marks the logical end of the data. It is followed by '0' if
3695 continue_transport was NULL at the end of transporting, otherwise '1'.
3696 We need to know when it becomes NULL during a delivery down a passed SMTP
3697 channel so that we don't try to pass anything more down it. Of course, for
3698 most normal messages it will remain NULL all the time. */
3699
3700 case 'Z':
57cc2785
JH
3701 if (*ptr == '0')
3702 {
3703 continue_transport = NULL;
3704 continue_hostname = NULL;
3705 }
3706 done = TRUE;
3707 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
3708 break;
059ec3d9
PH
3709
3710 /* Anything else is a disaster. */
3711
3712 default:
57cc2785
JH
3713 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3714 "process %d for transport %s", ptr[-1], pid,
3715 addr->transport->driver_name);
3716 done = TRUE;
3717 break;
059ec3d9
PH
3718 }
3719 }
3720
3721/* The done flag is inspected externally, to determine whether or not to
3722call the function again when the process finishes. */
3723
3724p->done = done;
3725
3726/* If the process hadn't finished, and we haven't seen the end of the data
2cee425a 3727or if we suffered a disaster, update the rest of the state, and return FALSE to
059ec3d9
PH
3728indicate "not finished". */
3729
3730if (!eop && !done)
3731 {
3732 p->addr = addr;
3733 p->msg = msg;
3734 return FALSE;
3735 }
3736
3737/* Close our end of the pipe, to prevent deadlock if the far end is still
3738pushing stuff into it. */
3739
f1e894f3 3740(void)close(fd);
059ec3d9
PH
3741p->fd = -1;
3742
3743/* If we have finished without error, but haven't had data for every address,
3744something is wrong. */
3745
d6c82d7b 3746if (!msg && addr)
059ec3d9
PH
3747 msg = string_sprintf("insufficient address data read from pipe "
3748 "for transport process %d for transport %s", pid,
3749 addr->transport->driver_name);
3750
3751/* If an error message is set, something has gone wrong in getting back
3752the delivery data. Put the message into each address and freeze it. */
3753
d6c82d7b
JH
3754if (msg)
3755 for (addr = addrlist; addr; addr = addr->next)
059ec3d9
PH
3756 {
3757 addr->transport_return = DEFER;
3758 addr->special_action = SPECIAL_FREEZE;
3759 addr->message = msg;
2cee425a 3760 log_write(0, LOG_MAIN|LOG_PANIC, "Delivery status for %s: %s\n", addr->address, addr->message);
059ec3d9 3761 }
059ec3d9
PH
3762
3763/* Return TRUE to indicate we have got all we need from this process, even
3764if it hasn't actually finished yet. */
3765
3766return TRUE;
3767}
3768
3769
3770
3771/*************************************************
3772* Post-process a set of remote addresses *
3773*************************************************/
3774
3775/* Do what has to be done immediately after a remote delivery for each set of
3776addresses, then re-write the spool if necessary. Note that post_process_one
3777puts the address on an appropriate queue; hence we must fish off the next
3778one first. This function is also called if there is a problem with setting
3779up a subprocess to do a remote delivery in parallel. In this case, the final
3780argument contains a message, and the action must be forced to DEFER.
3781
3782Argument:
3783 addr pointer to chain of address items
3784 logflags flags for logging
3785 msg NULL for normal cases; -> error message for unexpected problems
3786 fallback TRUE if processing fallback hosts
3787
3788Returns: nothing
3789*/
3790
3791static void
3792remote_post_process(address_item *addr, int logflags, uschar *msg,
3793 BOOL fallback)
3794{
3795host_item *h;
3796
3797/* If any host addresses were found to be unusable, add them to the unusable
3798tree so that subsequent deliveries don't try them. */
3799
d6c82d7b
JH
3800for (h = addr->host_list; h; h = h->next)
3801 if (h->address)
3802 if (h->status >= hstatus_unusable) tree_add_unusable(h);
059ec3d9
PH
3803
3804/* Now handle each address on the chain. The transport has placed '=' or '-'
3805into the special_action field for each successful delivery. */
3806
d6c82d7b 3807while (addr)
059ec3d9
PH
3808 {
3809 address_item *next = addr->next;
3810
3811 /* If msg == NULL (normal processing) and the result is DEFER and we are
3812 processing the main hosts and there are fallback hosts available, put the
3813 address on the list for fallback delivery. */
3814
d6c82d7b
JH
3815 if ( addr->transport_return == DEFER
3816 && addr->fallback_hosts
3817 && !fallback
3818 && !msg
3819 )
059ec3d9
PH
3820 {
3821 addr->host_list = addr->fallback_hosts;
3822 addr->next = addr_fallback;
3823 addr_fallback = addr;
3824 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3825 }
3826
3827 /* If msg is set (=> unexpected problem), set it in the address before
3828 doing the ordinary post processing. */
3829
3830 else
3831 {
d6c82d7b 3832 if (msg)
059ec3d9
PH
3833 {
3834 addr->message = msg;
3835 addr->transport_return = DEFER;
3836 }
3837 (void)post_process_one(addr, addr->transport_return, logflags,
ba936fb8 3838 EXIM_DTYPE_TRANSPORT, addr->special_action);
059ec3d9
PH
3839 }
3840
3841 /* Next address */
3842
3843 addr = next;
3844 }
3845
3846/* If we have just delivered down a passed SMTP channel, and that was
3847the last address, the channel will have been closed down. Now that
3848we have logged that delivery, set continue_sequence to 1 so that
3849any subsequent deliveries don't get "*" incorrectly logged. */
3850
d6c82d7b 3851if (!continue_transport) continue_sequence = 1;
059ec3d9
PH
3852}
3853
3854
3855
3856/*************************************************
3857* Wait for one remote delivery subprocess *
3858*************************************************/
3859
3860/* This function is called while doing remote deliveries when either the
3861maximum number of processes exist and we need one to complete so that another
3862can be created, or when waiting for the last ones to complete. It must wait for
3863the completion of one subprocess, empty the control block slot, and return a
3864pointer to the address chain.
3865
3866Arguments: none
3867Returns: pointer to the chain of addresses handled by the process;
3868 NULL if no subprocess found - this is an unexpected error
3869*/
3870
3871static address_item *
3872par_wait(void)
3873{
3874int poffset, status;
3875address_item *addr, *addrlist;
3876pid_t pid;
3877
3878set_process_info("delivering %s: waiting for a remote delivery subprocess "
3879 "to finish", message_id);
3880
3881/* Loop until either a subprocess completes, or there are no subprocesses in
3882existence - in which case give an error return. We cannot proceed just by
3883waiting for a completion, because a subprocess may have filled up its pipe, and
3884be waiting for it to be emptied. Therefore, if no processes have finished, we
3885wait for one of the pipes to acquire some data by calling select(), with a
3886timeout just in case.
3887
3888The simple approach is just to iterate after reading data from a ready pipe.
3889This leads to non-ideal behaviour when the subprocess has written its final Z
3890item, closed the pipe, and is in the process of exiting (the common case). A
3891call to waitpid() yields nothing completed, but select() shows the pipe ready -
3892reading it yields EOF, so you end up with busy-waiting until the subprocess has
3893actually finished.
3894
3895To avoid this, if all the data that is needed has been read from a subprocess
3896after select(), an explicit wait() for it is done. We know that all it is doing
3897is writing to the pipe and then exiting, so the wait should not be long.
3898
3899The non-blocking waitpid() is to some extent just insurance; if we could
3900reliably detect end-of-file on the pipe, we could always know when to do a
3901blocking wait() for a completed process. However, because some systems use
3902NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3903use code that functions without the need to recognize EOF.
3904
3905There's a double loop here just in case we end up with a process that is not in
3906the list of remote delivery processes. Something has obviously gone wrong if
3907this is the case. (For example, a process that is incorrectly left over from
3908routing or local deliveries might be found.) The damage can be minimized by
3909looping back and looking for another process. If there aren't any, the error
3910return will happen. */
3911
3912for (;;) /* Normally we do not repeat this loop */
3913 {
3914 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3915 {
3916 struct timeval tv;
3917 fd_set select_pipes;
3918 int maxpipe, readycount;
3919
3920 /* A return value of -1 can mean several things. If errno != ECHILD, it
3921 either means invalid options (which we discount), or that this process was
3922 interrupted by a signal. Just loop to try the waitpid() again.
3923
3924 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3925 in existence. This should never happen, and is an unexpected error.
3926 However, there is a nasty complication when running under Linux. If "strace
3927 -f" is being used under Linux to trace this process and its children,
3928 subprocesses are "stolen" from their parents and become the children of the
3929 tracing process. A general wait such as the one we've just obeyed returns
3930 as if there are no children while subprocesses are running. Once a
3931 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3932 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3933 palliative.
3934
3935 This does not happen using "truss" on Solaris, nor (I think) with other
3936 tracing facilities on other OS. It seems to be specific to Linux.
3937
3938 What we do to get round this is to use kill() to see if any of our
3939 subprocesses are still in existence. If kill() gives an OK return, we know
3940 it must be for one of our processes - it can't be for a re-use of the pid,
3941 because if our process had finished, waitpid() would have found it. If any
3942 of our subprocesses are in existence, we proceed to use select() as if
3943 waitpid() had returned zero. I think this is safe. */
3944
3945 if (pid < 0)
3946 {
3947 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3948
3949 DEBUG(D_deliver)
3950 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3951 "for process existence\n");
3952
3953 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3954 {
3955 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3956 {
3957 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3958 "stolen by strace\n", (int)pid);
3959 break; /* With poffset set */
3960 }
3961 }
3962
3963 if (poffset >= remote_max_parallel)
3964 {
3965 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3966 return NULL; /* This is the error return */
3967 }
3968 }
3969
3970 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3971 been handled above. A return value of zero means that there is at least one
3972 subprocess, but there are no completed subprocesses. See if any pipes are
3973 ready with any data for reading. */
3974
3975 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3976
3977 maxpipe = 0;
3978 FD_ZERO(&select_pipes);
3979 for (poffset = 0; poffset < remote_max_parallel; poffset++)
059ec3d9
PH
3980 if (parlist[poffset].pid != 0)
3981 {
3982 int fd = parlist[poffset].fd;
3983 FD_SET(fd, &select_pipes);
3984 if (fd > maxpipe) maxpipe = fd;
3985 }
059ec3d9
PH
3986
3987 /* Stick in a 60-second timeout, just in case. */
3988
3989 tv.tv_sec = 60;
3990 tv.tv_usec = 0;
3991
3992 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3993 NULL, NULL, &tv);
3994
3995 /* Scan through the pipes and read any that are ready; use the count
3996 returned by select() to stop when there are no more. Select() can return
3997 with no processes (e.g. if interrupted). This shouldn't matter.
3998
3999 If par_read_pipe() returns TRUE, it means that either the terminating Z was
4000 read, or there was a disaster. In either case, we are finished with this
4001 process. Do an explicit wait() for the process and break the main loop if
4002 it succeeds.
4003
4004 It turns out that we have to deal with the case of an interrupted system
4005 call, which can happen on some operating systems if the signal handling is
4006 set up to do that by default. */
4007
4008 for (poffset = 0;
4009 readycount > 0 && poffset < remote_max_parallel;
4010 poffset++)
4011 {
d6c82d7b
JH
4012 if ( (pid = parlist[poffset].pid) != 0
4013 && FD_ISSET(parlist[poffset].fd, &select_pipes)
4014 )
059ec3d9
PH
4015 {
4016 readycount--;
4017 if (par_read_pipe(poffset, FALSE)) /* Finished with this pipe */
059ec3d9
PH
4018 for (;;) /* Loop for signals */
4019 {
4020 pid_t endedpid = waitpid(pid, &status, 0);
4021 if (endedpid == pid) goto PROCESS_DONE;
4022 if (endedpid != (pid_t)(-1) || errno != EINTR)
4023 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return "
4024 "%d (errno = %d) from waitpid() for process %d",
4025 (int)endedpid, errno, (int)pid);
4026 }
059ec3d9
PH
4027 }
4028 }
4029
4030 /* Now go back and look for a completed subprocess again. */
4031 }
4032
4033 /* A completed process was detected by the non-blocking waitpid(). Find the
4034 data block that corresponds to this subprocess. */
4035
4036 for (poffset = 0; poffset < remote_max_parallel; poffset++)
4037 if (pid == parlist[poffset].pid) break;
4038
4039 /* Found the data block; this is a known remote delivery process. We don't
4040 need to repeat the outer loop. This should be what normally happens. */
4041
4042 if (poffset < remote_max_parallel) break;
4043
4044 /* This situation is an error, but it's probably better to carry on looking
4045 for another process than to give up (as we used to do). */
4046
4047 log_write(0, LOG_MAIN|LOG_PANIC, "Process %d finished: not found in remote "
4048 "transport process list", pid);
4049 } /* End of the "for" loop */
4050
4051/* Come here when all the data was completely read after a select(), and
4052the process in pid has been wait()ed for. */
4053
4054PROCESS_DONE:
4055
4056DEBUG(D_deliver)
4057 {
4058 if (status == 0)
4059 debug_printf("remote delivery process %d ended\n", (int)pid);
4060 else
4061 debug_printf("remote delivery process %d ended: status=%04x\n", (int)pid,
4062 status);
4063 }
4064
4065set_process_info("delivering %s", message_id);
4066
4067/* Get the chain of processed addresses */
4068
4069addrlist = parlist[poffset].addrlist;
4070
4071/* If the process did not finish cleanly, record an error and freeze (except
4072for SIGTERM, SIGKILL and SIGQUIT), and also ensure the journal is not removed,
4073in case the delivery did actually happen. */
4074
4075if ((status & 0xffff) != 0)
4076 {
4077 uschar *msg;
4078 int msb = (status >> 8) & 255;
4079 int lsb = status & 255;
4080 int code = (msb == 0)? (lsb & 0x7f) : msb;
4081
4082 msg = string_sprintf("%s transport process returned non-zero status 0x%04x: "
4083 "%s %d",
4084 addrlist->transport->driver_name,
4085 status,
4086 (msb == 0)? "terminated by signal" : "exit code",
4087 code);
4088
4089 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
4090 addrlist->special_action = SPECIAL_FREEZE;
4091
d6c82d7b 4092 for (addr = addrlist; addr; addr = addr->next)
059ec3d9
PH
4093 {
4094 addr->transport_return = DEFER;
4095 addr->message = msg;
4096 }
4097
4098 remove_journal = FALSE;
4099 }
4100
4101/* Else complete reading the pipe to get the result of the delivery, if all
4102the data has not yet been obtained. */
4103
4104else if (!parlist[poffset].done) (void)par_read_pipe(poffset, TRUE);
4105
4106/* Put the data count and return path into globals, mark the data slot unused,
4107decrement the count of subprocesses, and return the address chain. */
4108
4109transport_count = parlist[poffset].transport_count;
4110used_return_path = parlist[poffset].return_path;
4111parlist[poffset].pid = 0;
4112parcount--;
4113return addrlist;
4114}
4115
4116
4117
4118/*************************************************
4119* Wait for subprocesses and post-process *
4120*************************************************/
4121
4122/* This function waits for subprocesses until the number that are still running
4123is below a given threshold. For each complete subprocess, the addresses are
4124post-processed. If we can't find a running process, there is some shambles.
4125Better not bomb out, as that might lead to multiple copies of the message. Just
4126log and proceed as if all done.
4127
4128Arguments:
4129 max maximum number of subprocesses to leave running
4130 fallback TRUE if processing fallback hosts
4131
4132Returns: nothing
4133*/
4134
4135static void
4136par_reduce(int max, BOOL fallback)
4137{
4138while (parcount > max)
4139 {
4140 address_item *doneaddr = par_wait();
d6c82d7b 4141 if (!doneaddr)
059ec3d9
PH
4142 {
4143 log_write(0, LOG_MAIN|LOG_PANIC,
4144 "remote delivery process count got out of step");
4145 parcount = 0;
4146 }
fa41615d
JH
4147 else
4148 {
4149 transport_instance * tp = doneaddr->transport;
4150 if (tp->max_parallel)
4151 enq_end(string_sprintf("tpt-serialize-%s", tp->name));
4152
4153 remote_post_process(doneaddr, LOG_MAIN, NULL, fallback);
4154 }
059ec3d9
PH
4155 }
4156}
4157
1ac6b2e7 4158static void
2cee425a 4159rmt_dlv_checked_write(int fd, char id, char subid, void * buf, ssize_t size)
1ac6b2e7 4160{
6c454556
HSHR
4161uschar pipe_header[PIPE_HEADER_SIZE+1];
4162size_t total_len = PIPE_HEADER_SIZE + size;
4163
4164struct iovec iov[2] = {
4165 { pipe_header, PIPE_HEADER_SIZE }, /* indication about the data to expect */
4166 { buf, size } /* *the* data */
4167};
4168
2cee425a 4169ssize_t ret;
bd21a787
WB
4170
4171/* we assume that size can't get larger then BIG_BUFFER_SIZE which currently is set to 16k */
4172/* complain to log if someone tries with buffer sizes we can't handle*/
4173
6c454556 4174if (size > BIG_BUFFER_SIZE-1)
d6c82d7b 4175 {
94431adb 4176 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
6c454556
HSHR
4177 "Failed writing transport result to pipe: can't handle buffers > %d bytes. truncating!\n",
4178 BIG_BUFFER_SIZE-1);
4179 size = BIG_BUFFER_SIZE;
d6c82d7b 4180 }
bd21a787 4181
7845dbb3 4182/* Should we check that we do not write more than PIPE_BUF? What would
6c454556 4183that help? */
bd21a787
WB
4184
4185/* convert size to human readable string prepended by id and subid */
7845dbb3
JH
4186if (PIPE_HEADER_SIZE != snprintf(CS pipe_header, PIPE_HEADER_SIZE+1, "%c%c%05ld",
4187 id, subid, (long)size))
bd21a787 4188 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "header snprintf failed\n");
bd21a787 4189
7845dbb3
JH
4190DEBUG(D_deliver) debug_printf("header write id:%c,subid:%c,size:%ld,final:%s\n",
4191 id, subid, (long)size, pipe_header);
bd21a787 4192
6c454556 4193if ((ret = writev(fd, iov, 2)) != total_len)
7845dbb3
JH
4194 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
4195 "Failed writing transport result to pipe (%ld of %ld bytes): %s",
4196 (long)ret, (long)total_len, ret == -1 ? strerror(errno) : "short write");
1ac6b2e7
JH
4197}
4198
059ec3d9
PH
4199/*************************************************
4200* Do remote deliveries *
4201*************************************************/
4202
4203/* This function is called to process the addresses in addr_remote. We must
4204pick off the queue all addresses that have the same transport, remote
4205destination, and errors address, and hand them to the transport in one go,
4206subject to some configured limitations. If this is a run to continue delivering
4207to an existing delivery channel, skip all but those addresses that can go to
4208that channel. The skipped addresses just get deferred.
4209
4210If mua_wrapper is set, all addresses must be able to be sent in a single
4211transaction. If not, this function yields FALSE.
4212
4213In Exim 4, remote deliveries are always done in separate processes, even
4214if remote_max_parallel = 1 or if there's only one delivery to do. The reason
4215is so that the base process can retain privilege. This makes the
4216implementation of fallback transports feasible (though not initially done.)
4217
4218We create up to the configured number of subprocesses, each of which passes
4219back the delivery state via a pipe. (However, when sending down an existing
4220connection, remote_max_parallel is forced to 1.)
4221
4222Arguments:
4223 fallback TRUE if processing fallback hosts
4224
4225Returns: TRUE normally
4226 FALSE if mua_wrapper is set and the addresses cannot all be sent
4227 in one transaction
4228*/
4229
4230static BOOL
4231do_remote_deliveries(BOOL fallback)
4232{
4233int parmax;
4234int delivery_count;
4235int poffset;
4236
4237parcount = 0; /* Number of executing subprocesses */
4238
4239/* When sending down an existing channel, only do one delivery at a time.
4240We use a local variable (parmax) to hold the maximum number of processes;
4241this gets reduced from remote_max_parallel if we can't create enough pipes. */
4242
d6c82d7b 4243if (continue_transport) remote_max_parallel = 1;
059ec3d9
PH
4244parmax = remote_max_parallel;
4245
4246/* If the data for keeping a list of processes hasn't yet been
4247set up, do so. */
4248
d6c82d7b 4249if (!parlist)
059ec3d9
PH
4250 {
4251 parlist = store_get(remote_max_parallel * sizeof(pardata));
4252 for (poffset = 0; poffset < remote_max_parallel; poffset++)
4253 parlist[poffset].pid = 0;
4254 }
4255
4256/* Now loop for each remote delivery */
4257
d6c82d7b 4258for (delivery_count = 0; addr_remote; delivery_count++)
059ec3d9
PH
4259 {
4260 pid_t pid;
4261 uid_t uid;
4262 gid_t gid;
4263 int pfd[2];
4264 int address_count = 1;
4265 int address_count_max;
4266 BOOL multi_domain;
4267 BOOL use_initgroups;
4268 BOOL pipe_done = FALSE;
4269 transport_instance *tp;
4270 address_item **anchor = &addr_remote;
4271 address_item *addr = addr_remote;
4272 address_item *last = addr;
4273 address_item *next;
d6c82d7b 4274 uschar * panicmsg;
fa41615d 4275 uschar * serialize_key = NULL;
059ec3d9
PH
4276
4277 /* Pull the first address right off the list. */
4278
4279 addr_remote = addr->next;
4280 addr->next = NULL;
4281
4282 DEBUG(D_deliver|D_transport)
4283 debug_printf("--------> %s <--------\n", addr->address);
4284
4285 /* If no transport has been set, there has been a big screw-up somewhere. */
4286
d6c82d7b 4287 if (!(tp = addr->transport))
059ec3d9 4288 {
8768d548 4289 f.disable_logging = FALSE; /* Jic */
d6c82d7b
JH
4290 panicmsg = US"No transport set by router";
4291 goto panic_continue;
059ec3d9
PH
4292 }
4293
4294 /* Check that this base address hasn't previously been delivered to this
4295 transport. The check is necessary at this point to handle homonymic addresses
4296 correctly in cases where the pattern of redirection changes between delivery
4297 attempts. Non-homonymic previous delivery is detected earlier, at routing
4298 time. */
4299
c2c19e9d 4300 if (previously_transported(addr, FALSE)) continue;
059ec3d9
PH
4301
4302 /* Force failure if the message is too big. */
4303
d6c82d7b 4304 if (tp->message_size_limit)
059ec3d9
PH
4305 {
4306 int rc = check_message_size(tp, addr);
4307 if (rc != OK)
4308 {
4309 addr->transport_return = rc;
4310 remote_post_process(addr, LOG_MAIN, NULL, fallback);
4311 continue;
4312 }
4313 }
4314
4315 /* Get the flag which specifies whether the transport can handle different
506900af
JH
4316 domains that nevertheless resolve to the same set of hosts. If it needs
4317 expanding, get variables set: $address_data, $domain_data, $localpart_data,
4318 $host, $host_address, $host_port. */
4319 if (tp->expand_multi_domain)
4320 deliver_set_expansions(addr);
4321
4322 if (exp_bool(addr, US"transport", tp->name, D_transport,
4323 US"multi_domain", tp->multi_domain, tp->expand_multi_domain,
4324 &multi_domain) != OK)
4325 {
4326 deliver_set_expansions(NULL);
d6c82d7b
JH
4327 panicmsg = addr->message;
4328 goto panic_continue;
506900af 4329 }
059ec3d9
PH
4330
4331 /* Get the maximum it can handle in one envelope, with zero meaning
4332 unlimited, which is forced for the MUA wrapper case. */
4333
4334 address_count_max = tp->max_addresses;
4335 if (address_count_max == 0 || mua_wrapper) address_count_max = 999999;
4336
4337
4338 /************************************************************************/
4339 /***** This is slightly experimental code, but should be safe. *****/
4340
4341 /* The address_count_max value is the maximum number of addresses that the
4342 transport can send in one envelope. However, the transport must be capable of
4343 dealing with any number of addresses. If the number it gets exceeds its
4344 envelope limitation, it must send multiple copies of the message. This can be
4345 done over a single connection for SMTP, so uses less resources than making
4346 multiple connections. On the other hand, if remote_max_parallel is greater
4347 than one, it is perhaps a good idea to use parallel processing to move the
4348 message faster, even if that results in multiple simultaneous connections to
4349 the same host.
4350
4351 How can we come to some compromise between these two ideals? What we do is to
4352 limit the number of addresses passed to a single instance of a transport to
4353 the greater of (a) its address limit (rcpt_max for SMTP) and (b) the total
4354 number of addresses routed to remote transports divided by
4355 remote_max_parallel. For example, if the message has 100 remote recipients,
4356 remote max parallel is 2, and rcpt_max is 10, we'd never send more than 50 at
4357 once. But if rcpt_max is 100, we could send up to 100.
4358
4359 Of course, not all the remotely addresses in a message are going to go to the
4360 same set of hosts (except in smarthost configurations), so this is just a
4361 heuristic way of dividing up the work.
4362
4363 Furthermore (1), because this may not be wanted in some cases, and also to
4364 cope with really pathological cases, there is also a limit to the number of
4365 messages that are sent over one connection. This is the same limit that is
4366 used when sending several different messages over the same connection.
4367 Continue_sequence is set when in this situation, to the number sent so
4368 far, including this message.
4369
4370 Furthermore (2), when somebody explicitly sets the maximum value to 1, it
4371 is probably because they are using VERP, in which case they want to pass only
4372 one address at a time to the transport, in order to be able to use
4373 $local_part and $domain in constructing a new return path. We could test for
4374 the use of these variables, but as it is so likely they will be used when the
4375 maximum is 1, we don't bother. Just leave the value alone. */
4376
d6c82d7b
JH
4377 if ( address_count_max != 1
4378 && address_count_max < remote_delivery_count/remote_max_parallel
4379 )
059ec3d9
PH
4380 {
4381 int new_max = remote_delivery_count/remote_max_parallel;
4382 int message_max = tp->connection_max_messages;
4383 if (connection_max_messages >= 0) message_max = connection_max_messages;
4384 message_max -= continue_sequence - 1;
4385 if (message_max > 0 && new_max > address_count_max * message_max)
4386 new_max = address_count_max * message_max;
4387 address_count_max = new_max;
4388 }
4389
4390 /************************************************************************/
4391
4392
4393 /* Pick off all addresses which have the same transport, errors address,
4394 destination, and extra headers. In some cases they point to the same host
4395 list, but we also need to check for identical host lists generated from
4396 entirely different domains. The host list pointers can be NULL in the case
4397 where the hosts are defined in the transport. There is also a configured
4398 maximum limit of addresses that can be handled at once (see comments above
506900af
JH
4399 for how it is computed).
4400 If the transport does not handle multiple domains, enforce that also,
4401 and if it might need a per-address check for this, re-evaluate it.
4402 */
059ec3d9 4403
d6c82d7b 4404 while ((next = *anchor) && address_count < address_count_max)
059ec3d9 4405 {
506900af
JH
4406 BOOL md;
4407 if ( (multi_domain || Ustrcmp(next->domain, addr->domain) == 0)
4408 && tp == next->transport
4409 && same_hosts(next->host_list, addr->host_list)
d43cbe25
JH
4410 && same_strings(next->prop.errors_address, addr->prop.errors_address)
4411 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
506900af 4412 && same_ugid(tp, next, addr)
d43cbe25
JH
4413 && ( next->prop.remove_headers == addr->prop.remove_headers
4414 || ( next->prop.remove_headers
4415 && addr->prop.remove_headers
4416 && Ustrcmp(next->prop.remove_headers, addr->prop.remove_headers) == 0
506900af
JH
4417 ) )
4418 && ( !multi_domain
4419 || ( (
863bd541 4420 (void)(!tp->expand_multi_domain || ((void)deliver_set_expansions(next), 1)),
506900af
JH
4421 exp_bool(addr,
4422 US"transport", next->transport->name, D_transport,
4423 US"multi_domain", next->transport->multi_domain,
4424 next->transport->expand_multi_domain, &md) == OK
4425 )
4426 && md
4427 ) ) )
059ec3d9
PH
4428 {
4429 *anchor = next->next;
4430 next->next = NULL;
4431 next->first = addr; /* remember top one (for retry processing) */
4432 last->next = next;
4433 last = next;
4434 address_count++;
4435 }
4436 else anchor = &(next->next);
506900af 4437 deliver_set_expansions(NULL);
059ec3d9
PH
4438 }
4439
4440 /* If we are acting as an MUA wrapper, all addresses must go in a single
4441 transaction. If not, put them back on the chain and yield FALSE. */
4442
d6c82d7b 4443 if (mua_wrapper && addr_remote)
059ec3d9
PH
4444 {
4445 last->next = addr_remote;
4446 addr_remote = addr;
4447 return FALSE;
4448 }
4449
fa41615d
JH
4450 /* If the transport is limited for parallellism, enforce that here.
4451 The hints DB entry is decremented in par_reduce(), when we reap the
4452 transport process. */
4453
3070ceee
JH
4454 if (tpt_parallel_check(tp, addr, &serialize_key))
4455 if ((panicmsg = expand_string_message))
fa41615d 4456 goto panic_continue;
3070ceee
JH
4457 else
4458 continue; /* Loop for the next set of addresses. */
fa41615d 4459
059ec3d9
PH
4460 /* Set up the expansion variables for this set of addresses */
4461
4462 deliver_set_expansions(addr);
4463
018c60d7
JH
4464 /* Ensure any transport-set auth info is fresh */
4465 addr->authenticator = addr->auth_id = addr->auth_sndr = NULL;
4466
059ec3d9
PH
4467 /* Compute the return path, expanding a new one if required. The old one
4468 must be set first, as it might be referred to in the expansion. */
4469
d6c82d7b 4470 if(addr->prop.errors_address)
d43cbe25 4471 return_path = addr->prop.errors_address;
384152a6 4472#ifdef EXPERIMENTAL_SRS
d6c82d7b 4473 else if(addr->prop.srs_sender)
d43cbe25 4474 return_path = addr->prop.srs_sender;
384152a6
TK
4475#endif
4476 else
4477 return_path = sender_address;
059ec3d9 4478
d6c82d7b 4479 if (tp->return_path)
059ec3d9
PH
4480 {
4481 uschar *new_return_path = expand_string(tp->return_path);
d6c82d7b
JH
4482 if (new_return_path)
4483 return_path = new_return_path;
8768d548 4484 else if (!f.expand_string_forcedfail)
059ec3d9 4485 {
d6c82d7b
JH
4486 panicmsg = string_sprintf("Failed to expand return path \"%s\": %s",
4487 tp->return_path, expand_string_message);
fa41615d 4488 goto enq_continue;
059ec3d9 4489 }
059ec3d9
PH
4490 }
4491
929ba01c
PH
4492 /* Find the uid, gid, and use_initgroups setting for this transport. Failure
4493 logs and sets up error messages, so we just post-process and continue with
4494 the next address. */
4495
4496 if (!findugid(addr, tp, &uid, &gid, &use_initgroups))
4497 {
d6c82d7b 4498 panicmsg = NULL;
fa41615d 4499 goto enq_continue;
929ba01c
PH
4500 }
4501
059ec3d9
PH
4502 /* If this transport has a setup function, call it now so that it gets
4503 run in this process and not in any subprocess. That way, the results of
929ba01c
PH
4504 any setup that are retained by the transport can be reusable. One of the
4505 things the setup does is to set the fallback host lists in the addresses.
4506 That is why it is called at this point, before the continue delivery
4507 processing, because that might use the fallback hosts. */
059ec3d9 4508
d6c82d7b 4509 if (tp->setup)
929ba01c 4510 (void)((tp->setup)(addr->transport, addr, NULL, uid, gid, NULL));
059ec3d9 4511
57cc2785
JH
4512 /* If we have a connection still open from a verify stage (lazy-close)
4513 treat it as if it is a continued connection (apart from the counter used
4514 for the log line mark). */
4515
74f1a423 4516 if (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only)
57cc2785
JH
4517 {
4518 DEBUG(D_deliver)
4519 debug_printf("lazy-callout-close: have conn still open from verification\n");
4520 continue_transport = cutthrough.transport;
4521 continue_hostname = string_copy(cutthrough.host.name);
4522 continue_host_address = string_copy(cutthrough.host.address);
4523 continue_sequence = 1;
4524 sending_ip_address = cutthrough.snd_ip;
4525 sending_port = cutthrough.snd_port;
4526 smtp_peer_options = cutthrough.peer_options;
4527 }
4528
059ec3d9
PH
4529 /* If this is a run to continue delivery down an already-established
4530 channel, check that this set of addresses matches the transport and
4531 the channel. If it does not, defer the addresses. If a host list exists,
4532 we must check that the continue host is on the list. Otherwise, the
4533 host is set in the transport. */
4534
8768d548 4535 f.continue_more = FALSE; /* In case got set for the last lot */
d6c82d7b 4536 if (continue_transport)
059ec3d9
PH
4537 {
4538 BOOL ok = Ustrcmp(continue_transport, tp->name) == 0;
e39fdc85
JH
4539
4540 /* If the transport is about to override the host list do not check
4541 it here but take the cost of running the transport process to discover
4542 if the continued_hostname connection is suitable. This is a layering
4543 violation which is unfortunate as it requires we haul in the smtp
4544 include file. */
4545
4546 if (ok)
059ec3d9 4547 {
e39fdc85
JH
4548 smtp_transport_options_block * ob;
4549
d7571ac5 4550 if ( !( Ustrcmp(tp->info->driver_name, "smtp") == 0
e39fdc85
JH
4551 && (ob = (smtp_transport_options_block *)tp->options_block)
4552 && ob->hosts_override && ob->hosts
4553 )
4554 && addr->host_list
4555 )
4556 {
4557 host_item * h;
4558 ok = FALSE;
4559 for (h = addr->host_list; h; h = h->next)
4560 if (Ustrcmp(h->name, continue_hostname) == 0)
4561 /*XXX should also check port here */
4562 { ok = TRUE; break; }
4563 }
059ec3d9
PH
4564 }
4565
4566 /* Addresses not suitable; defer or queue for fallback hosts (which
4567 might be the continue host) and skip to next address. */
4568
4569 if (!ok)
4570 {
e47252f5
JH
4571 DEBUG(D_deliver) debug_printf("not suitable for continue_transport (%s)\n",
4572 Ustrcmp(continue_transport, tp->name) != 0
4573 ? string_sprintf("tpt %s vs %s", continue_transport, tp->name)
4574 : string_sprintf("no host matching %s", continue_hostname));
fa41615d 4575 if (serialize_key) enq_end(serialize_key);
059ec3d9 4576
d6c82d7b 4577 if (addr->fallback_hosts && !fallback)
059ec3d9 4578 {
fa41615d 4579 for (next = addr; ; next = next->next)
059ec3d9
PH
4580 {
4581 next->host_list = next->fallback_hosts;
4582 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", next->address);
d6c82d7b 4583 if (!next->next) break;
059ec3d9
PH
4584 }
4585 next->next = addr_fallback;
4586 addr_fallback = addr;
4587 }
4588
e3b1f624 4589 else
3070ceee 4590 {
e3b1f624
JH
4591 for (next = addr; ; next = next->next)
4592 {
4593 DEBUG(D_deliver) debug_printf(" %s to def list\n", next->address);
4594 if (!next->next) break;
4595 }
3070ceee
JH
4596 next->next = addr_defer;
4597 addr_defer = addr;
4598 }
059ec3d9
PH
4599
4600 continue;
4601 }
4602
4603 /* Set a flag indicating whether there are further addresses that list
4604 the continued host. This tells the transport to leave the channel open,
6c140c01
JH
4605 but not to pass it to another delivery process. We'd like to do that
4606 for non-continue_transport cases too but the knowlege of which host is
4607 connected to is too hard to manage. Perhaps we need a finer-grain
4608 interface to the transport. */
059ec3d9 4609
8768d548 4610 for (next = addr_remote; next && !f.continue_more; next = next->next)
059ec3d9
PH
4611 {
4612 host_item *h;
d6c82d7b 4613 for (h = next->host_list; h; h = h->next)
059ec3d9 4614 if (Ustrcmp(h->name, continue_hostname) == 0)
8768d548 4615 { f.continue_more = TRUE; break; }
059ec3d9
PH
4616 }
4617 }
4618
4619 /* The transports set up the process info themselves as they may connect
4620 to more than one remote machine. They also have to set up the filter
4621 arguments, if required, so that the host name and address are available
4622 for expansion. */
4623
4624 transport_filter_argv = NULL;
4625
059ec3d9
PH
4626 /* Create the pipe for inter-process communication. If pipe creation
4627 fails, it is probably because the value of remote_max_parallel is so
4628 large that too many file descriptors for pipes have been created. Arrange
4629 to wait for a process to finish, and then try again. If we still can't
4630 create a pipe when all processes have finished, break the retry loop. */
4631
4632 while (!pipe_done)
4633 {
4634 if (pipe(pfd) == 0) pipe_done = TRUE;
4635 else if (parcount > 0) parmax = parcount;
4636 else break;
4637
4638 /* We need to make the reading end of the pipe non-blocking. There are
4639 two different options for this. Exim is cunningly (I hope!) coded so
4640 that it can use either of them, though it prefers O_NONBLOCK, which
4641 distinguishes between EOF and no-more-data. */
4642
4c0a7a9c
HSHR
4643/* The data appears in a timely manner and we already did a select on
4644all pipes, so I do not see a reason to use non-blocking IO here
4645
a7538db1 4646#ifdef O_NONBLOCK
ff790e47 4647 (void)fcntl(pfd[pipe_read], F_SETFL, O_NONBLOCK);
a7538db1 4648#else
ff790e47 4649 (void)fcntl(pfd[pipe_read], F_SETFL, O_NDELAY);
a7538db1 4650#endif
4c0a7a9c 4651*/
059ec3d9
PH
4652
4653 /* If the maximum number of subprocesses already exist, wait for a process
4654 to finish. If we ran out of file descriptors, parmax will have been reduced
4655 from its initial value of remote_max_parallel. */
4656
4657 par_reduce(parmax - 1, fallback);
4658 }
4659
4660 /* If we failed to create a pipe and there were no processes to wait
4661 for, we have to give up on this one. Do this outside the above loop
4662 so that we can continue the main loop. */
4663
4664 if (!pipe_done)
4665 {
d6c82d7b 4666 panicmsg = string_sprintf("unable to create pipe: %s", strerror(errno));
fa41615d 4667 goto enq_continue;
059ec3d9
PH
4668 }
4669
4670 /* Find a free slot in the pardata list. Must do this after the possible
4671 waiting for processes to finish, because a terminating process will free
4672 up a slot. */
4673
4674 for (poffset = 0; poffset < remote_max_parallel; poffset++)
d6c82d7b
JH
4675 if (parlist[poffset].pid == 0)
4676 break;
059ec3d9
PH
4677
4678 /* If there isn't one, there has been a horrible disaster. */
4679
4680 if (poffset >= remote_max_parallel)
4681 {
f1e894f3
PH
4682 (void)close(pfd[pipe_write]);
4683 (void)close(pfd[pipe_read]);
d6c82d7b 4684 panicmsg = US"Unexpectedly no free subprocess slot";
fa41615d 4685 goto enq_continue;
059ec3d9
PH
4686 }
4687
4688 /* Now fork a subprocess to do the remote delivery, but before doing so,
4c04137d 4689 ensure that any cached resources are released so as not to interfere with
059ec3d9
PH
4690 what happens in the subprocess. */
4691
4692 search_tidyup();
4693
4694 if ((pid = fork()) == 0)
4695 {
4696 int fd = pfd[pipe_write];
4697 host_item *h;
4698
2a47f028
JH
4699 /* Setting this global in the subprocess means we need never clear it */
4700 transport_name = tp->name;
059ec3d9 4701
2a47f028 4702 /* There are weird circumstances in which logging is disabled */
8768d548 4703 f.disable_logging = tp->disable_logging;
059ec3d9
PH
4704
4705 /* Show pids on debug output if parallelism possible */
4706
d6c82d7b 4707 if (parmax > 1 && (parcount > 0 || addr_remote))
059ec3d9
PH
4708 {
4709 DEBUG(D_any|D_v) debug_selector |= D_pid;
4710 DEBUG(D_deliver) debug_printf("Remote delivery process started\n");
4711 }
4712
4713 /* Reset the random number generator, so different processes don't all
4714 have the same sequence. In the test harness we want different, but
4715 predictable settings for each delivery process, so do something explicit
4716 here rather they rely on the fixed reset in the random number function. */
4717
8768d548 4718 random_seed = f.running_in_test_harness ? 42 + 2*delivery_count : 0;
059ec3d9
PH
4719
4720 /* Set close-on-exec on the pipe so that it doesn't get passed on to
4721 a new process that may be forked to do another delivery down the same
4722 SMTP connection. */
4723
ff790e47 4724 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
059ec3d9
PH
4725
4726 /* Close open file descriptors for the pipes of other processes
4727 that are running in parallel. */
4728
4729 for (poffset = 0; poffset < remote_max_parallel; poffset++)
f1e894f3 4730 if (parlist[poffset].pid != 0) (void)close(parlist[poffset].fd);
059ec3d9
PH
4731
4732 /* This process has inherited a copy of the file descriptor
4733 for the data file, but its file pointer is shared with all the
4734 other processes running in parallel. Therefore, we have to re-open
4735 the file in order to get a new file descriptor with its own
4736 file pointer. We don't need to lock it, as the lock is held by
4737 the parent process. There doesn't seem to be any way of doing
4738 a dup-with-new-file-pointer. */
4739
f1e894f3 4740 (void)close(deliver_datafile);
41313d92
JH
4741 {
4742 uschar * fname = spool_fname(US"input", message_subdir, message_id, US"-D");
059ec3d9 4743
92b0827a
JH
4744 if ((deliver_datafile = Uopen(fname,
4745#ifdef O_CLOEXEC
4746 O_CLOEXEC |
4747#endif
4748 O_RDWR | O_APPEND, 0)) < 0)
059ec3d9 4749 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Failed to reopen %s for remote "
41313d92
JH
4750 "parallel delivery: %s", fname, strerror(errno));
4751 }
059ec3d9
PH
4752
4753 /* Set the close-on-exec flag */
92b0827a 4754#ifndef O_CLOEXEC
ff790e47 4755 (void)fcntl(deliver_datafile, F_SETFD, fcntl(deliver_datafile, F_GETFD) |
059ec3d9 4756 FD_CLOEXEC);
92b0827a 4757#endif
059ec3d9
PH
4758
4759 /* Set the uid/gid of this process; bombs out on failure. */
4760
4761 exim_setugid(uid, gid, use_initgroups,
4762 string_sprintf("remote delivery to %s with transport=%s",
4763 addr->address, tp->name));
4764
4765 /* Close the unwanted half of this process' pipe, set the process state,
4766 and run the transport. Afterwards, transport_count will contain the number
4767 of bytes written. */
4768
f1e894f3 4769 (void)close(pfd[pipe_read]);
059ec3d9
PH
4770 set_process_info("delivering %s using %s", message_id, tp->name);
4771 debug_print_string(tp->debug_string);
4772 if (!(tp->info->code)(addr->transport, addr)) replicate_status(addr);
4773
4774 set_process_info("delivering %s (just run %s for %s%s in subprocess)",
d6c82d7b 4775 message_id, tp->name, addr->address, addr->next ? ", ..." : "");
059ec3d9
PH
4776
4777 /* Ensure any cached resources that we used are now released */
4778
4779 search_tidyup();
4780
4781 /* Pass the result back down the pipe. This is a lot more information
4782 than is needed for a local delivery. We have to send back the error
4783 status for each address, the usability status for each host that is
4784 flagged as unusable, and all the retry items. When TLS is in use, we
4785 send also the cipher and peerdn information. Each type of information
4786 is flagged by an identifying byte, and is then in a fixed format (with
4787 strings terminated by zeros), and there is a final terminator at the
4788 end. The host information and retry information is all attached to
4789 the first address, so that gets sent at the start. */
4790
4791 /* Host unusability information: for most success cases this will
4792 be null. */
4793
d6c82d7b 4794 for (h = addr->host_list; h; h = h->next)
059ec3d9 4795 {
d6c82d7b 4796 if (!h->address || h->status < hstatus_unusable) continue;
bd21a787
WB
4797 sprintf(CS big_buffer, "%c%c%s", h->status, h->why, h->address);
4798 rmt_dlv_checked_write(fd, 'H', '0', big_buffer, Ustrlen(big_buffer+2) + 3);
059ec3d9
PH
4799 }
4800
4801 /* The number of bytes written. This is the same for each address. Even
4802 if we sent several copies of the message down the same connection, the
4803 size of each one is the same, and it's that value we have got because
4804 transport_count gets reset before calling transport_write_message(). */
4805
bd21a787
WB
4806 memcpy(big_buffer, &transport_count, sizeof(transport_count));
4807 rmt_dlv_checked_write(fd, 'S', '0', big_buffer, sizeof(transport_count));
059ec3d9 4808
6f123593
JH
4809 /* Information about what happened to each address. Four item types are
4810 used: an optional 'X' item first, for TLS information, then an optional "C"
4811 item for any client-auth info followed by 'R' items for any retry settings,
4812 and finally an 'A' item for the remaining data. */
059ec3d9 4813
d6c82d7b 4814 for(; addr; addr = addr->next)
059ec3d9
PH
4815 {
4816 uschar *ptr;
4817 retry_item *r;
4818
4819 /* The certificate verification status goes into the flags */
817d9f57 4820 if (tls_out.certificate_verified) setflag(addr, af_cert_verified);
c0635b6d 4821#ifdef SUPPORT_DANE
53a7196b 4822 if (tls_out.dane_verified) setflag(addr, af_dane_verified);
a0a2d1ee 4823#endif
059ec3d9
PH
4824
4825 /* Use an X item only if there's something to send */
a7538db1 4826#ifdef SUPPORT_TLS
9d1c15ef 4827 if (addr->cipher)
059ec3d9 4828 {
e6d2a989 4829 ptr = big_buffer + sprintf(CS big_buffer, "%.128s", addr->cipher) + 1;
9d1c15ef
JH
4830 if (!addr->peerdn)
4831 *ptr++ = 0;
4832 else
5e6d12ac 4833 ptr += sprintf(CS ptr, "%.512s", addr->peerdn) + 1;
9d1c15ef 4834
bd21a787 4835 rmt_dlv_checked_write(fd, 'X', '1', big_buffer, ptr - big_buffer);
059ec3d9 4836 }
5013d912
JH
4837 else if (continue_proxy_cipher)
4838 {
4839 ptr = big_buffer + sprintf(CS big_buffer, "%.128s", continue_proxy_cipher) + 1;
4840 *ptr++ = 0;
4841 rmt_dlv_checked_write(fd, 'X', '1', big_buffer, ptr - big_buffer);
4842 }
5e6d12ac 4843
9d1c15ef
JH
4844 if (addr->peercert)
4845 {
4846 ptr = big_buffer;
9d1c15ef
JH
4847 if (!tls_export_cert(ptr, big_buffer_size-2, addr->peercert))
4848 while(*ptr++);
4849 else
4850 *ptr++ = 0;
bd21a787 4851 rmt_dlv_checked_write(fd, 'X', '2', big_buffer, ptr - big_buffer);
9d1c15ef
JH
4852 }
4853 if (addr->ourcert)
4854 {
4855 ptr = big_buffer;
9d1c15ef
JH
4856 if (!tls_export_cert(ptr, big_buffer_size-2, addr->ourcert))
4857 while(*ptr++);
4858 else
4859 *ptr++ = 0;
bd21a787 4860 rmt_dlv_checked_write(fd, 'X', '3', big_buffer, ptr - big_buffer);
9d1c15ef 4861 }
a7538db1 4862# ifndef DISABLE_OCSP
018058b2
JH
4863 if (addr->ocsp > OCSP_NOT_REQ)
4864 {
e6d2a989 4865 ptr = big_buffer + sprintf(CS big_buffer, "%c", addr->ocsp + '0') + 1;
bd21a787 4866 rmt_dlv_checked_write(fd, 'X', '4', big_buffer, ptr - big_buffer);
018058b2 4867 }
a7538db1
JH
4868# endif
4869#endif /*SUPPORT_TLS*/
059ec3d9 4870
6f123593
JH
4871 if (client_authenticator)
4872 {
e6d2a989 4873 ptr = big_buffer + sprintf(CS big_buffer, "%.64s", client_authenticator) + 1;
bd21a787 4874 rmt_dlv_checked_write(fd, 'C', '1', big_buffer, ptr - big_buffer);
6f123593
JH
4875 }
4876 if (client_authenticated_id)
4877 {
e6d2a989 4878 ptr = big_buffer + sprintf(CS big_buffer, "%.64s", client_authenticated_id) + 1;
bd21a787 4879 rmt_dlv_checked_write(fd, 'C', '2', big_buffer, ptr - big_buffer);
6f123593 4880 }
c8e2fc1e
JH
4881 if (client_authenticated_sender)
4882 {
e6d2a989 4883 ptr = big_buffer + sprintf(CS big_buffer, "%.64s", client_authenticated_sender) + 1;
bd21a787 4884 rmt_dlv_checked_write(fd, 'C', '3', big_buffer, ptr - big_buffer);
c8e2fc1e 4885 }
6f123593 4886
a7538db1 4887#ifndef DISABLE_PRDR
7eb0e5d2 4888 if (testflag(addr, af_prdr_used))
bd21a787 4889 rmt_dlv_checked_write(fd, 'P', '0', NULL, 0);
b75960c8 4890#endif
fd98a5c6 4891
7c576fca 4892 if (testflag(addr, af_pipelining))
ee8b8090
JH
4893#ifdef EXPERIMENTAL_PIPE_CONNECT
4894 if (testflag(addr, af_early_pipe))
4895 rmt_dlv_checked_write(fd, 'L', '2', NULL, 0);
4896 else
4897#endif
7c576fca
JH
4898 rmt_dlv_checked_write(fd, 'L', '1', NULL, 0);
4899
7eb0e5d2 4900 if (testflag(addr, af_chunking_used))
e0cc6cda
JH
4901 rmt_dlv_checked_write(fd, 'K', '0', NULL, 0);
4902
1ccd5f67
JH
4903 if (testflag(addr, af_tcp_fastopen_conn))
4904 rmt_dlv_checked_write(fd, 'T',
ee8b8090
JH
4905 testflag(addr, af_tcp_fastopen) ? testflag(addr, af_tcp_fastopen_data)
4906 ? '2' : '1' : '0',
4907 NULL, 0);
a2673768 4908
bd21a787
WB
4909 memcpy(big_buffer, &addr->dsn_aware, sizeof(addr->dsn_aware));
4910 rmt_dlv_checked_write(fd, 'D', '0', big_buffer, sizeof(addr->dsn_aware));
6c1c3d1d 4911
059ec3d9
PH
4912 /* Retry information: for most success cases this will be null. */
4913
d6c82d7b 4914 for (r = addr->retries; r; r = r->next)
059ec3d9 4915 {
bd21a787 4916 sprintf(CS big_buffer, "%c%.500s", r->flags, r->key);
059ec3d9 4917 ptr = big_buffer + Ustrlen(big_buffer+2) + 3;
32dfdf8b 4918 memcpy(ptr, &r->basic_errno, sizeof(r->basic_errno));
059ec3d9 4919 ptr += sizeof(r->basic_errno);
32dfdf8b 4920 memcpy(ptr, &r->more_errno, sizeof(r->more_errno));
059ec3d9 4921 ptr += sizeof(r->more_errno);
d6c82d7b 4922 if (!r->message) *ptr++ = 0; else
059ec3d9
PH
4923 {
4924 sprintf(CS ptr, "%.512s", r->message);
4925 while(*ptr++);
4926 }
bd21a787 4927 rmt_dlv_checked_write(fd, 'R', '0', big_buffer, ptr - big_buffer);
059ec3d9
PH
4928 }
4929
e6d2a989
JH
4930#ifdef SUPPORT_SOCKS
4931 if (LOGGING(proxy) && proxy_session)
4932 {
4933 ptr = big_buffer;
4934 if (proxy_local_address)
4935 {
4936 DEBUG(D_deliver) debug_printf("proxy_local_address '%s'\n", proxy_local_address);
4937 ptr = big_buffer + sprintf(CS ptr, "%.128s", proxy_local_address) + 1;
4938 DEBUG(D_deliver) debug_printf("proxy_local_port %d\n", proxy_local_port);
4939 memcpy(ptr, &proxy_local_port, sizeof(proxy_local_port));
4940 ptr += sizeof(proxy_local_port);
4941 }
4942 else
4943 *ptr++ = '\0';
4944 rmt_dlv_checked_write(fd, 'A', '2', big_buffer, ptr - big_buffer);
4945 }
4946#endif
4947
895fbaf2
JH
4948#ifdef EXPERIMENTAL_DSN_INFO
4949/*um, are they really per-addr? Other per-conn stuff is not (auth, tls). But host_used is! */
4950 if (addr->smtp_greeting)
4951 {
895fbaf2 4952 DEBUG(D_deliver) debug_printf("smtp_greeting '%s'\n", addr->smtp_greeting);
e6d2a989 4953 ptr = big_buffer + sprintf(CS big_buffer, "%.128s", addr->smtp_greeting) + 1;
895fbaf2
JH
4954 if (addr->helo_response)
4955 {
4956 DEBUG(D_deliver) debug_printf("helo_response '%s'\n", addr->helo_response);
e6d2a989 4957 ptr += sprintf(CS ptr, "%.128s", addr->helo_response) + 1;
895fbaf2
JH
4958 }
4959 else
4960 *ptr++ = '\0';
4961 rmt_dlv_checked_write(fd, 'A', '1', big_buffer, ptr - big_buffer);
4962 }
4963#endif
4964
4965 /* The rest of the information goes in an 'A0' item. */
059ec3d9 4966
e6d2a989 4967 sprintf(CS big_buffer, "%c%c", addr->transport_return, addr->special_action);
895fbaf2 4968 ptr = big_buffer + 2;
32dfdf8b 4969 memcpy(ptr, &addr->basic_errno, sizeof(addr->basic_errno));
059ec3d9 4970 ptr += sizeof(addr->basic_errno);
32dfdf8b 4971 memcpy(ptr, &addr->more_errno, sizeof(addr->more_errno));
059ec3d9 4972 ptr += sizeof(addr->more_errno);
32dfdf8b
JH
4973 memcpy(ptr, &addr->delivery_usec, sizeof(addr->delivery_usec));
4974 ptr += sizeof(addr->delivery_usec);
4975 memcpy(ptr, &addr->flags, sizeof(addr->flags));
059ec3d9
PH
4976 ptr += sizeof(addr->flags);
4977
d6c82d7b 4978 if (!addr->message) *ptr++ = 0; else
e6d2a989 4979 ptr += sprintf(CS ptr, "%.1024s", addr->message) + 1;
059ec3d9 4980
d6c82d7b 4981 if (!addr->user_message) *ptr++ = 0; else
e6d2a989 4982 ptr += sprintf(CS ptr, "%.1024s", addr->user_message) + 1;
059ec3d9 4983
d6c82d7b 4984 if (!addr->host_used) *ptr++ = 0; else
059ec3d9 4985 {
e6d2a989
JH
4986 ptr += sprintf(CS ptr, "%.256s", addr->host_used->name) + 1;
4987 ptr += sprintf(CS ptr, "%.64s", addr->host_used->address) + 1;
32dfdf8b 4988 memcpy(ptr, &addr->host_used->port, sizeof(addr->host_used->port));
059ec3d9 4989 ptr += sizeof(addr->host_used->port);
783b385f
JH
4990
4991 /* DNS lookup status */
4992 *ptr++ = addr->host_used->dnssec==DS_YES ? '2'
4993 : addr->host_used->dnssec==DS_NO ? '1' : '0';
4994
059ec3d9 4995 }
bd21a787 4996 rmt_dlv_checked_write(fd, 'A', '0', big_buffer, ptr - big_buffer);
059ec3d9
PH
4997 }
4998
aeaf5db3 4999 /* Local interface address/port */
895fbaf2
JH
5000#ifdef EXPERIMENTAL_DSN_INFO
5001 if (sending_ip_address)
5002#else
6c6d6e48 5003 if (LOGGING(incoming_interface) && sending_ip_address)
895fbaf2 5004#endif
aeaf5db3 5005 {
e6d2a989
JH
5006 uschar * ptr;
5007 ptr = big_buffer + sprintf(CS big_buffer, "%.128s", sending_ip_address) + 1;
5008 ptr += sprintf(CS ptr, "%d", sending_port) + 1;
aeaf5db3
JH
5009 rmt_dlv_checked_write(fd, 'I', '0', big_buffer, ptr - big_buffer);
5010 }
5011
059ec3d9
PH
5012 /* Add termination flag, close the pipe, and that's it. The character
5013 after 'Z' indicates whether continue_transport is now NULL or not.
5014 A change from non-NULL to NULL indicates a problem with a continuing
5015 connection. */
5016
d6c82d7b 5017 big_buffer[0] = continue_transport ? '1' : '0';
bd21a787 5018 rmt_dlv_checked_write(fd, 'Z', '0', big_buffer, 1);
f1e894f3 5019 (void)close(fd);
059ec3d9
PH
5020 exit(EXIT_SUCCESS);
5021 }
5022
5023 /* Back in the mainline: close the unwanted half of the pipe. */
5024
f1e894f3 5025 (void)close(pfd[pipe_write]);
059ec3d9 5026
57cc2785
JH
5027 /* If we have a connection still open from a verify stage (lazy-close)
5028 release its TLS library context (if any) as responsibility was passed to
5029 the delivery child process. */
5030
74f1a423 5031 if (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only)
57cc2785
JH
5032 {
5033#ifdef SUPPORT_TLS
74f1a423
JH
5034 if (cutthrough.is_tls)
5035 tls_close(cutthrough.cctx.tls_ctx, TLS_NO_SHUTDOWN);
57cc2785 5036#endif
74f1a423 5037 (void) close(cutthrough.cctx.sock);
57cc2785
JH
5038 release_cutthrough_connection(US"passed to transport proc");
5039 }
5040
059ec3d9
PH
5041 /* Fork failed; defer with error message */
5042
2cee425a 5043 if (pid == -1)
059ec3d9 5044 {
f1e894f3 5045 (void)close(pfd[pipe_read]);
d6c82d7b
JH
5046 panicmsg = string_sprintf("fork failed for remote delivery to %s: %s",
5047 addr->domain, strerror(errno));
fa41615d 5048 goto enq_continue;
059ec3d9
PH
5049 }
5050
5051 /* Fork succeeded; increment the count, and remember relevant data for
5052 when the process finishes. */
5053
5054 parcount++;
5055 parlist[poffset].addrlist = parlist[poffset].addr = addr;
5056 parlist[poffset].pid = pid;
5057 parlist[poffset].fd = pfd[pipe_read];
5058 parlist[poffset].done = FALSE;
5059 parlist[poffset].msg = NULL;
5060 parlist[poffset].return_path = return_path;
5061
5062 /* If the process we've just started is sending a message down an existing
5063 channel, wait for it now. This ensures that only one such process runs at
5064 once, whatever the value of remote_max parallel. Otherwise, we might try to
5065 send two or more messages simultaneously down the same channel. This could
5066 happen if there are different domains that include the same host in otherwise
5067 different host lists.
5068
5069 Also, if the transport closes down the channel, this information gets back
5070 (continue_transport gets set to NULL) before we consider any other addresses
5071 in this message. */
5072
d6c82d7b 5073 if (continue_transport) par_reduce(0, fallback);
059ec3d9
PH
5074
5075 /* Otherwise, if we are running in the test harness, wait a bit, to let the
5076 newly created process get going before we create another process. This should
5077 ensure repeatability in the tests. We only need to wait a tad. */
5078
8768d548 5079 else if (f.running_in_test_harness) millisleep(500);
d6c82d7b
JH
5080
5081 continue;
5082
fa41615d
JH
5083enq_continue:
5084 if (serialize_key) enq_end(serialize_key);
d6c82d7b
JH
5085panic_continue:
5086 remote_post_process(addr, LOG_MAIN|LOG_PANIC, panicmsg, fallback);
5087 continue;
059ec3d9
PH
5088 }
5089
5090/* Reached the end of the list of addresses. Wait for all the subprocesses that
5091are still running and post-process their addresses. */
5092
5093par_reduce(0, fallback);
5094return TRUE;
5095}
5096
5097
5098
5099
5100/*************************************************
5101* Split an address into local part and domain *
5102*************************************************/
5103
5104/* This function initializes an address for routing by splitting it up into a
5105local part and a domain. The local part is set up twice - once in its original
5106casing, and once in lower case, and it is dequoted. We also do the "percent
5107hack" for configured domains. This may lead to a DEFER result if a lookup
5108defers. When a percent-hacking takes place, we insert a copy of the original
5109address as a new parent of this address, as if we have had a redirection.
5110
5111Argument:
5112 addr points to an addr_item block containing the address
5113
5114Returns: OK
5115 DEFER - could not determine if domain is %-hackable
5116*/
5117
5118int
fb5908b1 5119deliver_split_address(address_item * addr)
059ec3d9 5120{
fb5908b1
JH
5121uschar * address = addr->address;
5122uschar * domain;
5123uschar * t;
5124int len;
059ec3d9 5125
fb5908b1
JH
5126if (!(domain = Ustrrchr(address, '@')))
5127 return DEFER; /* should always have a domain, but just in case... */
5128
5129len = domain - address;
059ec3d9
PH
5130addr->domain = string_copylc(domain+1); /* Domains are always caseless */
5131
5132/* The implication in the RFCs (though I can't say I've seen it spelled out
5133explicitly) is that quoting should be removed from local parts at the point
5134where they are locally interpreted. [The new draft "821" is more explicit on
5135this, Jan 1999.] We know the syntax is valid, so this can be done by simply
5136removing quoting backslashes and any unquoted doublequotes. */
5137
5138t = addr->cc_local_part = store_get(len+1);
5139while(len-- > 0)
5140 {
fb5908b1 5141 int c = *address++;
059ec3d9
PH
5142 if (c == '\"') continue;
5143 if (c == '\\')
5144 {
5145 *t++ = *address++;
5146 len--;
5147 }
5148 else *t++ = c;
5149 }
5150*t = 0;
5151
5152/* We do the percent hack only for those domains that are listed in
5153percent_hack_domains. A loop is required, to copy with multiple %-hacks. */
5154
d6c82d7b 5155if (percent_hack_domains)
059ec3d9
PH
5156 {
5157 int rc;
5158 uschar *new_address = NULL;
5159 uschar *local_part = addr->cc_local_part;
5160
5161 deliver_domain = addr->domain; /* set $domain */
5162
d6c82d7b
JH
5163 while ( (rc = match_isinlist(deliver_domain, (const uschar **)&percent_hack_domains, 0,
5164 &domainlist_anchor, addr->domain_cache, MCL_DOMAIN, TRUE, NULL))
5165 == OK
5166 && (t = Ustrrchr(local_part, '%')) != NULL
5167 )
059ec3d9
PH
5168 {
5169 new_address = string_copy(local_part);
5170 new_address[t - local_part] = '@';
5171 deliver_domain = string_copylc(t+1);
5172 local_part = string_copyn(local_part, t - local_part);
5173 }
5174
5175 if (rc == DEFER) return DEFER; /* lookup deferred */
5176
5177 /* If hackery happened, set up new parent and alter the current address. */
5178
d6c82d7b 5179 if (new_address)
059ec3d9
PH
5180 {
5181 address_item *new_parent = store_get(sizeof(address_item));
5182 *new_parent = *addr;
5183 addr->parent = new_parent;
82f90600 5184 new_parent->child_count = 1;
059ec3d9
PH
5185 addr->address = new_address;
5186 addr->unique = string_copy(new_address);
5187 addr->domain = deliver_domain;
5188 addr->cc_local_part = local_part;
5189 DEBUG(D_deliver) debug_printf("%%-hack changed address to: %s\n",
5190 addr->address);
5191 }
5192 }
5193
5194/* Create the lowercased version of the final local part, and make that the
5195default one to be used. */
5196
5197addr->local_part = addr->lc_local_part = string_copylc(addr->cc_local_part);
5198return OK;
5199}
5200
5201
5202
5203
5204/*************************************************
5205* Get next error message text *
5206*************************************************/
5207
5208/* If f is not NULL, read the next "paragraph", from a customized error message
5209text file, terminated by a line containing ****, and expand it.
5210
5211Arguments:
5212 f NULL or a file to read from
5213 which string indicating which string (for errors)
5214
5215Returns: NULL or an expanded string
5216*/
5217
5218static uschar *
5219next_emf(FILE *f, uschar *which)
5220{
acec9514
JH
5221uschar *yield;
5222gstring * para;
059ec3d9
PH
5223uschar buffer[256];
5224
d6c82d7b 5225if (!f) return NULL;
059ec3d9 5226
d6c82d7b
JH
5227if (!Ufgets(buffer, sizeof(buffer), f) || Ustrcmp(buffer, "****\n") == 0)
5228 return NULL;
059ec3d9 5229
acec9514 5230para = string_get(256);
059ec3d9
PH
5231for (;;)
5232 {
acec9514 5233 para = string_cat(para, buffer);
d6c82d7b
JH
5234 if (!Ufgets(buffer, sizeof(buffer), f) || Ustrcmp(buffer, "****\n") == 0)
5235 break;
059ec3d9 5236 }
acec9514 5237if ((yield = expand_string(string_from_gstring(para))))
d6c82d7b 5238 return yield;
059ec3d9
PH
5239
5240log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand string from "
5241 "bounce_message_file or warn_message_file (%s): %s", which,
5242 expand_string_message);
5243return NULL;
5244}
5245
5246
5247
5248
5249/*************************************************
5250* Close down a passed transport channel *
5251*************************************************/
5252
5253/* This function is called when a passed transport channel cannot be used.
5254It attempts to close it down tidily. The yield is always DELIVER_NOT_ATTEMPTED
5255so that the function call can be the argument of a "return" statement.
5256
5257Arguments: None
5258Returns: DELIVER_NOT_ATTEMPTED
5259*/
5260
5261static int
5262continue_closedown(void)
5263{
d6c82d7b 5264if (continue_transport)
059ec3d9
PH
5265 {
5266 transport_instance *t;
d6c82d7b 5267 for (t = transports; t; t = t->next)
059ec3d9
PH
5268 if (Ustrcmp(t->name, continue_transport) == 0)
5269 {
d6c82d7b 5270 if (t->info->closedown) (t->info->closedown)(t);
059ec3d9
PH
5271 break;
5272 }
059ec3d9
PH
5273 }
5274return DELIVER_NOT_ATTEMPTED;
5275}
5276
5277
5278
5279
5280/*************************************************
5281* Print address information *
5282*************************************************/
5283
5284/* This function is called to output an address, or information about an
5285address, for bounce or defer messages. If the hide_child flag is set, all we
5286output is the original ancestor address.
5287
5288Arguments:
5289 addr points to the address
5290 f the FILE to print to
5291 si an initial string
5292 sc a continuation string for before "generated"
5293 se an end string
5294
5295Returns: TRUE if the address is not hidden
5296*/
5297
5298static BOOL
5299print_address_information(address_item *addr, FILE *f, uschar *si, uschar *sc,
5300 uschar *se)
5301{
5302BOOL yield = TRUE;
5303uschar *printed = US"";
5304address_item *ancestor = addr;
d6c82d7b 5305while (ancestor->parent) ancestor = ancestor->parent;
059ec3d9
PH
5306
5307fprintf(f, "%s", CS si);
5308
d6c82d7b 5309if (addr->parent && testflag(addr, af_hide_child))
059ec3d9
PH
5310 {
5311 printed = US"an undisclosed address";
5312 yield = FALSE;
5313 }
d6c82d7b 5314else if (!testflag(addr, af_pfr) || !addr->parent)
059ec3d9
PH
5315 printed = addr->address;
5316
5317else
5318 {
5319 uschar *s = addr->address;
5320 uschar *ss;
5321
5322 if (addr->address[0] == '>') { ss = US"mail"; s++; }
5323 else if (addr->address[0] == '|') ss = US"pipe";
5324 else ss = US"save";
5325
5326 fprintf(f, "%s to %s%sgenerated by ", ss, s, sc);
5327 printed = addr->parent->address;
5328 }
5329
5330fprintf(f, "%s", CS string_printing(printed));
5331
5332if (ancestor != addr)
5333 {
d6c82d7b
JH
5334 uschar *original = ancestor->onetime_parent;
5335 if (!original) original= ancestor->address;
059ec3d9
PH
5336 if (strcmpic(original, printed) != 0)
5337 fprintf(f, "%s(%sgenerated from %s)", sc,
d6c82d7b 5338 ancestor != addr->parent ? "ultimately " : "",
059ec3d9
PH
5339 string_printing(original));
5340 }
5341
c562fd30
JH
5342if (addr->host_used)
5343 fprintf(f, "\n host %s [%s]",
5344 addr->host_used->name, addr->host_used->address);
5345
059ec3d9
PH
5346fprintf(f, "%s", CS se);
5347return yield;
5348}
5349
5350
5351
5352
5353
059ec3d9
PH
5354/*************************************************
5355* Print error for an address *
5356*************************************************/
5357
5358/* This function is called to print the error information out of an address for
5359a bounce or a warning message. It tries to format the message reasonably by
5360introducing newlines. All lines are indented by 4; the initial printing
5361position must be set before calling.
5362
447d236c 5363This function used always to print the error. Nowadays we want to restrict it
75def545
PH
5364to cases such as LMTP/SMTP errors from a remote host, and errors from :fail:
5365and filter "fail". We no longer pass other information willy-nilly in bounce
5366and warning messages. Text in user_message is always output; text in message
5367only if the af_pass_message flag is set.
447d236c 5368
059ec3d9 5369Arguments:
447d236c 5370 addr the address
059ec3d9 5371 f the FILE to print on
75def545 5372 t some leading text
059ec3d9
PH
5373
5374Returns: nothing
5375*/
5376
5377static void
447d236c 5378print_address_error(address_item *addr, FILE *f, uschar *t)
059ec3d9 5379{
447d236c 5380int count = Ustrlen(t);
de6f74f2 5381uschar *s = testflag(addr, af_pass_message) ? addr->message : NULL;
447d236c 5382
0ba0ee97
JH
5383if (!s && !(s = addr->user_message))
5384 return;
447d236c
PH
5385
5386fprintf(f, "\n %s", t);
5387
0ba0ee97 5388while (*s)
447d236c
PH
5389 if (*s == '\\' && s[1] == 'n')
5390 {
5391 fprintf(f, "\n ");
5392 s += 2;
5393 count = 0;
5394 }
5395 else
059ec3d9 5396 {
447d236c
PH
5397 fputc(*s, f);
5398 count++;
5399 if (*s++ == ':' && isspace(*s) && count > 45)
059ec3d9 5400 {
447d236c 5401 fprintf(f, "\n "); /* sic (because space follows) */
059ec3d9
PH
5402 count = 0;
5403 }
059ec3d9 5404 }
059ec3d9
PH
5405}
5406
5407
21bc4865
WB
5408/***********************************************************
5409* Print Diagnostic-Code for an address *
5410************************************************************/
5411
5412/* This function is called to print the error information out of an address for
5413a bounce or a warning message. It tries to format the message reasonably as
5414required by RFC 3461 by adding a space after each newline
5415
ca410e79
WB
5416it uses the same logic as print_address_error() above. if af_pass_message is true
5417and addr->message is set it uses the remote host answer. if not addr->user_message
5418is used instead if available.
21bc4865
WB
5419
5420Arguments:
5421 addr the address
5422 f the FILE to print on
5423
5424Returns: nothing
5425*/
5426
5427static void
5428print_dsn_diagnostic_code(const address_item *addr, FILE *f)
5429{
ca410e79 5430uschar *s = testflag(addr, af_pass_message) ? addr->message : NULL;
21bc4865 5431
ca410e79
WB
5432/* af_pass_message and addr->message set ? print remote host answer */
5433if (s)
5434 {
5435 DEBUG(D_deliver)
5436 debug_printf("DSN Diagnostic-Code: addr->message = %s\n", addr->message);
447d236c 5437
ca410e79
WB
5438 /* search first ": ". we assume to find the remote-MTA answer there */
5439 if (!(s = Ustrstr(addr->message, ": ")))
5440 return; /* not found, bail out */
5441 s += 2; /* skip ": " */
5442 fprintf(f, "Diagnostic-Code: smtp; ");
5443 }
ca410e79 5444/* no message available. do nothing */
9031f51a 5445else return;
21bc4865 5446
21bc4865
WB
5447while (*s)
5448 if (*s == '\\' && s[1] == 'n')
5449 {
5450 fputs("\n ", f); /* as defined in RFC 3461 */
5451 s += 2;
5452 }
5453 else
5454 fputc(*s++, f);
5455
5456fputc('\n', f);
5457}
447d236c
PH
5458
5459
d7174846
PH
5460/*************************************************
5461* Check list of addresses for duplication *
5462*************************************************/
5463
5464/* This function was introduced when the test for duplicate addresses that are
5465not pipes, files, or autoreplies was moved from the middle of routing to when
5466routing was complete. That was to fix obscure cases when the routing history
dbcef0ea
PH
5467affects the subsequent routing of identical addresses. This function is called
5468after routing, to check that the final routed addresses are not duplicates.
d7174846 5469
dbcef0ea
PH
5470If we detect a duplicate, we remember what it is a duplicate of. Note that
5471pipe, file, and autoreply de-duplication is handled during routing, so we must
5472leave such "addresses" alone here, as otherwise they will incorrectly be
5473discarded.
d7174846
PH
5474
5475Argument: address of list anchor
5476Returns: nothing
5477*/
5478
5479static void
5480do_duplicate_check(address_item **anchor)
5481{
5482address_item *addr;
d6c82d7b 5483while ((addr = *anchor))
d7174846
PH
5484 {
5485 tree_node *tnode;
5486 if (testflag(addr, af_pfr))
5487 {
5488 anchor = &(addr->next);
5489 }
d6c82d7b 5490 else if ((tnode = tree_search(tree_duplicates, addr->unique)))
d7174846
PH
5491 {
5492 DEBUG(D_deliver|D_route)
5493 debug_printf("%s is a duplicate address: discarded\n", addr->unique);
5494 *anchor = addr->next;
5495 addr->dupof = tnode->data.ptr;
5496 addr->next = addr_duplicate;
5497 addr_duplicate = addr;
5498 }
5499 else
5500 {
5501 tree_add_duplicate(addr->unique, addr);
5502 anchor = &(addr->next);
5503 }
5504 }
5505}
5506
5507
5508
059ec3d9
PH
5509
5510/*************************************************
5511* Deliver one message *
5512*************************************************/
5513
5514/* This is the function which is called when a message is to be delivered. It
5515is passed the id of the message. It is possible that the message no longer
5516exists, if some other process has delivered it, and it is also possible that
5517the message is being worked on by another process, in which case the data file
5518will be locked.
5519
5520If no delivery is attempted for any of the above reasons, the function returns
5521DELIVER_NOT_ATTEMPTED.
5522
5523If the give_up flag is set true, do not attempt any deliveries, but instead
5524fail all outstanding addresses and return the message to the sender (or
5525whoever).
5526
5527A delivery operation has a process all to itself; we never deliver more than
5528one message in the same process. Therefore we needn't worry too much about
5529store leakage.
5530
92b0827a
JH
5531Liable to be called as root.
5532
059ec3d9
PH
5533Arguments:
5534 id the id of the message to be delivered
5535 forced TRUE if delivery was forced by an administrator; this overrides
5536 retry delays and causes a delivery to be tried regardless
5537 give_up TRUE if an administrator has requested that delivery attempts
5538 be abandoned
5539
5540Returns: When the global variable mua_wrapper is FALSE:
5541 DELIVER_ATTEMPTED_NORMAL if a delivery attempt was made
5542 DELIVER_NOT_ATTEMPTED otherwise (see comment above)
5543 When the global variable mua_wrapper is TRUE:
5544 DELIVER_MUA_SUCCEEDED if delivery succeeded
5545 DELIVER_MUA_FAILED if delivery failed
5546 DELIVER_NOT_ATTEMPTED if not attempted (should not occur)
5547*/
5548
5549int
5550deliver_message(uschar *id, BOOL forced, BOOL give_up)
5551{
5552int i, rc;
5553int final_yield = DELIVER_ATTEMPTED_NORMAL;
5554time_t now = time(NULL);
5555address_item *addr_last = NULL;
5556uschar *filter_message = NULL;
059ec3d9
PH
5557int process_recipients = RECIP_ACCEPT;
5558open_db dbblock;
5559open_db *dbm_file;
faa05a93 5560extern int acl_where;
059ec3d9 5561
d6c82d7b
JH
5562uschar *info = queue_run_pid == (pid_t)0
5563 ? string_sprintf("delivering %s", id)
5564 : string_sprintf("delivering %s (queue run pid %d)", id, queue_run_pid);
059ec3d9
PH
5565
5566/* If the D_process_info bit is on, set_process_info() will output debugging
5567information. If not, we want to show this initial information if D_deliver or
5568D_queue_run is set or in verbose mode. */
5569
5570set_process_info("%s", info);
5571
d6c82d7b
JH
5572if ( !(debug_selector & D_process_info)
5573 && (debug_selector & (D_deliver|D_queue_run|D_v))
5574 )
059ec3d9
PH
5575 debug_printf("%s\n", info);
5576
5577/* Ensure that we catch any subprocesses that are created. Although Exim
5578sets SIG_DFL as its initial default, some routes through the code end up
5579here with it set to SIG_IGN - cases where a non-synchronous delivery process
5580has been forked, but no re-exec has been done. We use sigaction rather than
5581plain signal() on those OS where SA_NOCLDWAIT exists, because we want to be
5582sure it is turned off. (There was a problem on AIX with this.) */
5583
5584#ifdef SA_NOCLDWAIT
5585 {
5586 struct sigaction act;
5587 act.sa_handler = SIG_DFL;
5588 sigemptyset(&(act.sa_mask));
5589 act.sa_flags = 0;
5590 sigaction(SIGCHLD, &act, NULL);
5591 }
5592#else
5593signal(SIGCHLD, SIG_DFL);
5594#endif
5595
5596/* Make the forcing flag available for routers and transports, set up the
5597global message id field, and initialize the count for returned files and the
5598message size. This use of strcpy() is OK because the length id is checked when
5599it is obtained from a command line (the -M or -q options), and otherwise it is
5600known to be a valid message id. */
5601
e30f4f43
JH
5602if (id != message_id)
5603 Ustrcpy(message_id, id);
8768d548 5604f.deliver_force = forced;
059ec3d9
PH
5605return_count = 0;
5606message_size = 0;
5607
5608/* Initialize some flags */
5609
5610update_spool = FALSE;
5611remove_journal = TRUE;
5612
faa05a93
JH
5613/* Set a known context for any ACLs we call via expansions */
5614acl_where = ACL_WHERE_DELIVERY;
5615
059ec3d9
PH
5616/* Reset the random number generator, so that if several delivery processes are
5617started from a queue runner that has already used random numbers (for sorting),
5618they don't all get the same sequence. */
5619
5620random_seed = 0;
5621
5622/* Open and lock the message's data file. Exim locks on this one because the
5623header file may get replaced as it is re-written during the delivery process.
5624Any failures cause messages to be written to the log, except for missing files
5625while queue running - another process probably completed delivery. As part of
5626opening the data file, message_subdir gets set. */
5627
789f8a4f 5628if ((deliver_datafile = spool_open_datafile(id)) < 0)
059ec3d9
PH
5629 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5630
5631/* The value of message_size at this point has been set to the data length,
5632plus one for the blank line that notionally precedes the data. */
5633
5634/* Now read the contents of the header file, which will set up the headers in
5635store, and also the list of recipients and the tree of non-recipients and
5636assorted flags. It updates message_size. If there is a reading or format error,
5637give up; if the message has been around for sufficiently long, remove it. */
5638
059ec3d9 5639 {
41313d92
JH
5640 uschar * spoolname = string_sprintf("%s-H", id);
5641 if ((rc = spool_read_header(spoolname, TRUE, TRUE)) != spool_read_OK)
059ec3d9 5642 {
41313d92
JH
5643 if (errno == ERRNO_SPOOLFORMAT)
5644 {
5645 struct stat statbuf;
5646 if (Ustat(spool_fname(US"input", message_subdir, spoolname, US""),
5647 &statbuf) == 0)
5648 log_write(0, LOG_MAIN, "Format error in spool file %s: "
5649 "size=" OFF_T_FMT, spoolname, statbuf.st_size);
5650 else
5651 log_write(0, LOG_MAIN, "Format error in spool file %s", spoolname);
5652 }
5653 else
5654 log_write(0, LOG_MAIN, "Error reading spool file %s: %s", spoolname,
5655 strerror(errno));
059ec3d9 5656
41313d92
JH
5657 /* If we managed to read the envelope data, received_time contains the
5658 time the message was received. Otherwise, we can calculate it from the
5659 message id. */
059ec3d9 5660
41313d92
JH
5661 if (rc != spool_read_hdrerror)
5662 {
32dfdf8b
JH
5663 received_time.tv_sec = received_time.tv_usec = 0;
5664 /*XXX subsec precision?*/
41313d92 5665 for (i = 0; i < 6; i++)
32dfdf8b 5666 received_time.tv_sec = received_time.tv_sec * BASE_62 + tab62[id[i] - '0'];
41313d92 5667 }
059ec3d9 5668
41313d92 5669 /* If we've had this malformed message too long, sling it. */
059ec3d9 5670
32dfdf8b 5671 if (now - received_time.tv_sec > keep_malformed)
41313d92
JH
5672 {
5673 Uunlink(spool_fname(US"msglog", message_subdir, id, US""));
5674 Uunlink(spool_fname(US"input", message_subdir, id, US"-D"));
5675 Uunlink(spool_fname(US"input", message_subdir, id, US"-H"));
5676 Uunlink(spool_fname(US"input", message_subdir, id, US"-J"));
5677 log_write(0, LOG_MAIN, "Message removed because older than %s",
5678 readconf_printtime(keep_malformed));
5679 }
059ec3d9 5680
41313d92
JH
5681 (void)close(deliver_datafile);
5682 deliver_datafile = -1;
5683 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5684 }
059ec3d9
PH
5685 }
5686
5687/* The spool header file has been read. Look to see if there is an existing
5688journal file for this message. If there is, it means that a previous delivery
5689attempt crashed (program or host) before it could update the spool header file.
5690Read the list of delivered addresses from the journal and add them to the
5691nonrecipients tree. Then update the spool file. We can leave the journal in
5692existence, as it will get further successful deliveries added to it in this
5693run, and it will be deleted if this function gets to its end successfully.
5694Otherwise it might be needed again. */
5695
059ec3d9 5696 {
41313d92 5697 uschar * fname = spool_fname(US"input", message_subdir, id, US"-J");
92b0827a 5698 FILE * jread;
41313d92 5699
92b0827a
JH
5700 if ( (journal_fd = Uopen(fname, O_RDWR|O_APPEND
5701#ifdef O_CLOEXEC
5702 | O_CLOEXEC
5703#endif
5704#ifdef O_NOFOLLOW
5705 | O_NOFOLLOW
5706#endif
5707 , SPOOL_MODE)) >= 0
5708 && lseek(journal_fd, 0, SEEK_SET) == 0
5709 && (jread = fdopen(journal_fd, "rb"))
5710 )
059ec3d9 5711 {
41313d92
JH
5712 while (Ufgets(big_buffer, big_buffer_size, jread))
5713 {
5714 int n = Ustrlen(big_buffer);
5715 big_buffer[n-1] = 0;
5716 tree_add_nonrecipient(big_buffer);
5717 DEBUG(D_deliver) debug_printf("Previously delivered address %s taken from "
5718 "journal file\n", big_buffer);
5719 }
92b0827a 5720 rewind(jread);
c0741086
JH
5721 if ((journal_fd = dup(fileno(jread))) < 0)
5722 journal_fd = fileno(jread);
5723 else
5724 (void) fclose(jread); /* Try to not leak the FILE resource */
5725
41313d92
JH
5726 /* Panic-dies on error */
5727 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
5728 }
5729 else if (errno != ENOENT)
5730 {
5731 log_write(0, LOG_MAIN|LOG_PANIC, "attempt to open journal for reading gave: "
5732 "%s", strerror(errno));
5733 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
059ec3d9 5734 }
059ec3d9 5735
41313d92 5736 /* A null recipients list indicates some kind of disaster. */
059ec3d9 5737
41313d92
JH
5738 if (!recipients_list)
5739 {
5740 (void)close(deliver_datafile);
5741 deliver_datafile = -1;
5742 log_write(0, LOG_MAIN, "Spool error: no recipients for %s", fname);
5743 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5744 }
059ec3d9
PH
5745 }
5746
5747
5748/* Handle a message that is frozen. There are a number of different things that
5749can happen, but in the default situation, unless forced, no delivery is
5750attempted. */
5751
8768d548 5752if (f.deliver_freeze)
059ec3d9 5753 {
a7538db1 5754#ifdef SUPPORT_MOVE_FROZEN_MESSAGES
059ec3d9
PH
5755 /* Moving to another directory removes the message from Exim's view. Other
5756 tools must be used to deal with it. Logging of this action happens in
5757 spool_move_message() and its subfunctions. */
5758
d6c82d7b
JH
5759 if ( move_frozen_messages
5760 && spool_move_message(id, message_subdir, US"", US"F")
5761 )
059ec3d9 5762 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
a7538db1 5763#endif
059ec3d9
PH
5764
5765 /* For all frozen messages (bounces or not), timeout_frozen_after sets the
5766 maximum time to keep messages that are frozen. Thaw if we reach it, with a
5767 flag causing all recipients to be failed. The time is the age of the
5768 message, not the time since freezing. */
5769
5770 if (timeout_frozen_after > 0 && message_age >= timeout_frozen_after)
5771 {
5772 log_write(0, LOG_MAIN, "cancelled by timeout_frozen_after");
5773 process_recipients = RECIP_FAIL_TIMEOUT;
5774 }
5775
5776 /* For bounce messages (and others with no sender), thaw if the error message
5777 ignore timer is exceeded. The message will be discarded if this delivery
5778 fails. */
5779
10c50704 5780 else if (!*sender_address && message_age >= ignore_bounce_errors_after)
059ec3d9 5781 log_write(0, LOG_MAIN, "Unfrozen by errmsg timer");
059ec3d9 5782
ef213c3b
PH
5783 /* If this is a bounce message, or there's no auto thaw, or we haven't
5784 reached the auto thaw time yet, and this delivery is not forced by an admin
5785 user, do not attempt delivery of this message. Note that forced is set for
5786 continuing messages down the same channel, in order to skip load checking and
5787 ignore hold domains, but we don't want unfreezing in that case. */
059ec3d9
PH
5788
5789 else
5790 {
d6c82d7b
JH
5791 if ( ( sender_address[0] == 0
5792 || auto_thaw <= 0
5793 || now <= deliver_frozen_at + auto_thaw
5794 )
8768d548
JH
5795 && ( !forced || !f.deliver_force_thaw
5796 || !f.admin_user || continue_hostname
d6c82d7b 5797 ) )
059ec3d9 5798 {
f1e894f3 5799 (void)close(deliver_datafile);
059ec3d9
PH
5800 deliver_datafile = -1;
5801 log_write(L_skip_delivery, LOG_MAIN, "Message is frozen");
5802 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5803 }
5804
5805 /* If delivery was forced (by an admin user), assume a manual thaw.
5806 Otherwise it's an auto thaw. */
5807
5808 if (forced)
5809 {
8768d548 5810 f.deliver_manual_thaw = TRUE;
059ec3d9
PH
5811 log_write(0, LOG_MAIN, "Unfrozen by forced delivery");
5812 }
5813 else log_write(0, LOG_MAIN, "Unfrozen by auto-thaw");
5814 }
5815
5816 /* We get here if any of the rules for unfreezing have triggered. */
5817
8768d548 5818 f.deliver_freeze = FALSE;
059ec3d9
PH
5819 update_spool = TRUE;
5820 }
5821
5822
5823/* Open the message log file if we are using them. This records details of
5824deliveries, deferments, and failures for the benefit of the mail administrator.
5825The log is not used by exim itself to track the progress of a message; that is
5826done by rewriting the header spool file. */
5827
5828if (message_logs)
5829 {
41313d92
JH
5830 uschar * fname = spool_fname(US"msglog", message_subdir, id, US"");
5831 uschar * error;
059ec3d9
PH
5832 int fd;
5833
41313d92 5834 if ((fd = open_msglog_file(fname, SPOOL_MODE, &error)) < 0)
059ec3d9
PH
5835 {
5836 log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't %s message log %s: %s", error,
41313d92 5837 fname, strerror(errno));
059ec3d9
PH
5838 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5839 }
5840
5841 /* Make a C stream out of it. */
5842
d6c82d7b 5843 if (!(message_log = fdopen(fd, "a")))
059ec3d9
PH
5844 {
5845 log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't fdopen message log %s: %s",
41313d92 5846 fname, strerror(errno));
059ec3d9
PH
5847 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5848 }
5849 }
5850
5851
5852/* If asked to give up on a message, log who did it, and set the action for all
5853the addresses. */
5854
5855if (give_up)
5856 {
5857 struct passwd *pw = getpwuid(real_uid);
d6c82d7b
JH
5858 log_write(0, LOG_MAIN, "cancelled by %s",
5859 pw ? US pw->pw_name : string_sprintf("uid %ld", (long int)real_uid));
059ec3d9
PH
5860 process_recipients = RECIP_FAIL;
5861 }
5862
5863/* Otherwise, if there are too many Received: headers, fail all recipients. */
5864
5865else if (received_count > received_headers_max)
5866 process_recipients = RECIP_FAIL_LOOP;
5867
5868/* Otherwise, if a system-wide, address-independent message filter is
5869specified, run it now, except in the case when we are failing all recipients as
5870a result of timeout_frozen_after. If the system filter yields "delivered", then
5871ignore the true recipients of the message. Failure of the filter file is
5872logged, and the delivery attempt fails. */
5873
d6c82d7b 5874else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT)
059ec3d9
PH
5875 {
5876 int rc;
5877 int filtertype;
5878 ugid_block ugid;
5879 redirect_block redirect;
5880
5881 if (system_filter_uid_set)
5882 {
5883 ugid.uid = system_filter_uid;
5884 ugid.gid = system_filter_gid;
5885 ugid.uid_set = ugid.gid_set = TRUE;
5886 }
5887 else
059ec3d9 5888 ugid.uid_set = ugid.gid_set = FALSE;
059ec3d9
PH
5889
5890 return_path = sender_address;
8768d548
JH
5891 f.enable_dollar_recipients = TRUE; /* Permit $recipients in system filter */
5892 f.system_filtering = TRUE;
059ec3d9
PH
5893
5894 /* Any error in the filter file causes a delivery to be abandoned. */
5895
5896 redirect.string = system_filter;
5897 redirect.isfile = TRUE;
5898 redirect.check_owner = redirect.check_group = FALSE;
5899 redirect.owners = NULL;
5900 redirect.owngroups = NULL;
5901 redirect.pw = NULL;
5902 redirect.modemask = 0;
5903
5904 DEBUG(D_deliver|D_filter) debug_printf("running system filter\n");
5905
5906 rc = rda_interpret(
5907 &redirect, /* Where the data is */
5908 RDO_DEFER | /* Turn on all the enabling options */
5909 RDO_FAIL | /* Leave off all the disabling options */
5910 RDO_FILTER |
5911 RDO_FREEZE |
5912 RDO_REALLOG |
5913 RDO_REWRITE,
5914 NULL, /* No :include: restriction (not used in filter) */
5915 NULL, /* No sieve vacation directory (not sieve!) */
efd9a422 5916 NULL, /* No sieve enotify mailto owner (not sieve!) */
e4a89c47
PH
5917 NULL, /* No sieve user address (not sieve!) */
5918 NULL, /* No sieve subaddress (not sieve!) */
059ec3d9
PH
5919 &ugid, /* uid/gid data */
5920 &addr_new, /* Where to hang generated addresses */
5921 &filter_message, /* Where to put error message */
5922 NULL, /* Don't skip syntax errors */
5923 &filtertype, /* Will always be set to FILTER_EXIM for this call */
5924 US"system filter"); /* For error messages */
5925
5926 DEBUG(D_deliver|D_filter) debug_printf("system filter returned %d\n", rc);
5927
5928 if (rc == FF_ERROR || rc == FF_NONEXIST)
5929 {
f1e894f3 5930 (void)close(deliver_datafile);
059ec3d9
PH
5931 deliver_datafile = -1;
5932 log_write(0, LOG_MAIN|LOG_PANIC, "Error in system filter: %s",
5933 string_printing(filter_message));
5934 return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */
5935 }
5936
5937 /* Reset things. If the filter message is an empty string, which can happen
5938 for a filter "fail" or "freeze" command with no text, reset it to NULL. */
5939
8768d548
JH
5940 f.system_filtering = FALSE;
5941 f.enable_dollar_recipients = FALSE;
d6c82d7b 5942 if (filter_message && filter_message[0] == 0) filter_message = NULL;
059ec3d9
PH
5943
5944 /* Save the values of the system filter variables so that user filters
5945 can use them. */
5946
5947 memcpy(filter_sn, filter_n, sizeof(filter_sn));
5948
5949 /* The filter can request that delivery of the original addresses be
5950 deferred. */
5951
5952 if (rc == FF_DEFER)
5953 {
5954 process_recipients = RECIP_DEFER;
5955 deliver_msglog("Delivery deferred by system filter\n");
5956 log_write(0, LOG_MAIN, "Delivery deferred by system filter");
5957 }
5958
5959 /* The filter can request that a message be frozen, but this does not
5960 take place if the message has been manually thawed. In that case, we must
5961 unset "delivered", which is forced by the "freeze" command to make -bF
5962 work properly. */
5963
8768d548 5964 else if (rc == FF_FREEZE && !f.deliver_manual_thaw)
059ec3d9 5965 {
8768d548 5966 f.deliver_freeze = TRUE;
059ec3d9
PH
5967 deliver_frozen_at = time(NULL);
5968 process_recipients = RECIP_DEFER;
5969 frozen_info = string_sprintf(" by the system filter%s%s",
d6c82d7b
JH
5970 filter_message ? US": " : US"",
5971 filter_message ? filter_message : US"");
059ec3d9
PH
5972 }
5973
5974 /* The filter can request that a message be failed. The error message may be
5975 quite long - it is sent back to the sender in the bounce - but we don't want
5976 to fill up the log with repetitions of it. If it starts with << then the text
5977 between << and >> is written to the log, with the rest left for the bounce
5978 message. */
5979
5980 else if (rc == FF_FAIL)
5981 {
5982 uschar *colon = US"";
5983 uschar *logmsg = US"";
5984 int loglen = 0;
5985
5986 process_recipients = RECIP_FAIL_FILTER;
5987
d6c82d7b 5988 if (filter_message)
059ec3d9
PH
5989 {
5990 uschar *logend;
5991 colon = US": ";
d6c82d7b
JH
5992 if ( filter_message[0] == '<'
5993 && filter_message[1] == '<'
5994 && (logend = Ustrstr(filter_message, ">>"))
5995 )
059ec3d9
PH
5996 {
5997 logmsg = filter_message + 2;
5998 loglen = logend - logmsg;
5999 filter_message = logend + 2;
6000 if (filter_message[0] == 0) filter_message = NULL;
6001 }
6002 else
6003 {
6004 logmsg = filter_message;
6005 loglen = Ustrlen(filter_message);
6006 }
6007 }
6008
6009 log_write(0, LOG_MAIN, "cancelled by system filter%s%.*s", colon, loglen,
6010 logmsg);
6011 }
6012
6013 /* Delivery can be restricted only to those recipients (if any) that the
6014 filter specified. */
6015
6016 else if (rc == FF_DELIVERED)
6017 {
6018 process_recipients = RECIP_IGNORE;
d6c82d7b 6019 if (addr_new)
059ec3d9 6020 log_write(0, LOG_MAIN, "original recipients ignored (system filter)");
d6c82d7b
JH
6021 else
6022 log_write(0, LOG_MAIN, "=> discarded (system filter)");
059ec3d9
PH
6023 }
6024
6025 /* If any new addresses were created by the filter, fake up a "parent"
6026 for them. This is necessary for pipes, etc., which are expected to have
6027 parents, and it also gives some sensible logging for others. Allow
6028 pipes, files, and autoreplies, and run them as the filter uid if set,
6029 otherwise as the current uid. */
6030
d6c82d7b 6031 if (addr_new)
059ec3d9
PH
6032 {
6033 int uid = (system_filter_uid_set)? system_filter_uid : geteuid();
6034 int gid = (system_filter_gid_set)? system_filter_gid : getegid();
6035
6036 /* The text "system-filter" is tested in transport_set_up_command() and in
6037 set_up_shell_command() in the pipe transport, to enable them to permit
6038 $recipients, so don't change it here without also changing it there. */
6039
6040 address_item *p = addr_new;
6041 address_item *parent = deliver_make_addr(US"system-filter", FALSE);
6042
6043 parent->domain = string_copylc(qualify_domain_recipient);
6044 parent->local_part = US"system-filter";
6045
6046 /* As part of this loop, we arrange for addr_last to end up pointing
6047 at the final address. This is used if we go on to add addresses for the
6048 original recipients. */
6049
d6c82d7b 6050 while (p)
059ec3d9 6051 {
82f90600 6052 if (parent->child_count == USHRT_MAX)
4362ff0d 6053 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "system filter generated more "
82f90600 6054 "than %d delivery addresses", USHRT_MAX);
059ec3d9
PH
6055 parent->child_count++;
6056 p->parent = parent;
6057
6058 if (testflag(p, af_pfr))
6059 {
6060 uschar *tpname;
6061 uschar *type;
6062 p->uid = uid;
6063 p->gid = gid;
7eb0e5d2
JH
6064 setflag(p, af_uid_set);
6065 setflag(p, af_gid_set);
6066 setflag(p, af_allow_file);
6067 setflag(p, af_allow_pipe);
6068 setflag(p, af_allow_reply);
059ec3d9
PH
6069
6070 /* Find the name of the system filter's appropriate pfr transport */
6071
6072 if (p->address[0] == '|')
6073 {
6074 type = US"pipe";
6075 tpname = system_filter_pipe_transport;
6076 address_pipe = p->address;
6077 }
6078 else if (p->address[0] == '>')
6079 {
6080 type = US"reply";
6081 tpname = system_filter_reply_transport;
6082 }
6083 else
6084 {
6085 if (p->address[Ustrlen(p->address)-1] == '/')
6086 {
6087 type = US"directory";
6088 tpname = system_filter_directory_transport;
6089 }
6090 else
6091 {
6092 type = US"file";
6093 tpname = system_filter_file_transport;
6094 }
6095 address_file = p->address;
6096 }
6097
6098 /* Now find the actual transport, first expanding the name. We have
6099 set address_file or address_pipe above. */
6100
d6c82d7b 6101 if (tpname)
059ec3d9
PH
6102 {
6103 uschar *tmp = expand_string(tpname);
6104 address_file = address_pipe = NULL;
d6c82d7b 6105 if (!tmp)
059ec3d9
PH
6106 p->message = string_sprintf("failed to expand \"%s\" as a "
6107 "system filter transport name", tpname);
6108 tpname = tmp;
6109 }
6110 else
059ec3d9
PH
6111 p->message = string_sprintf("system_filter_%s_transport is unset",
6112 type);
059ec3d9 6113
d6c82d7b 6114 if (tpname)
059ec3d9
PH
6115 {
6116 transport_instance *tp;
d6c82d7b 6117 for (tp = transports; tp; tp = tp->next)
059ec3d9
PH
6118 if (Ustrcmp(tp->name, tpname) == 0)
6119 {
6120 p->transport = tp;
6121 break;
6122 }
d6c82d7b 6123 if (!tp)
059ec3d9
PH
6124 p->message = string_sprintf("failed to find \"%s\" transport "
6125 "for system filter delivery", tpname);
6126 }
6127
6128 /* If we couldn't set up a transport, defer the delivery, putting the
6129 error on the panic log as well as the main log. */
6130
d6c82d7b 6131 if (!p->transport)
059ec3d9
PH
6132 {
6133 address_item *badp = p;
6134 p = p->next;
d6c82d7b 6135 if (!addr_last) addr_new = p; else addr_last->next = p;
059ec3d9 6136 badp->local_part = badp->address; /* Needed for log line */
ba936fb8 6137 post_process_one(badp, DEFER, LOG_MAIN|LOG_PANIC, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6138 continue;
6139 }
6140 } /* End of pfr handling */
6141
6142 /* Either a non-pfr delivery, or we found a transport */
6143
6144 DEBUG(D_deliver|D_filter)
6145 debug_printf("system filter added %s\n", p->address);
6146
6147 addr_last = p;
6148 p = p->next;
6149 } /* Loop through all addr_new addresses */
6150 }
6151 }
6152
6153
6154/* Scan the recipients list, and for every one that is not in the non-
6155recipients tree, add an addr item to the chain of new addresses. If the pno
6156value is non-negative, we must set the onetime parent from it. This which
6157points to the relevant entry in the recipients list.
6158
6159This processing can be altered by the setting of the process_recipients
6160variable, which is changed if recipients are to be ignored, failed, or
6161deferred. This can happen as a result of system filter activity, or if the -Mg
6162option is used to fail all of them.
6163
6164Duplicate addresses are handled later by a different tree structure; we can't
6165just extend the non-recipients tree, because that will be re-written to the
6166spool if the message is deferred, and in any case there are casing
6167complications for local addresses. */
6168
6169if (process_recipients != RECIP_IGNORE)
059ec3d9 6170 for (i = 0; i < recipients_count; i++)
d6c82d7b 6171 if (!tree_search(tree_nonrecipients, recipients_list[i].address))
059ec3d9
PH
6172 {
6173 recipient_item *r = recipients_list + i;
6174 address_item *new = deliver_make_addr(r->address, FALSE);
d43cbe25 6175 new->prop.errors_address = r->errors_to;
8c5d388a 6176#ifdef SUPPORT_I18N
3c8b3577
JH
6177 if ((new->prop.utf8_msg = message_smtputf8))
6178 {
6179 new->prop.utf8_downcvt = message_utf8_downconvert == 1;
6180 new->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
0ec7e948
JH
6181 DEBUG(D_deliver) debug_printf("utf8, downconvert %s\n",
6182 new->prop.utf8_downcvt ? "yes"
6183 : new->prop.utf8_downcvt_maybe ? "ifneeded"
6184 : "no");
3c8b3577 6185 }
7ade712c 6186#endif
059ec3d9
PH
6187
6188 if (r->pno >= 0)
6189 new->onetime_parent = recipients_list[r->pno].address;
6190
94431adb 6191 /* If DSN support is enabled, set the dsn flags and the original receipt
6c1c3d1d
WB
6192 to be passed on to other DSN enabled MTAs */
6193 new->dsn_flags = r->dsn_flags & rf_dsnflags;
6194 new->dsn_orcpt = r->orcpt;
f923454a 6195 DEBUG(D_deliver) debug_printf("DSN: set orcpt: %s flags: %d\n",
d011368a 6196 new->dsn_orcpt ? new->dsn_orcpt : US"", new->dsn_flags);
6c1c3d1d 6197
059ec3d9
PH
6198 switch (process_recipients)
6199 {
6200 /* RECIP_DEFER is set when a system filter freezes a message. */
6201
6202 case RECIP_DEFER:
1afce5bb
MK
6203 new->next = addr_defer;
6204 addr_defer = new;
6205 break;
059ec3d9
PH
6206
6207
6208 /* RECIP_FAIL_FILTER is set when a system filter has obeyed a "fail"
6209 command. */
6210
6211 case RECIP_FAIL_FILTER:
1afce5bb
MK
6212 new->message =
6213 filter_message ? filter_message : US"delivery cancelled";
6214 setflag(new, af_pass_message);
6215 goto RECIP_QUEUE_FAILED; /* below */
059ec3d9
PH
6216
6217
6218 /* RECIP_FAIL_TIMEOUT is set when a message is frozen, but is older
6219 than the value in timeout_frozen_after. Treat non-bounce messages
6220 similarly to -Mg; for bounce messages we just want to discard, so
6221 don't put the address on the failed list. The timeout has already
6222 been logged. */
6223
6224 case RECIP_FAIL_TIMEOUT:
1afce5bb
MK
6225 new->message = US"delivery cancelled; message timed out";
6226 goto RECIP_QUEUE_FAILED; /* below */
059ec3d9
PH
6227
6228
6229 /* RECIP_FAIL is set when -Mg has been used. */
6230
6231 case RECIP_FAIL:
1afce5bb
MK
6232 new->message = US"delivery cancelled by administrator";
6233 /* Fall through */
059ec3d9
PH
6234
6235 /* Common code for the failure cases above. If this is not a bounce
6236 message, put the address on the failed list so that it is used to
6237 create a bounce. Otherwise do nothing - this just discards the address.
6238 The incident has already been logged. */
6239
6240 RECIP_QUEUE_FAILED:
1afce5bb
MK
6241 if (sender_address[0])
6242 {
6243 new->next = addr_failed;
6244 addr_failed = new;
6245 }
059ec3d9
PH
6246 break;
6247
6248
6249 /* RECIP_FAIL_LOOP is set when there are too many Received: headers
6250 in the message. Process each address as a routing failure; if this
6251 is a bounce message, it will get frozen. */
6252
6253 case RECIP_FAIL_LOOP:
1afce5bb
MK
6254 new->message = US"Too many \"Received\" headers - suspected mail loop";
6255 post_process_one(new, FAIL, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
6256 break;
059ec3d9
PH
6257
6258
6259 /* Value should be RECIP_ACCEPT; take this as the safe default. */
6260
6261 default:
1afce5bb
MK
6262 if (!addr_new) addr_new = new; else addr_last->next = new;
6263 addr_last = new;
6264 break;
059ec3d9 6265 }
14a465c3 6266
0cbf2b82 6267#ifndef DISABLE_EVENT
14a465c3
JH
6268 if (process_recipients != RECIP_ACCEPT)
6269 {
6270 uschar * save_local = deliver_localpart;
55414b25 6271 const uschar * save_domain = deliver_domain;
7ea1237c
MK
6272 uschar * addr = new->address, * errmsg = NULL;
6273 int start, end, dom;
14a465c3 6274
7ea1237c
MK
6275 if (!parse_extract_address(addr, &errmsg, &start, &end, &dom, TRUE))
6276 log_write(0, LOG_MAIN|LOG_PANIC,
6277 "failed to parse address '%.100s': %s\n", addr, errmsg);
6278 else
6279 {
6280 deliver_localpart =
6281 string_copyn(addr+start, dom ? (dom-1) - start : end - start);
6282 deliver_domain = dom ? CUS string_copyn(addr+dom, end - dom) : CUS"";
14a465c3 6283
7ea1237c 6284 event_raise(event_action, US"msg:fail:internal", new->message);
14a465c3 6285
7ea1237c
MK
6286 deliver_localpart = save_local;
6287 deliver_domain = save_domain;
6288 }
14a465c3
JH
6289 }
6290#endif
059ec3d9 6291 }
059ec3d9
PH
6292
6293DEBUG(D_deliver)
6294 {
d6c82d7b 6295 address_item *p;
059ec3d9 6296 debug_printf("Delivery address list:\n");
d6c82d7b
JH
6297 for (p = addr_new; p; p = p->next)
6298 debug_printf(" %s %s\n", p->address,
6299 p->onetime_parent ? p->onetime_parent : US"");
059ec3d9
PH
6300 }
6301
6302/* Set up the buffers used for copying over the file when delivering. */
6303
6304deliver_in_buffer = store_malloc(DELIVER_IN_BUFFER_SIZE);
6305deliver_out_buffer = store_malloc(DELIVER_OUT_BUFFER_SIZE);
6306
6307
6308
6309/* Until there are no more new addresses, handle each one as follows:
6310
6311 . If this is a generated address (indicated by the presence of a parent
6312 pointer) then check to see whether it is a pipe, file, or autoreply, and
6313 if so, handle it directly here. The router that produced the address will
6314 have set the allow flags into the address, and also set the uid/gid required.
6315 Having the routers generate new addresses and then checking them here at
6316 the outer level is tidier than making each router do the checking, and
6317 means that routers don't need access to the failed address queue.
6318
6319 . Break up the address into local part and domain, and make lowercased
6320 versions of these strings. We also make unquoted versions of the local part.
6321
6322 . Handle the percent hack for those domains for which it is valid.
6323
6324 . For child addresses, determine if any of the parents have the same address.
6325 If so, generate a different string for previous delivery checking. Without
6326 this code, if the address spqr generates spqr via a forward or alias file,
6327 delivery of the generated spqr stops further attempts at the top level spqr,
6328 which is not what is wanted - it may have generated other addresses.
6329
6330 . Check on the retry database to see if routing was previously deferred, but
6331 only if in a queue run. Addresses that are to be routed are put on the
6332 addr_route chain. Addresses that are to be deferred are put on the
6333 addr_defer chain. We do all the checking first, so as not to keep the
6334 retry database open any longer than necessary.
6335
6336 . Now we run the addresses through the routers. A router may put the address
6337 on either the addr_local or the addr_remote chain for local or remote
6338 delivery, respectively, or put it on the addr_failed chain if it is
6339 undeliveable, or it may generate child addresses and put them on the
6340 addr_new chain, or it may defer an address. All the chain anchors are
6341 passed as arguments so that the routers can be called for verification
6342 purposes as well.
6343
6344 . If new addresses have been generated by the routers, da capo.
6345*/
6346
8768d548 6347f.header_rewritten = FALSE; /* No headers rewritten yet */
d6c82d7b 6348while (addr_new) /* Loop until all addresses dealt with */
059ec3d9
PH
6349 {
6350 address_item *addr, *parent;
059ec3d9
PH
6351
6352 /* Failure to open the retry database is treated the same as if it does
6353 not exist. In both cases, dbm_file is NULL. */
6354
d6c82d7b 6355 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
059ec3d9
PH
6356 DEBUG(D_deliver|D_retry|D_route|D_hints_lookup)
6357 debug_printf("no retry data available\n");
059ec3d9
PH
6358
6359 /* Scan the current batch of new addresses, to handle pipes, files and
6360 autoreplies, and determine which others are ready for routing. */
6361
d6c82d7b 6362 while (addr_new)
059ec3d9
PH
6363 {
6364 int rc;
6365 uschar *p;
6366 tree_node *tnode;
6367 dbdata_retry *domain_retry_record;
6368 dbdata_retry *address_retry_record;
6369
6370 addr = addr_new;
6371 addr_new = addr->next;
6372
6373 DEBUG(D_deliver|D_retry|D_route)
6374 {
6375 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6376 debug_printf("Considering: %s\n", addr->address);
6377 }
6378
6379 /* Handle generated address that is a pipe or a file or an autoreply. */
6380
6381 if (testflag(addr, af_pfr))
6382 {
424a1c63
PH
6383 /* If an autoreply in a filter could not generate a syntactically valid
6384 address, give up forthwith. Set af_ignore_error so that we don't try to
6385 generate a bounce. */
6386
6387 if (testflag(addr, af_bad_reply))
6388 {
6389 addr->basic_errno = ERRNO_BADADDRESS2;
6390 addr->local_part = addr->address;
6391 addr->message =
6392 US"filter autoreply generated syntactically invalid recipient";
7eb0e5d2 6393 addr->prop.ignore_error = TRUE;
ba936fb8 6394 (void) post_process_one(addr, FAIL, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
424a1c63
PH
6395 continue; /* with the next new address */
6396 }
059ec3d9
PH
6397
6398 /* If two different users specify delivery to the same pipe or file or
6399 autoreply, there should be two different deliveries, so build a unique
6400 string that incorporates the original address, and use this for
6401 duplicate testing and recording delivery, and also for retrying. */
6402
6403 addr->unique =
424a1c63
PH
6404 string_sprintf("%s:%s", addr->address, addr->parent->unique +
6405 (testflag(addr->parent, af_homonym)? 3:0));
059ec3d9
PH
6406
6407 addr->address_retry_key = addr->domain_retry_key =
6408 string_sprintf("T:%s", addr->unique);
6409
6410 /* If a filter file specifies two deliveries to the same pipe or file,
6411 we want to de-duplicate, but this is probably not wanted for two mail
6412 commands to the same address, where probably both should be delivered.
6413 So, we have to invent a different unique string in that case. Just
6414 keep piling '>' characters on the front. */
6415
6416 if (addr->address[0] == '>')
6417 {
d6c82d7b 6418 while (tree_search(tree_duplicates, addr->unique))
059ec3d9
PH
6419 addr->unique = string_sprintf(">%s", addr->unique);
6420 }
6421
d6c82d7b 6422 else if ((tnode = tree_search(tree_duplicates, addr->unique)))
059ec3d9
PH
6423 {
6424 DEBUG(D_deliver|D_route)
6425 debug_printf("%s is a duplicate address: discarded\n", addr->address);
6426 addr->dupof = tnode->data.ptr;
6427 addr->next = addr_duplicate;
6428 addr_duplicate = addr;
6429 continue;
6430 }
6431
6432 DEBUG(D_deliver|D_route) debug_printf("unique = %s\n", addr->unique);
6433
6434 /* Check for previous delivery */
6435
d6c82d7b 6436 if (tree_search(tree_nonrecipients, addr->unique))
059ec3d9
PH
6437 {
6438 DEBUG(D_deliver|D_route)
6439 debug_printf("%s was previously delivered: discarded\n", addr->address);
6440 child_done(addr, tod_stamp(tod_log));
6441 continue;
6442 }
6443
6444 /* Save for checking future duplicates */
6445
6446 tree_add_duplicate(addr->unique, addr);
6447
6448 /* Set local part and domain */
6449
6450 addr->local_part = addr->address;
6451 addr->domain = addr->parent->domain;
6452
6453 /* Ensure that the delivery is permitted. */
6454
6455 if (testflag(addr, af_file))
6456 {
6457 if (!testflag(addr, af_allow_file))
6458 {
6459 addr->basic_errno = ERRNO_FORBIDFILE;
6460 addr->message = US"delivery to file forbidden";
ba936fb8 6461 (void)post_process_one(addr, FAIL, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6462 continue; /* with the next new address */
6463 }
6464 }
6465 else if (addr->address[0] == '|')
6466 {
6467 if (!testflag(addr, af_allow_pipe))
6468 {
6469 addr->basic_errno = ERRNO_FORBIDPIPE;
6470 addr->message = US"delivery to pipe forbidden";
ba936fb8 6471 (void)post_process_one(addr, FAIL, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6472 continue; /* with the next new address */
6473 }
6474 }
6475 else if (!testflag(addr, af_allow_reply))
6476 {
6477 addr->basic_errno = ERRNO_FORBIDREPLY;
6478 addr->message = US"autoreply forbidden";
ba936fb8 6479 (void)post_process_one(addr, FAIL, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6480 continue; /* with the next new address */
6481 }
6482
6483 /* If the errno field is already set to BADTRANSPORT, it indicates
6484 failure to expand a transport string, or find the associated transport,
6485 or an unset transport when one is required. Leave this test till now so
6486 that the forbid errors are given in preference. */
6487
6488 if (addr->basic_errno == ERRNO_BADTRANSPORT)
6489 {
ba936fb8 6490 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6491 continue;
6492 }
6493
6494 /* Treat /dev/null as a special case and abandon the delivery. This
6495 avoids having to specify a uid on the transport just for this case.
6496 Arrange for the transport name to be logged as "**bypassed**". */
6497
6498 if (Ustrcmp(addr->address, "/dev/null") == 0)
6499 {
6500 uschar *save = addr->transport->name;
6501 addr->transport->name = US"**bypassed**";
ba936fb8 6502 (void)post_process_one(addr, OK, LOG_MAIN, EXIM_DTYPE_TRANSPORT, '=');
059ec3d9
PH
6503 addr->transport->name = save;
6504 continue; /* with the next new address */
6505 }
6506
6507 /* Pipe, file, or autoreply delivery is to go ahead as a normal local
6508 delivery. */
6509
6510 DEBUG(D_deliver|D_route)
6511 debug_printf("queued for %s transport\n", addr->transport->name);
6512 addr->next = addr_local;
6513 addr_local = addr;
6514 continue; /* with the next new address */
6515 }
6516
6517 /* Handle normal addresses. First, split up into local part and domain,
6518 handling the %-hack if necessary. There is the possibility of a defer from
6519 a lookup in percent_hack_domains. */
6520
6521 if ((rc = deliver_split_address(addr)) == DEFER)
6522 {
6523 addr->message = US"cannot check percent_hack_domains";
6524 addr->basic_errno = ERRNO_LISTDEFER;
ba936fb8 6525 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_NONE, 0);
059ec3d9
PH
6526 continue;
6527 }
6528
6529 /* Check to see if the domain is held. If so, proceed only if the
6530 delivery was forced by hand. */
6531
6532 deliver_domain = addr->domain; /* set $domain */
d6c82d7b
JH
6533 if ( !forced && hold_domains
6534 && (rc = match_isinlist(addr->domain, (const uschar **)&hold_domains, 0,
059ec3d9 6535 &domainlist_anchor, addr->domain_cache, MCL_DOMAIN, TRUE,
d6c82d7b
JH
6536 NULL)) != FAIL
6537 )
059ec3d9
PH
6538 {
6539 if (rc == DEFER)
6540 {
6541 addr->message = US"hold_domains lookup deferred";
6542 addr->basic_errno = ERRNO_LISTDEFER;
6543 }
6544 else
6545 {
6546 addr->message = US"domain is held";
6547 addr->basic_errno = ERRNO_HELD;
6548 }
ba936fb8 6549 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_NONE, 0);
059ec3d9
PH
6550 continue;
6551 }
6552
6553 /* Now we can check for duplicates and previously delivered addresses. In
6554 order to do this, we have to generate a "unique" value for each address,
6555 because there may be identical actual addresses in a line of descendents.
6556 The "unique" field is initialized to the same value as the "address" field,
6557 but gets changed here to cope with identically-named descendents. */
6558
d6c82d7b 6559 for (parent = addr->parent; parent; parent = parent->parent)
059ec3d9
PH
6560 if (strcmpic(addr->address, parent->address) == 0) break;
6561
6562 /* If there's an ancestor with the same name, set the homonym flag. This
6563 influences how deliveries are recorded. Then add a prefix on the front of
6564 the unique address. We use \n\ where n starts at 0 and increases each time.
6565 It is unlikely to pass 9, but if it does, it may look odd but will still
6566 work. This means that siblings or cousins with the same names are treated
6567 as duplicates, which is what we want. */
6568
d6c82d7b 6569 if (parent)
059ec3d9
PH
6570 {
6571 setflag(addr, af_homonym);
6572 if (parent->unique[0] != '\\')
6573 addr->unique = string_sprintf("\\0\\%s", addr->address);
6574 else
6575 addr->unique = string_sprintf("\\%c\\%s", parent->unique[1] + 1,
6576 addr->address);
6577 }
6578
6579 /* Ensure that the domain in the unique field is lower cased, because
6580 domains are always handled caselessly. */
6581
6582 p = Ustrrchr(addr->unique, '@');
6583 while (*p != 0) { *p = tolower(*p); p++; }
6584
6585 DEBUG(D_deliver|D_route) debug_printf("unique = %s\n", addr->unique);
6586
d6c82d7b 6587 if (tree_search(tree_nonrecipients, addr->unique))
059ec3d9
PH
6588 {
6589 DEBUG(D_deliver|D_route)
6590 debug_printf("%s was previously delivered: discarded\n", addr->unique);
6591 child_done(addr, tod_stamp(tod_log));
6592 continue;
6593 }
6594
059ec3d9 6595 /* Get the routing retry status, saving the two retry keys (with and
48c7f9e2
PH
6596 without the local part) for subsequent use. If there is no retry record for
6597 the standard address routing retry key, we look for the same key with the
6598 sender attached, because this form is used by the smtp transport after a
6599 4xx response to RCPT when address_retry_include_sender is true. */
059ec3d9
PH
6600
6601 addr->domain_retry_key = string_sprintf("R:%s", addr->domain);
6602 addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part,
6603 addr->domain);
6604
d6c82d7b 6605 if (dbm_file)
059ec3d9
PH
6606 {
6607 domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key);
d6c82d7b
JH
6608 if ( domain_retry_record
6609 && now - domain_retry_record->time_stamp > retry_data_expire
6610 )
de6f74f2
JH
6611 {
6612 DEBUG(D_deliver|D_retry)
6613 debug_printf("domain retry record present but expired\n");
48c7f9e2 6614 domain_retry_record = NULL; /* Ignore if too old */
de6f74f2 6615 }
059ec3d9
PH
6616
6617 address_retry_record = dbfn_read(dbm_file, addr->address_retry_key);
d6c82d7b
JH
6618 if ( address_retry_record
6619 && now - address_retry_record->time_stamp > retry_data_expire
6620 )
de6f74f2
JH
6621 {
6622 DEBUG(D_deliver|D_retry)
6623 debug_printf("address retry record present but expired\n");
48c7f9e2 6624 address_retry_record = NULL; /* Ignore if too old */
de6f74f2 6625 }
48c7f9e2 6626
d6c82d7b 6627 if (!address_retry_record)
48c7f9e2 6628 {
09945f1e 6629 uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key,
48c7f9e2
PH
6630 sender_address);
6631 address_retry_record = dbfn_read(dbm_file, altkey);
d6c82d7b
JH
6632 if ( address_retry_record
6633 && now - address_retry_record->time_stamp > retry_data_expire)
de6f74f2
JH
6634 {
6635 DEBUG(D_deliver|D_retry)
6636 debug_printf("address<sender> retry record present but expired\n");
48c7f9e2 6637 address_retry_record = NULL; /* Ignore if too old */
de6f74f2 6638 }
48c7f9e2 6639 }
059ec3d9 6640 }
d6c82d7b
JH
6641 else
6642 domain_retry_record = address_retry_record = NULL;
059ec3d9
PH
6643
6644 DEBUG(D_deliver|D_retry)
6645 {
d6c82d7b 6646 if (!domain_retry_record)
de6f74f2
JH
6647 debug_printf("no domain retry record\n");
6648 else
6649 debug_printf("have domain retry record; next_try = now%+d\n",
6650 f.running_in_test_harness ? 0 :
6651 (int)(domain_retry_record->next_try - now));
6652
d6c82d7b 6653 if (!address_retry_record)
de6f74f2
JH
6654 debug_printf("no address retry record\n");
6655 else
6656 debug_printf("have address retry record; next_try = now%+d\n",
6657 f.running_in_test_harness ? 0 :
6658 (int)(address_retry_record->next_try - now));
059ec3d9
PH
6659 }
6660
6661 /* If we are sending a message down an existing SMTP connection, we must
6662 assume that the message which created the connection managed to route
6663 an address to that connection. We do not want to run the risk of taking
6664 a long time over routing here, because if we do, the server at the other
6665 end of the connection may time it out. This is especially true for messages
6666 with lots of addresses. For this kind of delivery, queue_running is not
6667 set, so we would normally route all addresses. We take a pragmatic approach
6668 and defer routing any addresses that have any kind of domain retry record.
6669 That is, we don't even look at their retry times. It doesn't matter if this
6670 doesn't work occasionally. This is all just an optimization, after all.
6671
6672 The reason for not doing the same for address retries is that they normally
6673 arise from 4xx responses, not DNS timeouts. */
6674
d6c82d7b 6675 if (continue_hostname && domain_retry_record)
059ec3d9
PH
6676 {
6677 addr->message = US"reusing SMTP connection skips previous routing defer";
6678 addr->basic_errno = ERRNO_RRETRY;
ba936fb8 6679 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
de6f74f2
JH
6680
6681 addr->message = domain_retry_record->text;
6682 setflag(addr, af_pass_message);
059ec3d9
PH
6683 }
6684
f6c332bd
PH
6685 /* If we are in a queue run, defer routing unless there is no retry data or
6686 we've passed the next retry time, or this message is forced. In other
6687 words, ignore retry data when not in a queue run.
6688
6689 However, if the domain retry time has expired, always allow the routing
6690 attempt. If it fails again, the address will be failed. This ensures that
059ec3d9
PH
6691 each address is routed at least once, even after long-term routing
6692 failures.
6693
6694 If there is an address retry, check that too; just wait for the next
6695 retry time. This helps with the case when the temporary error on the
6696 address was really message-specific rather than address specific, since
f6c332bd
PH
6697 it allows other messages through.
6698
6699 We also wait for the next retry time if this is a message sent down an
6700 existing SMTP connection (even though that will be forced). Otherwise there
6701 will be far too many attempts for an address that gets a 4xx error. In
6702 fact, after such an error, we should not get here because, the host should
6703 not be remembered as one this message needs. However, there was a bug that
ba9af0af
TF
6704 used to cause this to happen, so it is best to be on the safe side.
6705
1ddeb334
TF
6706 Even if we haven't reached the retry time in the hints, there is one more
6707 check to do, which is for the ultimate address timeout. We only do this
6708 check if there is an address retry record and there is not a domain retry
6709 record; this implies that previous attempts to handle the address had the
6710 retry_use_local_parts option turned on. We use this as an approximation
6711 for the destination being like a local delivery, for example delivery over
6712 LMTP to an IMAP message store. In this situation users are liable to bump
6713 into their quota and thereby have intermittently successful deliveries,
6714 which keep the retry record fresh, which can lead to us perpetually
6715 deferring messages. */
059ec3d9 6716
8768d548 6717 else if ( ( f.queue_running && !f.deliver_force
d6c82d7b
JH
6718 || continue_hostname
6719 )
6720 && ( ( domain_retry_record
6721 && now < domain_retry_record->next_try
6722 && !domain_retry_record->expired
6723 )
6724 || ( address_retry_record
6725 && now < address_retry_record->next_try
6726 ) )
6727 && ( domain_retry_record
6728 || !address_retry_record
6729 || !retry_ultimate_address_timeout(addr->address_retry_key,
6730 addr->domain, address_retry_record, now)
6731 ) )
059ec3d9
PH
6732 {
6733 addr->message = US"retry time not reached";
6734 addr->basic_errno = ERRNO_RRETRY;
ba936fb8 6735 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
de6f74f2
JH
6736
6737 /* For remote-retry errors (here and just above) that we've not yet
6738 hit the rery time, use the error recorded in the retry database
6739 as info in the warning message. This lets us send a message even
6740 when we're not failing on a fresh attempt. We assume that this
6741 info is not sensitive. */
6742
6743 addr->message = domain_retry_record
6744 ? domain_retry_record->text : address_retry_record->text;
6745 setflag(addr, af_pass_message);
059ec3d9
PH
6746 }
6747
6748 /* The domain is OK for routing. Remember if retry data exists so it
6749 can be cleaned up after a successful delivery. */
6750
6751 else
6752 {
d6c82d7b 6753 if (domain_retry_record || address_retry_record)
059ec3d9
PH
6754 setflag(addr, af_dr_retry_exists);
6755 addr->next = addr_route;
6756 addr_route = addr;
6757 DEBUG(D_deliver|D_route)
6758 debug_printf("%s: queued for routing\n", addr->address);
6759 }
6760 }
6761
6762 /* The database is closed while routing is actually happening. Requests to
6763 update it are put on a chain and all processed together at the end. */
6764
d6c82d7b 6765 if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
6766
6767 /* If queue_domains is set, we don't even want to try routing addresses in
6768 those domains. During queue runs, queue_domains is forced to be unset.
6769 Optimize by skipping this pass through the addresses if nothing is set. */
6770
8768d548 6771 if (!f.deliver_force && queue_domains)
059ec3d9
PH
6772 {
6773 address_item *okaddr = NULL;
d6c82d7b 6774 while (addr_route)
059ec3d9
PH
6775 {
6776 address_item *addr = addr_route;
6777 addr_route = addr->next;
6778
6779 deliver_domain = addr->domain; /* set $domain */
55414b25 6780 if ((rc = match_isinlist(addr->domain, (const uschar **)&queue_domains, 0,
059ec3d9
PH
6781 &domainlist_anchor, addr->domain_cache, MCL_DOMAIN, TRUE, NULL))
6782 != OK)
059ec3d9
PH
6783 if (rc == DEFER)
6784 {
6785 addr->basic_errno = ERRNO_LISTDEFER;
6786 addr->message = US"queue_domains lookup deferred";
ba936fb8 6787 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6788 }
6789 else
6790 {
6791 addr->next = okaddr;
6792 okaddr = addr;
6793 }
059ec3d9
PH
6794 else
6795 {
6796 addr->basic_errno = ERRNO_QUEUE_DOMAIN;
6797 addr->message = US"domain is in queue_domains";
ba936fb8 6798 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6799 }
6800 }
6801
6802 addr_route = okaddr;
6803 }
6804
6805 /* Now route those addresses that are not deferred. */
6806
d6c82d7b 6807 while (addr_route)
059ec3d9
PH
6808 {
6809 int rc;
6810 address_item *addr = addr_route;
55414b25 6811 const uschar *old_domain = addr->domain;
059ec3d9
PH
6812 uschar *old_unique = addr->unique;
6813 addr_route = addr->next;
6814 addr->next = NULL;
6815
6816 /* Just in case some router parameter refers to it. */
6817
d6c82d7b
JH
6818 if (!(return_path = addr->prop.errors_address))
6819 return_path = sender_address;
059ec3d9
PH
6820
6821 /* If a router defers an address, add a retry item. Whether or not to
6822 use the local part in the key is a property of the router. */
6823
6824 if ((rc = route_address(addr, &addr_local, &addr_remote, &addr_new,
6825 &addr_succeed, v_none)) == DEFER)
d6c82d7b
JH
6826 retry_add_item(addr,
6827 addr->router->retry_use_local_part
36d295f1 6828 ? string_sprintf("R:%s@%s", addr->local_part, addr->domain)
d6c82d7b
JH
6829 : string_sprintf("R:%s", addr->domain),
6830 0);
059ec3d9
PH
6831
6832 /* Otherwise, if there is an existing retry record in the database, add
09945f1e
PH
6833 retry items to delete both forms. We must also allow for the possibility
6834 of a routing retry that includes the sender address. Since the domain might
6835 have been rewritten (expanded to fully qualified) as a result of routing,
6836 ensure that the rewritten form is also deleted. */
059ec3d9
PH
6837
6838 else if (testflag(addr, af_dr_retry_exists))
6839 {
09945f1e
PH
6840 uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key,
6841 sender_address);
6842 retry_add_item(addr, altkey, rf_delete);
059ec3d9
PH
6843 retry_add_item(addr, addr->address_retry_key, rf_delete);
6844 retry_add_item(addr, addr->domain_retry_key, rf_delete);
6845 if (Ustrcmp(addr->domain, old_domain) != 0)
6846 retry_add_item(addr, string_sprintf("R:%s", old_domain), rf_delete);
6847 }
6848
6849 /* DISCARD is given for :blackhole: and "seen finish". The event has been
6850 logged, but we need to ensure the address (and maybe parents) is marked
6851 done. */
6852
6853 if (rc == DISCARD)
6854 {
6855 address_done(addr, tod_stamp(tod_log));
6856 continue; /* route next address */
6857 }
6858
6859 /* The address is finished with (failed or deferred). */
6860
6861 if (rc != OK)
6862 {
ba936fb8 6863 (void)post_process_one(addr, rc, LOG_MAIN, EXIM_DTYPE_ROUTER, 0);
059ec3d9
PH
6864 continue; /* route next address */
6865 }
6866
6867 /* The address has been routed. If the router changed the domain, it will
6868 also have changed the unique address. We have to test whether this address
6869 has already been delivered, because it's the unique address that finally
6870 gets recorded. */
6871
d6c82d7b
JH
6872 if ( addr->unique != old_unique
6873 && tree_search(tree_nonrecipients, addr->unique) != 0
6874 )
059ec3d9
PH
6875 {
6876 DEBUG(D_deliver|D_route) debug_printf("%s was previously delivered: "
6877 "discarded\n", addr->address);
6878 if (addr_remote == addr) addr_remote = addr->next;
6879 else if (addr_local == addr) addr_local = addr->next;
6880 }
6881
6882 /* If the router has same_domain_copy_routing set, we are permitted to copy
6883 the routing for any other addresses with the same domain. This is an
6884 optimisation to save repeated DNS lookups for "standard" remote domain
6885 routing. The option is settable only on routers that generate host lists.
6886 We play it very safe, and do the optimization only if the address is routed
6887 to a remote transport, there are no header changes, and the domain was not
6888 modified by the router. */
6889
d6c82d7b
JH
6890 if ( addr_remote == addr
6891 && addr->router->same_domain_copy_routing
6892 && !addr->prop.extra_headers
6893 && !addr->prop.remove_headers
6894 && old_domain == addr->domain
6895 )
059ec3d9
PH
6896 {
6897 address_item **chain = &addr_route;
d6c82d7b 6898 while (*chain)
059ec3d9
PH
6899 {
6900 address_item *addr2 = *chain;
6901 if (Ustrcmp(addr2->domain, addr->domain) != 0)
6902 {
6903 chain = &(addr2->next);
6904 continue;
6905 }
6906
6907 /* Found a suitable address; take it off the routing list and add it to
6908 the remote delivery list. */
6909
6910 *chain = addr2->next;
6911 addr2->next = addr_remote;
6912 addr_remote = addr2;
6913
6914 /* Copy the routing data */
6915
6916 addr2->domain = addr->domain;
6917 addr2->router = addr->router;
6918 addr2->transport = addr->transport;
6919 addr2->host_list = addr->host_list;
6920 addr2->fallback_hosts = addr->fallback_hosts;
d43cbe25 6921 addr2->prop.errors_address = addr->prop.errors_address;
7eb0e5d2
JH
6922 copyflag(addr2, addr, af_hide_child);
6923 copyflag(addr2, addr, af_local_host_removed);
059ec3d9
PH
6924
6925 DEBUG(D_deliver|D_route)
059ec3d9
PH
6926 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
6927 "routing %s\n"
6928 "Routing for %s copied from %s\n",
6929 addr2->address, addr2->address, addr->address);
059ec3d9
PH
6930 }
6931 }
6932 } /* Continue with routing the next address. */
6933 } /* Loop to process any child addresses that the routers created, and
6934 any rerouted addresses that got put back on the new chain. */
6935
6936
6937/* Debugging: show the results of the routing */
6938
6939DEBUG(D_deliver|D_retry|D_route)
6940 {
d6c82d7b 6941 address_item *p;
059ec3d9
PH
6942 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6943 debug_printf("After routing:\n Local deliveries:\n");
d6c82d7b 6944 for (p = addr_local; p; p = p->next)
059ec3d9 6945 debug_printf(" %s\n", p->address);
059ec3d9 6946
059ec3d9 6947 debug_printf(" Remote deliveries:\n");
d6c82d7b 6948 for (p = addr_remote; p; p = p->next)
059ec3d9 6949 debug_printf(" %s\n", p->address);
059ec3d9 6950
059ec3d9 6951 debug_printf(" Failed addresses:\n");
d6c82d7b 6952 for (p = addr_failed; p; p = p->next)
059ec3d9 6953 debug_printf(" %s\n", p->address);
059ec3d9 6954
059ec3d9 6955 debug_printf(" Deferred addresses:\n");
d6c82d7b 6956 for (p = addr_defer; p; p = p->next)
059ec3d9 6957 debug_printf(" %s\n", p->address);
059ec3d9
PH
6958 }
6959
6960/* Free any resources that were cached during routing. */
6961
6962search_tidyup();
6963route_tidyup();
6964
6965/* These two variables are set only during routing, after check_local_user.
6966Ensure they are not set in transports. */
6967
6968local_user_gid = (gid_t)(-1);
6969local_user_uid = (uid_t)(-1);
6970
d7174846
PH
6971/* Check for any duplicate addresses. This check is delayed until after
6972routing, because the flexibility of the routing configuration means that
6973identical addresses with different parentage may end up being redirected to
6974different addresses. Checking for duplicates too early (as we previously used
6975to) makes this kind of thing not work. */
6976
6977do_duplicate_check(&addr_local);
6978do_duplicate_check(&addr_remote);
6979
059ec3d9
PH
6980/* When acting as an MUA wrapper, we proceed only if all addresses route to a
6981remote transport. The check that they all end up in one transaction happens in
6982the do_remote_deliveries() function. */
6983
d6c82d7b
JH
6984if ( mua_wrapper
6985 && (addr_local || addr_failed || addr_defer)
6986 )
059ec3d9
PH
6987 {
6988 address_item *addr;
6989 uschar *which, *colon, *msg;
6990
d6c82d7b 6991 if (addr_local)
059ec3d9
PH
6992 {
6993 addr = addr_local;
6994 which = US"local";
6995 }
d6c82d7b 6996 else if (addr_defer)
059ec3d9
PH
6997 {
6998 addr = addr_defer;
6999 which = US"deferred";
7000 }
7001 else
7002 {
7003 addr = addr_failed;
7004 which = US"failed";
7005 }
7006
d6c82d7b 7007 while (addr->parent) addr = addr->parent;
059ec3d9 7008
d6c82d7b 7009 if (addr->message)
059ec3d9
PH
7010 {
7011 colon = US": ";
7012 msg = addr->message;
7013 }
7014 else colon = msg = US"";
7015
7016 /* We don't need to log here for a forced failure as it will already
7017 have been logged. Defer will also have been logged, but as a defer, so we do
7018 need to do the failure logging. */
7019
7020 if (addr != addr_failed)
7021 log_write(0, LOG_MAIN, "** %s routing yielded a %s delivery",
7022 addr->address, which);
7023
7024 /* Always write an error to the caller */
7025
7026 fprintf(stderr, "routing %s yielded a %s delivery%s%s\n", addr->address,
7027 which, colon, msg);
7028
7029 final_yield = DELIVER_MUA_FAILED;
7030 addr_failed = addr_defer = NULL; /* So that we remove the message */
7031 goto DELIVERY_TIDYUP;
7032 }
7033
7034
7035/* If this is a run to continue deliveries to an external channel that is
7036already set up, defer any local deliveries. */
7037
d6c82d7b 7038if (continue_transport)
059ec3d9 7039 {
d6c82d7b 7040 if (addr_defer)
059ec3d9
PH
7041 {
7042 address_item *addr = addr_defer;
d6c82d7b 7043 while (addr->next) addr = addr->next;
059ec3d9
PH
7044 addr->next = addr_local;
7045 }
d6c82d7b
JH
7046 else
7047 addr_defer = addr_local;
059ec3d9
PH
7048 addr_local = NULL;
7049 }
7050
7051
7052/* Because address rewriting can happen in the routers, we should not really do
7053ANY deliveries until all addresses have been routed, so that all recipients of
7054the message get the same headers. However, this is in practice not always
7055possible, since sometimes remote addresses give DNS timeouts for days on end.
7056The pragmatic approach is to deliver what we can now, saving any rewritten
7057headers so that at least the next lot of recipients benefit from the rewriting
7058that has already been done.
7059
7060If any headers have been rewritten during routing, update the spool file to
7061remember them for all subsequent deliveries. This can be delayed till later if
7062there is only address to be delivered - if it succeeds the spool write need not
7063happen. */
7064
8768d548 7065if ( f.header_rewritten
92b0827a
JH
7066 && ( addr_local && (addr_local->next || addr_remote)
7067 || addr_remote && addr_remote->next
d6c82d7b 7068 ) )
059ec3d9
PH
7069 {
7070 /* Panic-dies on error */
7071 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
8768d548 7072 f.header_rewritten = FALSE;
059ec3d9
PH
7073 }
7074
7075
de6f74f2 7076/* If there are any deliveries to do and we do not already have the journal
92b0827a
JH
7077file, create it. This is used to record successful deliveries as soon as
7078possible after each delivery is known to be complete. A file opened with
7079O_APPEND is used so that several processes can run simultaneously.
059ec3d9
PH
7080
7081The journal is just insurance against crashes. When the spool file is
7082ultimately updated at the end of processing, the journal is deleted. If a
7083journal is found to exist at the start of delivery, the addresses listed
7084therein are added to the non-recipients. */
7085
d6c82d7b 7086if (addr_local || addr_remote)
059ec3d9 7087 {
92b0827a 7088 if (journal_fd < 0)
059ec3d9 7089 {
92b0827a 7090 uschar * fname = spool_fname(US"input", message_subdir, id, US"-J");
4c0a7a9c 7091
92b0827a
JH
7092 if ((journal_fd = Uopen(fname,
7093#ifdef O_CLOEXEC
7094 O_CLOEXEC |
7095#endif
7096 O_WRONLY|O_APPEND|O_CREAT|O_EXCL, SPOOL_MODE)) < 0)
7097 {
7098 log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't open journal file %s: %s",
7099 fname, strerror(errno));
7100 return DELIVER_NOT_ATTEMPTED;
7101 }
059ec3d9 7102
92b0827a
JH
7103 /* Set the close-on-exec flag, make the file owned by Exim, and ensure
7104 that the mode is correct - the group setting doesn't always seem to get
7105 set automatically. */
059ec3d9 7106
92b0827a
JH
7107 if( fchown(journal_fd, exim_uid, exim_gid)
7108 || fchmod(journal_fd, SPOOL_MODE)
7109#ifndef O_CLOEXEC
7110 || fcntl(journal_fd, F_SETFD, fcntl(journal_fd, F_GETFD) | FD_CLOEXEC)
7111#endif
7112 )
7113 {
7114 int ret = Uunlink(fname);
7115 log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't set perms on journal file %s: %s",
7116 fname, strerror(errno));
7117 if(ret && errno != ENOENT)
7118 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
7119 fname, strerror(errno));
7120 return DELIVER_NOT_ATTEMPTED;
7121 }
1ac6b2e7 7122 }
059ec3d9 7123 }
92b0827a
JH
7124else if (journal_fd >= 0)
7125 {
7126 close(journal_fd);
7127 journal_fd = -1;
7128 }
059ec3d9
PH
7129
7130
1ac6b2e7 7131
059ec3d9
PH
7132/* Now we can get down to the business of actually doing deliveries. Local
7133deliveries are done first, then remote ones. If ever the problems of how to
7134handle fallback transports are figured out, this section can be put into a loop
7135for handling fallbacks, though the uid switching will have to be revised. */
7136
f1513293
PH
7137/* Precompile a regex that is used to recognize a parameter in response
7138to an LHLO command, if is isn't already compiled. This may be used on both
7139local and remote LMTP deliveries. */
7140
d6c82d7b
JH
7141if (!regex_IGNOREQUOTA)
7142 regex_IGNOREQUOTA =
7143 regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE);
f1513293
PH
7144
7145/* Handle local deliveries */
7146
d6c82d7b 7147if (addr_local)
059ec3d9
PH
7148 {
7149 DEBUG(D_deliver|D_transport)
7150 debug_printf(">>>>>>>>>>>>>>>> Local deliveries >>>>>>>>>>>>>>>>\n");
7151 do_local_deliveries();
8768d548 7152 f.disable_logging = FALSE;
059ec3d9
PH
7153 }
7154
7155/* If queue_run_local is set, we do not want to attempt any remote deliveries,
7156so just queue them all. */
7157
8768d548 7158if (f.queue_run_local)
d6c82d7b 7159 while (addr_remote)
059ec3d9
PH
7160 {
7161 address_item *addr = addr_remote;
7162 addr_remote = addr->next;
7163 addr->next = NULL;
7164 addr->basic_errno = ERRNO_LOCAL_ONLY;
7165 addr->message = US"remote deliveries suppressed";
ba936fb8 7166 (void)post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_TRANSPORT, 0);
059ec3d9 7167 }
059ec3d9
PH
7168
7169/* Handle remote deliveries */
7170
d6c82d7b 7171if (addr_remote)
059ec3d9
PH
7172 {
7173 DEBUG(D_deliver|D_transport)
7174 debug_printf(">>>>>>>>>>>>>>>> Remote deliveries >>>>>>>>>>>>>>>>\n");
7175
7176 /* Precompile some regex that are used to recognize parameters in response
7177 to an EHLO command, if they aren't already compiled. */
7178
476be7e2 7179 deliver_init();
6c1c3d1d 7180
059ec3d9
PH
7181 /* Now sort the addresses if required, and do the deliveries. The yield of
7182 do_remote_deliveries is FALSE when mua_wrapper is set and all addresses
7183 cannot be delivered in one transaction. */
7184
d6c82d7b 7185 if (remote_sort_domains) sort_remote_deliveries();
059ec3d9
PH
7186 if (!do_remote_deliveries(FALSE))
7187 {
7188 log_write(0, LOG_MAIN, "** mua_wrapper is set but recipients cannot all "
7189 "be delivered in one transaction");
7190 fprintf(stderr, "delivery to smarthost failed (configuration problem)\n");
7191
7192 final_yield = DELIVER_MUA_FAILED;
7193 addr_failed = addr_defer = NULL; /* So that we remove the message */
7194 goto DELIVERY_TIDYUP;
7195 }
7196
7197 /* See if any of the addresses that failed got put on the queue for delivery
7198 to their fallback hosts. We do it this way because often the same fallback
7199 host is used for many domains, so all can be sent in a single transaction
7200 (if appropriately configured). */
7201
d6c82d7b 7202 if (addr_fallback && !mua_wrapper)
059ec3d9
PH
7203 {
7204 DEBUG(D_deliver) debug_printf("Delivering to fallback hosts\n");
7205 addr_remote = addr_fallback;
7206 addr_fallback = NULL;
d6c82d7b 7207 if (remote_sort_domains) sort_remote_deliveries();
059ec3d9
PH
7208 do_remote_deliveries(TRUE);
7209 }
8768d548 7210 f.disable_logging = FALSE;
059ec3d9
PH
7211 }
7212
7213
7214/* All deliveries are now complete. Ignore SIGTERM during this tidying up
7215phase, to minimize cases of half-done things. */
7216
7217DEBUG(D_deliver)
7218 debug_printf(">>>>>>>>>>>>>>>> deliveries are done >>>>>>>>>>>>>>>>\n");
ef698bf6 7219cancel_cutthrough_connection(TRUE, US"deliveries are done");
059ec3d9
PH
7220
7221/* Root privilege is no longer needed */
7222
7223exim_setugid(exim_uid, exim_gid, FALSE, US"post-delivery tidying");
7224
7225set_process_info("tidying up after delivering %s", message_id);
7226signal(SIGTERM, SIG_IGN);
7227
7228/* When we are acting as an MUA wrapper, the smtp transport will either have
47c7a64a
PH
7229succeeded for all addresses, or failed them all in normal cases. However, there
7230are some setup situations (e.g. when a named port does not exist) that cause an
7231immediate exit with deferral of all addresses. Convert those into failures. We
7232do not ever want to retry, nor do we want to send a bounce message. */
059ec3d9
PH
7233
7234if (mua_wrapper)
7235 {
d6c82d7b 7236 if (addr_defer)
47c7a64a
PH
7237 {
7238 address_item *addr, *nextaddr;
d6c82d7b 7239 for (addr = addr_defer; addr; addr = nextaddr)
47c7a64a
PH
7240 {
7241 log_write(0, LOG_MAIN, "** %s mua_wrapper forced failure for deferred "
7242 "delivery", addr->address);
7243 nextaddr = addr->next;
7244 addr->next = addr_failed;
7245 addr_failed = addr;
7246 }
7247 addr_defer = NULL;
7248 }
7249
7250 /* Now all should either have succeeded or failed. */
7251
d6c82d7b
JH
7252 if (!addr_failed)
7253 final_yield = DELIVER_MUA_SUCCEEDED;
7254 else
059ec3d9 7255 {
c562fd30 7256 host_item * host;
d6c82d7b
JH
7257 uschar *s = addr_failed->user_message;
7258
7259 if (!s) s = addr_failed->message;
059ec3d9
PH
7260
7261 fprintf(stderr, "Delivery failed: ");
7262 if (addr_failed->basic_errno > 0)
7263 {
7264 fprintf(stderr, "%s", strerror(addr_failed->basic_errno));
d6c82d7b 7265 if (s) fprintf(stderr, ": ");
059ec3d9 7266 }
c562fd30
JH
7267 if ((host = addr_failed->host_used))
7268 fprintf(stderr, "H=%s [%s]: ", host->name, host->address);
d6c82d7b
JH
7269 if (s)
7270 fprintf(stderr, "%s", CS s);
7271 else if (addr_failed->basic_errno <= 0)
7272 fprintf(stderr, "unknown error");
059ec3d9
PH
7273 fprintf(stderr, "\n");
7274
7275 final_yield = DELIVER_MUA_FAILED;
7276 addr_failed = NULL;
7277 }
7278 }
7279
7280/* In a normal configuration, we now update the retry database. This is done in
7281one fell swoop at the end in order not to keep opening and closing (and
7282locking) the database. The code for handling retries is hived off into a
7283separate module for convenience. We pass it the addresses of the various
7284chains, because deferred addresses can get moved onto the failed chain if the
7285retry cutoff time has expired for all alternative destinations. Bypass the
7286updating of the database if the -N flag is set, which is a debugging thing that
7287prevents actual delivery. */
7288
8768d548 7289else if (!f.dont_deliver)
d6c82d7b 7290 retry_update(&addr_defer, &addr_failed, &addr_succeed);
059ec3d9 7291
f40d5be3 7292/* Send DSN for successful messages if requested */
6c1c3d1d
WB
7293addr_senddsn = NULL;
7294
f40d5be3 7295for (addr_dsntmp = addr_succeed; addr_dsntmp; addr_dsntmp = addr_dsntmp->next)
6c1c3d1d 7296 {
6c1c3d1d 7297 /* af_ignore_error not honored here. it's not an error */
f40d5be3 7298 DEBUG(D_deliver) debug_printf("DSN: processing router : %s\n"
9671784a
JH
7299 "DSN: processing successful delivery address: %s\n"
7300 "DSN: Sender_address: %s\n"
7301 "DSN: orcpt: %s flags: %d\n"
7302 "DSN: envid: %s ret: %d\n"
7303 "DSN: Final recipient: %s\n"
7304 "DSN: Remote SMTP server supports DSN: %d\n",
d011368a 7305 addr_dsntmp->router ? addr_dsntmp->router->name : US"(unknown)",
9671784a
JH
7306 addr_dsntmp->address,
7307 sender_address,
d011368a
JH
7308 addr_dsntmp->dsn_orcpt ? addr_dsntmp->dsn_orcpt : US"NULL",
7309 addr_dsntmp->dsn_flags,
7310 dsn_envid ? dsn_envid : US"NULL", dsn_ret,
9671784a
JH
7311 addr_dsntmp->address,
7312 addr_dsntmp->dsn_aware
7313 );
6c1c3d1d
WB
7314
7315 /* send report if next hop not DSN aware or a router flagged "last DSN hop"
7316 and a report was requested */
9671784a
JH
7317 if ( ( addr_dsntmp->dsn_aware != dsn_support_yes
7318 || addr_dsntmp->dsn_flags & rf_dsnlasthop
7319 )
9671784a
JH
7320 && addr_dsntmp->dsn_flags & rf_notify_success
7321 )
6c1c3d1d
WB
7322 {
7323 /* copy and relink address_item and send report with all of them at once later */
82f90600 7324 address_item * addr_next = addr_senddsn;
6c1c3d1d 7325 addr_senddsn = store_get(sizeof(address_item));
82f90600 7326 *addr_senddsn = *addr_dsntmp;
6c1c3d1d
WB
7327 addr_senddsn->next = addr_next;
7328 }
7329 else
94431adb 7330 DEBUG(D_deliver) debug_printf("DSN: not sending DSN success message\n");
6c1c3d1d
WB
7331 }
7332
9671784a 7333if (addr_senddsn)
6c1c3d1d
WB
7334 {
7335 pid_t pid;
7336 int fd;
7337
94431adb 7338 /* create exim process to send message */
6c1c3d1d
WB
7339 pid = child_open_exim(&fd);
7340
7341 DEBUG(D_deliver) debug_printf("DSN: child_open_exim returns: %d\n", pid);
94431adb 7342
6c1c3d1d
WB
7343 if (pid < 0) /* Creation of child failed */
7344 {
7345 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %d (parent %d) failed to "
7346 "create child process to send failure message: %s", getpid(),
7347 getppid(), strerror(errno));
7348
1c5b1499 7349 DEBUG(D_deliver) debug_printf("DSN: child_open_exim failed\n");
94431adb 7350 }
6c1c3d1d
WB
7351 else /* Creation of child succeeded */
7352 {
ee8b8090 7353 FILE * f = fdopen(fd, "wb");
6c1c3d1d 7354 /* header only as required by RFC. only failure DSN needs to honor RET=FULL */
45500060 7355 uschar * bound;
cab0c277 7356 transport_ctx tctx = {{0}};
94431adb 7357
1c5b1499
JH
7358 DEBUG(D_deliver)
7359 debug_printf("sending error message to: %s\n", sender_address);
94431adb 7360
6c1c3d1d 7361 /* build unique id for MIME boundary */
45500060
JH
7362 bound = string_sprintf(TIME_T_FMT "-eximdsn-%d", time(NULL), rand());
7363 DEBUG(D_deliver) debug_printf("DSN: MIME boundary: %s\n", bound);
94431adb 7364
c007c974
JH
7365 if (errors_reply_to)
7366 fprintf(f, "Reply-To: %s\n", errors_reply_to);
94431adb 7367
c007c974
JH
7368 fprintf(f, "Auto-Submitted: auto-generated\n"
7369 "From: Mail Delivery System <Mailer-Daemon@%s>\n"
7370 "To: %s\n"
7371 "Subject: Delivery Status Notification\n"
7372 "Content-Type: multipart/report; report-type=delivery-status; boundary=%s\n"
7373 "MIME-Version: 1.0\n\n"
7374
7375 "--%s\n"
7376 "Content-type: text/plain; charset=us-ascii\n\n"
94431adb 7377
c007c974 7378 "This message was created automatically by mail delivery software.\n"
8864c2c4 7379 " ----- The following addresses had successful delivery notifications -----\n",
45500060 7380 qualify_domain_sender, sender_address, bound, bound);
6c1c3d1d 7381
1c5b1499
JH
7382 for (addr_dsntmp = addr_senddsn; addr_dsntmp;
7383 addr_dsntmp = addr_dsntmp->next)
c007c974
JH
7384 fprintf(f, "<%s> (relayed %s)\n\n",
7385 addr_dsntmp->address,
ea97267c
JH
7386 addr_dsntmp->dsn_flags & rf_dsnlasthop ? "via non DSN router"
7387 : addr_dsntmp->dsn_aware == dsn_support_no ? "to non-DSN-aware mailer"
7388 : "via non \"Remote SMTP\" router"
c007c974 7389 );
1c5b1499 7390
c007c974
JH
7391 fprintf(f, "--%s\n"
7392 "Content-type: message/delivery-status\n\n"
7393 "Reporting-MTA: dns; %s\n",
45500060 7394 bound, smtp_active_hostname);
c007c974 7395
1c5b1499
JH
7396 if (dsn_envid)
7397 { /* must be decoded from xtext: see RFC 3461:6.3a */
6c1c3d1d
WB
7398 uschar *xdec_envid;
7399 if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0)
c007c974 7400 fprintf(f, "Original-Envelope-ID: %s\n", dsn_envid);
6c1c3d1d 7401 else
968178ac 7402 fprintf(f, "X-Original-Envelope-ID: error decoding xtext formatted ENVID\n");
6c1c3d1d 7403 }
c007c974 7404 fputc('\n', f);
6c1c3d1d 7405
c007c974
JH
7406 for (addr_dsntmp = addr_senddsn;
7407 addr_dsntmp;
7408 addr_dsntmp = addr_dsntmp->next)
6c1c3d1d 7409 {
c007c974 7410 if (addr_dsntmp->dsn_orcpt)
6c1c3d1d 7411 fprintf(f,"Original-Recipient: %s\n", addr_dsntmp->dsn_orcpt);
c007c974
JH
7412
7413 fprintf(f, "Action: delivered\n"
7414 "Final-Recipient: rfc822;%s\n"
7415 "Status: 2.0.0\n",
7416 addr_dsntmp->address);
7417
7418 if (addr_dsntmp->host_used && addr_dsntmp->host_used->name)
1c5b1499 7419 fprintf(f, "Remote-MTA: dns; %s\nDiagnostic-Code: smtp; 250 Ok\n\n",
c007c974 7420 addr_dsntmp->host_used->name);
6c1c3d1d 7421 else
1c5b1499 7422 fprintf(f, "Diagnostic-Code: X-Exim; relayed via non %s router\n\n",
ea97267c 7423 addr_dsntmp->dsn_flags & rf_dsnlasthop ? "DSN" : "SMTP");
6c1c3d1d
WB
7424 }
7425
45500060 7426 fprintf(f, "--%s\nContent-type: text/rfc822-headers\n\n", bound);
94431adb 7427
6c1c3d1d
WB
7428 fflush(f);
7429 transport_filter_argv = NULL; /* Just in case */
7430 return_path = sender_address; /* In case not previously set */
94431adb 7431
6c1c3d1d 7432 /* Write the original email out */
6b46ecc6 7433
ee8b8090 7434 tctx.u.fd = fd;
6b46ecc6 7435 tctx.options = topt_add_return_path | topt_no_body;
ee8b8090
JH
7436 /*XXX hmm, retval ignored.
7437 Could error for any number of reasons, and they are not handled. */
42055a33 7438 transport_write_message(&tctx, 0);
6c1c3d1d
WB
7439 fflush(f);
7440
21bc4865 7441 fprintf(f,"\n--%s--\n", bound);
6c1c3d1d
WB
7442
7443 fflush(f);
7444 fclose(f);
7445 rc = child_close(pid, 0); /* Waits for child to close, no timeout */
7446 }
7447 }
6c1c3d1d 7448
059ec3d9
PH
7449/* If any addresses failed, we must send a message to somebody, unless
7450af_ignore_error is set, in which case no action is taken. It is possible for
7451several messages to get sent if there are addresses with different
7452requirements. */
7453
d6c82d7b 7454while (addr_failed)
059ec3d9
PH
7455 {
7456 pid_t pid;
7457 int fd;
7458 uschar *logtod = tod_stamp(tod_log);
7459 address_item *addr;
7460 address_item *handled_addr = NULL;
7461 address_item **paddr;
7462 address_item *msgchain = NULL;
7463 address_item **pmsgchain = &msgchain;
7464
7465 /* There are weird cases when logging is disabled in the transport. However,
7466 there may not be a transport (address failed by a router). */
7467
8768d548 7468 f.disable_logging = FALSE;
d6c82d7b 7469 if (addr_failed->transport)
8768d548 7470 f.disable_logging = addr_failed->transport->disable_logging;
059ec3d9
PH
7471
7472 DEBUG(D_deliver)
7473 debug_printf("processing failed address %s\n", addr_failed->address);
7474
7475 /* There are only two ways an address in a bounce message can get here:
7476
7477 (1) When delivery was initially deferred, but has now timed out (in the call
7478 to retry_update() above). We can detect this by testing for
7479 af_retry_timedout. If the address does not have its own errors address,
7480 we arrange to ignore the error.
7481
7482 (2) If delivery failures for bounce messages are being ignored. We can detect
7483 this by testing for af_ignore_error. This will also be set if a bounce
7484 message has been autothawed and the ignore_bounce_errors_after time has
7485 passed. It might also be set if a router was explicitly configured to
7486 ignore errors (errors_to = "").
7487
7488 If neither of these cases obtains, something has gone wrong. Log the
7489 incident, but then ignore the error. */
7490
d6c82d7b 7491 if (sender_address[0] == 0 && !addr_failed->prop.errors_address)
059ec3d9 7492 {
9671784a 7493 if ( !testflag(addr_failed, af_retry_timedout)
7eb0e5d2 7494 && !addr_failed->prop.ignore_error)
059ec3d9
PH
7495 log_write(0, LOG_MAIN|LOG_PANIC, "internal error: bounce message "
7496 "failure is neither frozen nor ignored (it's been ignored)");
7eb0e5d2
JH
7497
7498 addr_failed->prop.ignore_error = TRUE;
059ec3d9
PH
7499 }
7500
7501 /* If the first address on the list has af_ignore_error set, just remove
7502 it from the list, throw away any saved message file, log it, and
7503 mark the recipient done. */
7504
7eb0e5d2 7505 if ( addr_failed->prop.ignore_error
ea97267c
JH
7506 || addr_failed->dsn_flags & (rf_dsnflags & ~rf_notify_failure)
7507 )
1c5b1499 7508 {
059ec3d9
PH
7509 addr = addr_failed;
7510 addr_failed = addr->next;
d6c82d7b 7511 if (addr->return_filename) Uunlink(addr->return_filename);
059ec3d9 7512
ffbc20ed
MK
7513#ifndef DISABLE_EVENT
7514 msg_event_raise(US"msg:fail:delivery", addr);
7515#endif
059ec3d9
PH
7516 log_write(0, LOG_MAIN, "%s%s%s%s: error ignored",
7517 addr->address,
1c5b1499
JH
7518 !addr->parent ? US"" : US" <",
7519 !addr->parent ? US"" : addr->parent->address,
7520 !addr->parent ? US"" : US">");
059ec3d9
PH
7521
7522 address_done(addr, logtod);
7523 child_done(addr, logtod);
7524 /* Panic-dies on error */
7525 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
7526 }
7527
7528 /* Otherwise, handle the sending of a message. Find the error address for
7529 the first address, then send a message that includes all failed addresses
7530 that have the same error address. Note the bounce_recipient is a global so
4c04137d 7531 that it can be accessed by $bounce_recipient while creating a customized
059ec3d9
PH
7532 error message. */
7533
7534 else
7535 {
d6c82d7b
JH
7536 if (!(bounce_recipient = addr_failed->prop.errors_address))
7537 bounce_recipient = sender_address;
059ec3d9
PH
7538
7539 /* Make a subprocess to send a message */
7540
1c5b1499 7541 if ((pid = child_open_exim(&fd)) < 0)
059ec3d9
PH
7542 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %d (parent %d) failed to "
7543 "create child process to send failure message: %s", getpid(),
7544 getppid(), strerror(errno));
7545
7546 /* Creation of child succeeded */
7547
7548 else
7549 {
7550 int ch, rc;
7551 int filecount = 0;
7552 int rcount = 0;
7553 uschar *bcc, *emf_text;
8768d548
JH
7554 FILE * fp = fdopen(fd, "wb");
7555 FILE * emf = NULL;
059ec3d9
PH
7556 BOOL to_sender = strcmpic(sender_address, bounce_recipient) == 0;
7557 int max = (bounce_return_size_limit/DELIVER_IN_BUFFER_SIZE + 1) *
7558 DELIVER_IN_BUFFER_SIZE;
45500060 7559 uschar * bound;
0f9d3aea
JH
7560 uschar *dsnlimitmsg;
7561 uschar *dsnnotifyhdr;
7562 int topt;
059ec3d9
PH
7563
7564 DEBUG(D_deliver)
7565 debug_printf("sending error message to: %s\n", bounce_recipient);
7566
7567 /* Scan the addresses for all that have the same errors address, removing
7568 them from the addr_failed chain, and putting them on msgchain. */
7569
7570 paddr = &addr_failed;
d6c82d7b 7571 for (addr = addr_failed; addr; addr = *paddr)
d43cbe25
JH
7572 if (Ustrcmp(bounce_recipient, addr->prop.errors_address
7573 ? addr->prop.errors_address : sender_address) == 0)
1c5b1499 7574 { /* The same - dechain */
059ec3d9
PH
7575 *paddr = addr->next;
7576 *pmsgchain = addr;
7577 addr->next = NULL;
7578 pmsgchain = &(addr->next);
7579 }
1c5b1499
JH
7580 else
7581 paddr = &addr->next; /* Not the same; skip */
059ec3d9
PH
7582
7583 /* Include X-Failed-Recipients: for automatic interpretation, but do
7584 not let any one header line get too long. We do this by starting a
7585 new header every 50 recipients. Omit any addresses for which the
7586 "hide_child" flag is set. */
7587
d6c82d7b 7588 for (addr = msgchain; addr; addr = addr->next)
059ec3d9
PH
7589 {
7590 if (testflag(addr, af_hide_child)) continue;
7591 if (rcount >= 50)
7592 {
8768d548 7593 fprintf(fp, "\n");
059ec3d9
PH
7594 rcount = 0;
7595 }
8768d548 7596 fprintf(fp, "%s%s",
d6c82d7b
JH
7597 rcount++ == 0
7598 ? "X-Failed-Recipients: "
7599 : ",\n ",
7600 testflag(addr, af_pfr) && addr->parent
7601 ? string_printing(addr->parent->address)
7602 : string_printing(addr->address));
059ec3d9 7603 }
8768d548 7604 if (rcount > 0) fprintf(fp, "\n");
059ec3d9
PH
7605
7606 /* Output the standard headers */
7607
1c5b1499 7608 if (errors_reply_to)
8768d548
JH
7609 fprintf(fp, "Reply-To: %s\n", errors_reply_to);
7610 fprintf(fp, "Auto-Submitted: auto-replied\n");
7611 moan_write_from(fp);
7612 fprintf(fp, "To: %s\n", bounce_recipient);
059ec3d9 7613
6c1c3d1d 7614 /* generate boundary string and output MIME-Headers */
45500060 7615 bound = string_sprintf(TIME_T_FMT "-eximdsn-%d", time(NULL), rand());
c007c974 7616
8768d548 7617 fprintf(fp, "Content-Type: multipart/report;"
c007c974
JH
7618 " report-type=delivery-status; boundary=%s\n"
7619 "MIME-Version: 1.0\n",
45500060 7620 bound);
6c1c3d1d 7621
059ec3d9
PH
7622 /* Open a template file if one is provided. Log failure to open, but
7623 carry on - default texts will be used. */
7624
c007c974
JH
7625 if (bounce_message_file)
7626 if (!(emf = Ufopen(bounce_message_file, "rb")))
059ec3d9
PH
7627 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to open %s for error "
7628 "message texts: %s", bounce_message_file, strerror(errno));
059ec3d9
PH
7629
7630 /* Quietly copy to configured additional addresses if required. */
7631
c007c974 7632 if ((bcc = moan_check_errorcopy(bounce_recipient)))
8768d548 7633 fprintf(fp, "Bcc: %s\n", bcc);
059ec3d9
PH
7634
7635 /* The texts for the message can be read from a template file; if there
7636 isn't one, or if it is too short, built-in texts are used. The first
7637 emf text is a Subject: and any other headers. */
7638
c007c974 7639 if ((emf_text = next_emf(emf, US"header")))
8768d548 7640 fprintf(fp, "%s\n", emf_text);
c007c974 7641 else
8768d548 7642 fprintf(fp, "Subject: Mail delivery failed%s\n\n",
059ec3d9 7643 to_sender? ": returning message to sender" : "");
059ec3d9 7644
6c1c3d1d 7645 /* output human readable part as text/plain section */
8768d548 7646 fprintf(fp, "--%s\n"
c007c974 7647 "Content-type: text/plain; charset=us-ascii\n\n",
45500060 7648 bound);
6c1c3d1d 7649
c007c974 7650 if ((emf_text = next_emf(emf, US"intro")))
8768d548 7651 fprintf(fp, "%s", CS emf_text);
c007c974 7652 else
059ec3d9 7653 {
8768d548 7654 fprintf(fp,
059ec3d9
PH
7655/* This message has been reworded several times. It seems to be confusing to
7656somebody, however it is worded. I have retreated to the original, simple
7657wording. */
7658"This message was created automatically by mail delivery software.\n");
c007c974
JH
7659
7660 if (bounce_message_text)
8768d548 7661 fprintf(fp, "%s", CS bounce_message_text);
059ec3d9 7662 if (to_sender)
8768d548 7663 fprintf(fp,
059ec3d9
PH
7664"\nA message that you sent could not be delivered to one or more of its\n"
7665"recipients. This is a permanent error. The following address(es) failed:\n");
059ec3d9 7666 else
8768d548 7667 fprintf(fp,
059ec3d9
PH
7668"\nA message sent by\n\n <%s>\n\n"
7669"could not be delivered to one or more of its recipients. The following\n"
7670"address(es) failed:\n", sender_address);
059ec3d9 7671 }
8768d548 7672 fputc('\n', fp);
059ec3d9
PH
7673
7674 /* Process the addresses, leaving them on the msgchain if they have a
7675 file name for a return message. (There has already been a check in
447d236c
PH
7676 post_process_one() for the existence of data in the message file.) A TRUE
7677 return from print_address_information() means that the address is not
7678 hidden. */
059ec3d9
PH
7679
7680 paddr = &msgchain;
d6c82d7b 7681 for (addr = msgchain; addr; addr = *paddr)
059ec3d9 7682 {
8768d548
JH
7683 if (print_address_information(addr, fp, US" ", US"\n ", US""))
7684 print_address_error(addr, fp, US"");
059ec3d9
PH
7685
7686 /* End the final line for the address */
7687
8768d548 7688 fputc('\n', fp);
059ec3d9
PH
7689
7690 /* Leave on msgchain if there's a return file. */
7691
7692 if (addr->return_file >= 0)
7693 {
7694 paddr = &(addr->next);
7695 filecount++;
7696 }
7697
7698 /* Else save so that we can tick off the recipient when the
7699 message is sent. */
7700
7701 else
7702 {
7703 *paddr = addr->next;
7704 addr->next = handled_addr;
7705 handled_addr = addr;
7706 }
7707 }
7708
8768d548 7709 fputc('\n', fp);
059ec3d9
PH
7710
7711 /* Get the next text, whether we need it or not, so as to be
7712 positioned for the one after. */
7713
7714 emf_text = next_emf(emf, US"generated text");
7715
7716 /* If there were any file messages passed by the local transports,
7717 include them in the message. Then put the address on the handled chain.
7718 In the case of a batch of addresses that were all sent to the same
7719 transport, the return_file field in all of them will contain the same
7720 fd, and the return_filename field in the *last* one will be set (to the
7721 name of the file). */
7722
c007c974 7723 if (msgchain)
059ec3d9
PH
7724 {
7725 address_item *nextaddr;
7726
c007c974 7727 if (emf_text)
8768d548 7728 fprintf(fp, "%s", CS emf_text);
c007c974 7729 else
8768d548 7730 fprintf(fp,
059ec3d9
PH
7731 "The following text was generated during the delivery "
7732 "attempt%s:\n", (filecount > 1)? "s" : "");
7733
d6c82d7b 7734 for (addr = msgchain; addr; addr = nextaddr)
059ec3d9
PH
7735 {
7736 FILE *fm;
7737 address_item *topaddr = addr;
7738
7739 /* List all the addresses that relate to this file */
7740
8768d548 7741 fputc('\n', fp);
c007c974 7742 while(addr) /* Insurance */
059ec3d9 7743 {
8768d548 7744 print_address_information(addr, fp, US"------ ", US"\n ",
059ec3d9 7745 US" ------\n");
c007c974 7746 if (addr->return_filename) break;
059ec3d9
PH
7747 addr = addr->next;
7748 }
8768d548 7749 fputc('\n', fp);
059ec3d9
PH
7750
7751 /* Now copy the file */
7752
d6c82d7b 7753 if (!(fm = Ufopen(addr->return_filename, "rb")))
8768d548 7754 fprintf(fp, " +++ Exim error... failed to open text file: %s\n",
059ec3d9
PH
7755 strerror(errno));
7756 else
7757 {
8768d548 7758 while ((ch = fgetc(fm)) != EOF) fputc(ch, fp);
f1e894f3 7759 (void)fclose(fm);
059ec3d9
PH
7760 }
7761 Uunlink(addr->return_filename);
7762
7763 /* Can now add to handled chain, first fishing off the next
7764 address on the msgchain. */
7765
7766 nextaddr = addr->next;
7767 addr->next = handled_addr;
7768 handled_addr = topaddr;
7769 }
8768d548 7770 fputc('\n', fp);
059ec3d9
PH
7771 }
7772
6c1c3d1d 7773 /* output machine readable part */
8c5d388a 7774#ifdef SUPPORT_I18N
9671784a 7775 if (message_smtputf8)
8768d548 7776 fprintf(fp, "--%s\n"
9671784a
JH
7777 "Content-type: message/global-delivery-status\n\n"
7778 "Reporting-MTA: dns; %s\n",
7779 bound, smtp_active_hostname);
7780 else
7781#endif
8768d548 7782 fprintf(fp, "--%s\n"
9671784a
JH
7783 "Content-type: message/delivery-status\n\n"
7784 "Reporting-MTA: dns; %s\n",
7785 bound, smtp_active_hostname);
c007c974
JH
7786
7787 if (dsn_envid)
7788 {
6c1c3d1d
WB
7789 /* must be decoded from xtext: see RFC 3461:6.3a */
7790 uschar *xdec_envid;
7791 if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0)
8768d548 7792 fprintf(fp, "Original-Envelope-ID: %s\n", dsn_envid);
6c1c3d1d 7793 else
8768d548 7794 fprintf(fp, "X-Original-Envelope-ID: error decoding xtext formatted ENVID\n");
6c1c3d1d 7795 }
8768d548 7796 fputc('\n', fp);
94431adb 7797
c007c974 7798 for (addr = handled_addr; addr; addr = addr->next)
6c1c3d1d 7799 {
895fbaf2 7800 host_item * hu;
8768d548 7801 fprintf(fp, "Action: failed\n"
8864c2c4
JH
7802 "Final-Recipient: rfc822;%s\n"
7803 "Status: 5.0.0\n",
7804 addr->address);
895fbaf2
JH
7805 if ((hu = addr->host_used) && hu->name)
7806 {
8768d548 7807 fprintf(fp, "Remote-MTA: dns; %s\n", hu->name);
895fbaf2 7808#ifdef EXPERIMENTAL_DSN_INFO
0e8aed8a
JH
7809 {
7810 const uschar * s;
895fbaf2
JH
7811 if (hu->address)
7812 {
7813 uschar * p = hu->port == 25
7814 ? US"" : string_sprintf(":%d", hu->port);
8768d548 7815 fprintf(fp, "Remote-MTA: X-ip; [%s]%s\n", hu->address, p);
895fbaf2
JH
7816 }
7817 if ((s = addr->smtp_greeting) && *s)
8768d548 7818 fprintf(fp, "X-Remote-MTA-smtp-greeting: X-str; %s\n", s);
895fbaf2 7819 if ((s = addr->helo_response) && *s)
8768d548 7820 fprintf(fp, "X-Remote-MTA-helo-response: X-str; %s\n", s);
895fbaf2 7821 if ((s = addr->message) && *s)
8768d548 7822 fprintf(fp, "X-Exim-Diagnostic: X-str; %s\n", s);
0e8aed8a 7823 }
895fbaf2 7824#endif
8768d548 7825 print_dsn_diagnostic_code(addr, fp);
895fbaf2 7826 }
8768d548 7827 fputc('\n', fp);
6c1c3d1d 7828 }
6c1c3d1d 7829
059ec3d9
PH
7830 /* Now copy the message, trying to give an intelligible comment if
7831 it is too long for it all to be copied. The limit isn't strictly
7832 applied because of the buffering. There is, however, an option
7833 to suppress copying altogether. */
7834
7835 emf_text = next_emf(emf, US"copy");
7836
6c1c3d1d 7837 /* add message body
94431adb 7838 we ignore the intro text from template and add
6c1c3d1d 7839 the text for bounce_return_size_limit at the end.
94431adb 7840
6c1c3d1d
WB
7841 bounce_return_message is ignored
7842 in case RET= is defined we honor these values
7843 otherwise bounce_return_body is honored.
94431adb 7844
6c1c3d1d
WB
7845 bounce_return_size_limit is always honored.
7846 */
94431adb 7847
8768d548 7848 fprintf(fp, "--%s\n", bound);
6c1c3d1d 7849
0f9d3aea
JH
7850 dsnlimitmsg = US"X-Exim-DSN-Information: Due to administrative limits only headers are returned";
7851 dsnnotifyhdr = NULL;
7852 topt = topt_add_return_path;
7853
6c1c3d1d
WB
7854 /* RET=HDRS? top priority */
7855 if (dsn_ret == dsn_ret_hdrs)
7856 topt |= topt_no_body;
7857 else
62b7cd08
JH
7858 {
7859 struct stat statbuf;
7860
6c1c3d1d
WB
7861 /* no full body return at all? */
7862 if (!bounce_return_body)
7863 {
7864 topt |= topt_no_body;
7865 /* add header if we overrule RET=FULL */
7866 if (dsn_ret == dsn_ret_full)
7867 dsnnotifyhdr = dsnlimitmsg;
7868 }
62b7cd08 7869 /* line length limited... return headers only if oversize */
6c1c3d1d 7870 /* size limited ... return headers only if limit reached */
62b7cd08
JH
7871 else if ( max_received_linelength > bounce_return_linesize_limit
7872 || ( bounce_return_size_limit > 0
7873 && fstat(deliver_datafile, &statbuf) == 0
7874 && statbuf.st_size > max
7875 ) )
7876 {
7877 topt |= topt_no_body;
7878 dsnnotifyhdr = dsnlimitmsg;
6c1c3d1d 7879 }
62b7cd08 7880 }
94431adb 7881
8c5d388a 7882#ifdef SUPPORT_I18N
9671784a
JH
7883 if (message_smtputf8)
7884 fputs(topt & topt_no_body ? "Content-type: message/global-headers\n\n"
7885 : "Content-type: message/global\n\n",
8768d548 7886 fp);
9671784a
JH
7887 else
7888#endif
7889 fputs(topt & topt_no_body ? "Content-type: text/rfc822-headers\n\n"
7890 : "Content-type: message/rfc822\n\n",
8768d548 7891 fp);
6c1c3d1d 7892
8768d548 7893 fflush(fp);
6c1c3d1d
WB
7894 transport_filter_argv = NULL; /* Just in case */
7895 return_path = sender_address; /* In case not previously set */
6b46ecc6 7896 { /* Dummy transport for headers add */
cab0c277 7897 transport_ctx tctx = {{0}};
6d5c916c 7898 transport_instance tb = {0};
6b46ecc6 7899
8768d548 7900 tctx.u.fd = fileno(fp);
6d5c916c
JH
7901 tctx.tblock = &tb;
7902 tctx.options = topt;
7903 tb.add_headers = dsnnotifyhdr;
6b46ecc6 7904
ee8b8090 7905 /*XXX no checking for failure! buggy! */
42055a33 7906 transport_write_message(&tctx, 0);
6b46ecc6 7907 }
8768d548 7908 fflush(fp);
94431adb 7909
6c1c3d1d 7910 /* we never add the final text. close the file */
c007c974 7911 if (emf)
6c1c3d1d 7912 (void)fclose(emf);
94431adb 7913
8768d548 7914 fprintf(fp, "\n--%s--\n", bound);
059ec3d9
PH
7915
7916 /* Close the file, which should send an EOF to the child process
7917 that is receiving the message. Wait for it to finish. */
7918
8768d548 7919 (void)fclose(fp);
059ec3d9
PH
7920 rc = child_close(pid, 0); /* Waits for child to close, no timeout */
7921
7922 /* In the test harness, let the child do it's thing first. */
7923
8768d548 7924 if (f.running_in_test_harness) millisleep(500);
059ec3d9
PH
7925
7926 /* If the process failed, there was some disaster in setting up the
7927 error message. Unless the message is very old, ensure that addr_defer
7928 is non-null, which will have the effect of leaving the message on the
7929 spool. The failed addresses will get tried again next time. However, we
7930 don't really want this to happen too often, so freeze the message unless
7931 there are some genuine deferred addresses to try. To do this we have
7932 to call spool_write_header() here, because with no genuine deferred
7933 addresses the normal code below doesn't get run. */
7934
7935 if (rc != 0)
7936 {
7937 uschar *s = US"";
32dfdf8b 7938 if (now - received_time.tv_sec < retry_maximum_timeout && !addr_defer)
059ec3d9
PH
7939 {
7940 addr_defer = (address_item *)(+1);
8768d548 7941 f.deliver_freeze = TRUE;
059ec3d9
PH
7942 deliver_frozen_at = time(NULL);
7943 /* Panic-dies on error */
7944 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
7945 s = US" (frozen)";
7946 }
7947 deliver_msglog("Process failed (%d) when writing error message "
7948 "to %s%s", rc, bounce_recipient, s);
7949 log_write(0, LOG_MAIN, "Process failed (%d) when writing error message "
7950 "to %s%s", rc, bounce_recipient, s);
7951 }
7952
7953 /* The message succeeded. Ensure that the recipients that failed are
7954 now marked finished with on the spool and their parents updated. */
7955
7956 else
7957 {
d6c82d7b 7958 for (addr = handled_addr; addr; addr = addr->next)
059ec3d9
PH
7959 {
7960 address_done(addr, logtod);
7961 child_done(addr, logtod);
7962 }
7963 /* Panic-dies on error */
7964 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
7965 }
7966 }
7967 }
7968 }
7969
8768d548 7970f.disable_logging = FALSE; /* In case left set */
059ec3d9
PH
7971
7972/* Come here from the mua_wrapper case if routing goes wrong */
7973
7974DELIVERY_TIDYUP:
7975
7976/* If there are now no deferred addresses, we are done. Preserve the
7977message log if so configured, and we are using them. Otherwise, sling it.
7978Then delete the message itself. */
7979
d6c82d7b 7980if (!addr_defer)
059ec3d9 7981 {
41313d92
JH
7982 uschar * fname;
7983
059ec3d9
PH
7984 if (message_logs)
7985 {
41313d92 7986 fname = spool_fname(US"msglog", message_subdir, id, US"");
059ec3d9
PH
7987 if (preserve_message_logs)
7988 {
7989 int rc;
41313d92
JH
7990 uschar * moname = spool_fname(US"msglog.OLD", US"", id, US"");
7991
7992 if ((rc = Urename(fname, moname)) < 0)
059ec3d9 7993 {
41313d92
JH
7994 (void)directory_make(spool_directory,
7995 spool_sname(US"msglog.OLD", US""),
7996 MSGLOG_DIRECTORY_MODE, TRUE);
7997 rc = Urename(fname, moname);
059ec3d9
PH
7998 }
7999 if (rc < 0)
8000 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to move %s to the "
41313d92 8001 "msglog.OLD directory", fname);
059ec3d9
PH
8002 }
8003 else
41313d92 8004 if (Uunlink(fname) < 0)
0761d44e 8005 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
41313d92 8006 fname, strerror(errno));
059ec3d9
PH
8007 }
8008
8009 /* Remove the two message files. */
8e669ac1 8010
41313d92
JH
8011 fname = spool_fname(US"input", message_subdir, id, US"-D");
8012 if (Uunlink(fname) < 0)
0ca0cf52 8013 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
41313d92
JH
8014 fname, strerror(errno));
8015 fname = spool_fname(US"input", message_subdir, id, US"-H");
8016 if (Uunlink(fname) < 0)
0ca0cf52 8017 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
41313d92 8018 fname, strerror(errno));
2ac0e484
PH
8019
8020 /* Log the end of this message, with queue time if requested. */
8021
6c6d6e48 8022 if (LOGGING(queue_time_overall))
306c6c77 8023 log_write(0, LOG_MAIN, "Completed QT=%s", string_timesince(&received_time));
2ac0e484
PH
8024 else
8025 log_write(0, LOG_MAIN, "Completed");
0e8a9471
MH
8026
8027 /* Unset deliver_freeze so that we won't try to move the spool files further down */
8768d548 8028 f.deliver_freeze = FALSE;
14a465c3 8029
0cbf2b82 8030#ifndef DISABLE_EVENT
774ef2d7 8031 (void) event_raise(event_action, US"msg:complete", NULL);
14a465c3 8032#endif
fa41615d 8033 }
059ec3d9
PH
8034
8035/* If there are deferred addresses, we are keeping this message because it is
8036not yet completed. Lose any temporary files that were catching output from
8037pipes for any of the deferred addresses, handle one-time aliases, and see if
8038the message has been on the queue for so long that it is time to send a warning
8039message to the sender, unless it is a mailer-daemon. If all deferred addresses
8040have the same domain, we can set deliver_domain for the expansion of
8041delay_warning_ condition - if any of them are pipes, files, or autoreplies, use
8042the parent's domain.
8043
8044If all the deferred addresses have an error number that indicates "retry time
8045not reached", skip sending the warning message, because it won't contain the
8046reason for the delay. It will get sent at the next real delivery attempt.
de6f74f2
JH
8047 Exception: for retries caused by a remote peer we use the error message
8048 store in the retry DB as the reason.
059ec3d9
PH
8049However, if at least one address has tried, we'd better include all of them in
8050the message.
8051
8052If we can't make a process to send the message, don't worry.
8053
8054For mailing list expansions we want to send the warning message to the
8055mailing list manager. We can't do a perfect job here, as some addresses may
8056have different errors addresses, but if we take the errors address from
8057each deferred address it will probably be right in most cases.
8058
8059If addr_defer == +1, it means there was a problem sending an error message
8060for failed addresses, and there were no "real" deferred addresses. The value
8061was set just to keep the message on the spool, so there is nothing to do here.
8062*/
8063
8064else if (addr_defer != (address_item *)(+1))
8065 {
8066 address_item *addr;
8067 uschar *recipients = US"";
de6f74f2 8068 BOOL want_warning_msg = FALSE;
059ec3d9 8069
d6c82d7b
JH
8070 deliver_domain = testflag(addr_defer, af_pfr)
8071 ? addr_defer->parent->domain : addr_defer->domain;
059ec3d9 8072
d6c82d7b 8073 for (addr = addr_defer; addr; addr = addr->next)
059ec3d9
PH
8074 {
8075 address_item *otaddr;
8076
de6f74f2 8077 if (addr->basic_errno > ERRNO_WARN_BASE) want_warning_msg = TRUE;
059ec3d9 8078
d6c82d7b 8079 if (deliver_domain)
059ec3d9 8080 {
55414b25
JH
8081 const uschar *d = testflag(addr, af_pfr)
8082 ? addr->parent->domain : addr->domain;
059ec3d9
PH
8083
8084 /* The domain may be unset for an address that has never been routed
8085 because the system filter froze the message. */
8086
d6c82d7b
JH
8087 if (!d || Ustrcmp(d, deliver_domain) != 0)
8088 deliver_domain = NULL;
059ec3d9
PH
8089 }
8090
d6c82d7b 8091 if (addr->return_filename) Uunlink(addr->return_filename);
059ec3d9
PH
8092
8093 /* Handle the case of one-time aliases. If any address in the ancestry
8094 of this one is flagged, ensure it is in the recipients list, suitably
8095 flagged, and that its parent is marked delivered. */
8096
d6c82d7b
JH
8097 for (otaddr = addr; otaddr; otaddr = otaddr->parent)
8098 if (otaddr->onetime_parent) break;
059ec3d9 8099
d6c82d7b 8100 if (otaddr)
059ec3d9
PH
8101 {
8102 int i;
8103 int t = recipients_count;
8104
8105 for (i = 0; i < recipients_count; i++)
8106 {
8107 uschar *r = recipients_list[i].address;
8108 if (Ustrcmp(otaddr->onetime_parent, r) == 0) t = i;
8109 if (Ustrcmp(otaddr->address, r) == 0) break;
8110 }
8111
8112 /* Didn't find the address already in the list, and did find the
d1f9fb42
JH
8113 ultimate parent's address in the list, and they really are different
8114 (i.e. not from an identity-redirect). After adding the recipient,
059ec3d9
PH
8115 update the errors address in the recipients list. */
8116
d1f9fb42
JH
8117 if ( i >= recipients_count && t < recipients_count
8118 && Ustrcmp(otaddr->address, otaddr->parent->address) != 0)
059ec3d9
PH
8119 {
8120 DEBUG(D_deliver) debug_printf("one_time: adding %s in place of %s\n",
8121 otaddr->address, otaddr->parent->address);
8122 receive_add_recipient(otaddr->address, t);
d43cbe25 8123 recipients_list[recipients_count-1].errors_to = otaddr->prop.errors_address;
059ec3d9
PH
8124 tree_add_nonrecipient(otaddr->parent->address);
8125 update_spool = TRUE;
8126 }
8127 }
8128
8129 /* Except for error messages, ensure that either the errors address for
8130 this deferred address or, if there is none, the sender address, is on the
8131 list of recipients for a warning message. */
8132
d1f9fb42
JH
8133 if (sender_address[0])
8134 {
8135 uschar * s = addr->prop.errors_address;
8136 if (!s) s = sender_address;
8137 if (Ustrstr(recipients, s) == NULL)
8138 recipients = string_sprintf("%s%s%s", recipients,
8139 recipients[0] ? "," : "", s);
8140 }
059ec3d9
PH
8141 }
8142
8143 /* Send a warning message if the conditions are right. If the condition check
8144 fails because of a lookup defer, there is nothing we can do. The warning
8145 is not sent. Another attempt will be made at the next delivery attempt (if
8146 it also defers). */
8147
8768d548 8148 if ( !f.queue_2stage
de6f74f2 8149 && want_warning_msg
ea97267c
JH
8150 && ( !(addr_defer->dsn_flags & rf_dsnflags)
8151 || addr_defer->dsn_flags & rf_notify_delay
50dc7409
JH
8152 )
8153 && delay_warning[1] > 0
8154 && sender_address[0] != 0
d6c82d7b 8155 && ( !delay_warning_condition
50dc7409
JH
8156 || expand_check_condition(delay_warning_condition,
8157 US"delay_warning", US"option")
8158 )
8159 )
059ec3d9
PH
8160 {
8161 int count;
8162 int show_time;
32dfdf8b 8163 int queue_time = time(NULL) - received_time.tv_sec;
059ec3d9
PH
8164
8165 /* When running in the test harness, there's an option that allows us to
8166 fudge this time so as to get repeatability of the tests. Take the first
8167 time off the list. In queue runs, the list pointer gets updated in the
8168 calling process. */
8169
8768d548 8170 if (f.running_in_test_harness && fudged_queue_times[0] != 0)
059ec3d9
PH
8171 {
8172 int qt = readconf_readtime(fudged_queue_times, '/', FALSE);
8173 if (qt >= 0)
8174 {
8175 DEBUG(D_deliver) debug_printf("fudged queue_times = %s\n",
8176 fudged_queue_times);
8177 queue_time = qt;
8178 }
8179 }
8180
8181 /* See how many warnings we should have sent by now */
8182
8183 for (count = 0; count < delay_warning[1]; count++)
8184 if (queue_time < delay_warning[count+2]) break;
8185
8186 show_time = delay_warning[count+1];
8187
8188 if (count >= delay_warning[1])
8189 {
8190 int extra;
8191 int last_gap = show_time;
8192 if (count > 1) last_gap -= delay_warning[count];
8193 extra = (queue_time - delay_warning[count+1])/last_gap;
8194 show_time += last_gap * extra;
8195 count += extra;
8196 }
8197
8198 DEBUG(D_deliver)
8199 {
de6f74f2 8200 debug_printf("time on queue = %s id %s addr %s\n", readconf_printtime(queue_time), message_id, addr_defer->address);
059ec3d9
PH
8201 debug_printf("warning counts: required %d done %d\n", count,
8202 warning_count);
8203 }
8204
8205 /* We have computed the number of warnings there should have been by now.
8206 If there haven't been enough, send one, and up the count to what it should
8207 have been. */
8208
8209 if (warning_count < count)
8210 {
8211 header_line *h;
8212 int fd;
8213 pid_t pid = child_open_exim(&fd);
8214
8215 if (pid > 0)
8216 {
8217 uschar *wmf_text;
8218 FILE *wmf = NULL;
8219 FILE *f = fdopen(fd, "wb");
45500060 8220 uschar * bound;
cab0c277 8221 transport_ctx tctx = {{0}};
059ec3d9 8222
c007c974 8223 if (warn_message_file)
d6c82d7b 8224 if (!(wmf = Ufopen(warn_message_file, "rb")))
059ec3d9
PH
8225 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to open %s for warning "
8226 "message texts: %s", warn_message_file, strerror(errno));
059ec3d9
PH
8227
8228 warnmsg_recipients = recipients;
d6c82d7b
JH
8229 warnmsg_delay = queue_time < 120*60
8230 ? string_sprintf("%d minutes", show_time/60)
8231 : string_sprintf("%d hours", show_time/3600);
059ec3d9 8232
c007c974 8233 if (errors_reply_to)
059ec3d9 8234 fprintf(f, "Reply-To: %s\n", errors_reply_to);
456682f5 8235 fprintf(f, "Auto-Submitted: auto-replied\n");
0e22dfd1 8236 moan_write_from(f);
059ec3d9
PH
8237 fprintf(f, "To: %s\n", recipients);
8238
6c1c3d1d 8239 /* generated boundary string and output MIME-Headers */
45500060 8240 bound = string_sprintf(TIME_T_FMT "-eximdsn-%d", time(NULL), rand());
c007c974
JH
8241
8242 fprintf(f, "Content-Type: multipart/report;"
8243 " report-type=delivery-status; boundary=%s\n"
8244 "MIME-Version: 1.0\n",
45500060 8245 bound);
6c1c3d1d 8246
c007c974 8247 if ((wmf_text = next_emf(wmf, US"header")))
059ec3d9
PH
8248 fprintf(f, "%s\n", wmf_text);
8249 else
8250 fprintf(f, "Subject: Warning: message %s delayed %s\n\n",
8251 message_id, warnmsg_delay);
8252
6c1c3d1d 8253 /* output human readable part as text/plain section */
c007c974
JH
8254 fprintf(f, "--%s\n"
8255 "Content-type: text/plain; charset=us-ascii\n\n",
45500060 8256 bound);
6c1c3d1d 8257
c007c974
JH
8258 if ((wmf_text = next_emf(wmf, US"intro")))
8259 fprintf(f, "%s", CS wmf_text);
8260 else
059ec3d9
PH
8261 {
8262 fprintf(f,
8263"This message was created automatically by mail delivery software.\n");
8264
8265 if (Ustrcmp(recipients, sender_address) == 0)
8266 fprintf(f,
8267"A message that you sent has not yet been delivered to one or more of its\n"
8268"recipients after more than ");
8269
c007c974
JH
8270 else
8271 fprintf(f,
059ec3d9
PH
8272"A message sent by\n\n <%s>\n\n"
8273"has not yet been delivered to one or more of its recipients after more than \n",
c007c974 8274 sender_address);
059ec3d9 8275
c007c974
JH
8276 fprintf(f, "%s on the queue on %s.\n\n"
8277 "The message identifier is: %s\n",
8278 warnmsg_delay, primary_hostname, message_id);
059ec3d9 8279
d6c82d7b 8280 for (h = header_list; h; h = h->next)
059ec3d9
PH
8281 if (strncmpic(h->text, US"Subject:", 8) == 0)
8282 fprintf(f, "The subject of the message is: %s", h->text + 9);
8283 else if (strncmpic(h->text, US"Date:", 5) == 0)
8284 fprintf(f, "The date of the message is: %s", h->text + 6);
c007c974 8285 fputc('\n', f);
059ec3d9
PH
8286
8287 fprintf(f, "The address%s to which the message has not yet been "
8288 "delivered %s:\n",
c007c974
JH
8289 !addr_defer->next ? "" : "es",
8290 !addr_defer->next ? "is": "are");
059ec3d9
PH
8291 }
8292
447d236c 8293 /* List the addresses, with error information if allowed */
059ec3d9 8294
6c1c3d1d
WB
8295 /* store addr_defer for machine readable part */
8296 address_item *addr_dsndefer = addr_defer;
c007c974
JH
8297 fputc('\n', f);
8298 while (addr_defer)
059ec3d9
PH
8299 {
8300 address_item *addr = addr_defer;
8301 addr_defer = addr->next;
447d236c
PH
8302 if (print_address_information(addr, f, US" ", US"\n ", US""))
8303 print_address_error(addr, f, US"Delay reason: ");
c007c974 8304 fputc('\n', f);
059ec3d9 8305 }
c007c974 8306 fputc('\n', f);
059ec3d9
PH
8307
8308 /* Final text */
8309
c007c974 8310 if (wmf)
059ec3d9 8311 {
c007c974
JH
8312 if ((wmf_text = next_emf(wmf, US"final")))
8313 fprintf(f, "%s", CS wmf_text);
f1e894f3 8314 (void)fclose(wmf);
059ec3d9
PH
8315 }
8316 else
8317 {
8318 fprintf(f,
8319"No action is required on your part. Delivery attempts will continue for\n"
8320"some time, and this warning may be repeated at intervals if the message\n"
8321"remains undelivered. Eventually the mail delivery software will give up,\n"
8322"and when that happens, the message will be returned to you.\n");
8323 }
8324
6c1c3d1d 8325 /* output machine readable part */
c007c974
JH
8326 fprintf(f, "\n--%s\n"
8327 "Content-type: message/delivery-status\n\n"
8328 "Reporting-MTA: dns; %s\n",
45500060 8329 bound,
c007c974 8330 smtp_active_hostname);
94431adb 8331
c007c974
JH
8332
8333 if (dsn_envid)
8334 {
6c1c3d1d
WB
8335 /* must be decoded from xtext: see RFC 3461:6.3a */
8336 uschar *xdec_envid;
8337 if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0)
8338 fprintf(f,"Original-Envelope-ID: %s\n", dsn_envid);
8339 else
968178ac 8340 fprintf(f,"X-Original-Envelope-ID: error decoding xtext formatted ENVID\n");
6c1c3d1d 8341 }
c007c974 8342 fputc('\n', f);
6c1c3d1d 8343
1c5b1499 8344 for ( ; addr_dsndefer; addr_dsndefer = addr_dsndefer->next)
6c1c3d1d 8345 {
c007c974 8346 if (addr_dsndefer->dsn_orcpt)
1c5b1499 8347 fprintf(f, "Original-Recipient: %s\n", addr_dsndefer->dsn_orcpt);
c007c974 8348
1c5b1499
JH
8349 fprintf(f, "Action: delayed\n"
8350 "Final-Recipient: rfc822;%s\n"
8351 "Status: 4.0.0\n",
8352 addr_dsndefer->address);
c007c974 8353 if (addr_dsndefer->host_used && addr_dsndefer->host_used->name)
21bc4865 8354 {
94431adb 8355 fprintf(f, "Remote-MTA: dns; %s\n",
21bc4865
WB
8356 addr_dsndefer->host_used->name);
8357 print_dsn_diagnostic_code(addr_dsndefer, f);
8358 }
1c5b1499 8359 fputc('\n', f);
6c1c3d1d
WB
8360 }
8361
1c5b1499 8362 fprintf(f, "--%s\n"
c007c974 8363 "Content-type: text/rfc822-headers\n\n",
45500060 8364 bound);
6c1c3d1d
WB
8365
8366 fflush(f);
8367 /* header only as required by RFC. only failure DSN needs to honor RET=FULL */
42055a33 8368 tctx.u.fd = fileno(f);
6b46ecc6 8369 tctx.options = topt_add_return_path | topt_no_body;
6c1c3d1d
WB
8370 transport_filter_argv = NULL; /* Just in case */
8371 return_path = sender_address; /* In case not previously set */
6b46ecc6 8372
6c1c3d1d 8373 /* Write the original email out */
ee8b8090 8374 /*XXX no checking for failure! buggy! */
42055a33 8375 transport_write_message(&tctx, 0);
6c1c3d1d
WB
8376 fflush(f);
8377
45500060 8378 fprintf(f,"\n--%s--\n", bound);
6c1c3d1d
WB
8379
8380 fflush(f);
6c1c3d1d 8381
059ec3d9
PH
8382 /* Close and wait for child process to complete, without a timeout.
8383 If there's an error, don't update the count. */
8384
f1e894f3 8385 (void)fclose(f);
059ec3d9
PH
8386 if (child_close(pid, 0) == 0)
8387 {
8388 warning_count = count;
8389 update_spool = TRUE; /* Ensure spool rewritten */
8390 }
8391 }
8392 }
8393 }
8394
8395 /* Clear deliver_domain */
8396
8397 deliver_domain = NULL;
8398
8399 /* If this was a first delivery attempt, unset the first time flag, and
8400 ensure that the spool gets updated. */
8401
8768d548 8402 if (f.deliver_firsttime)
059ec3d9 8403 {
8768d548 8404 f.deliver_firsttime = FALSE;
059ec3d9
PH
8405 update_spool = TRUE;
8406 }
8407
8408 /* If delivery was frozen and freeze_tell is set, generate an appropriate
8409 message, unless the message is a local error message (to avoid loops). Then
8410 log the freezing. If the text in "frozen_info" came from a system filter,
8411 it has been escaped into printing characters so as not to mess up log lines.
8412 For the "tell" message, we turn \n back into newline. Also, insert a newline
8413 near the start instead of the ": " string. */
8414
8768d548 8415 if (f.deliver_freeze)
059ec3d9 8416 {
8768d548 8417 if (freeze_tell && freeze_tell[0] != 0 && !f.local_error_message)
059ec3d9
PH
8418 {
8419 uschar *s = string_copy(frozen_info);
8420 uschar *ss = Ustrstr(s, " by the system filter: ");
8421
8422 if (ss != NULL)
8423 {
8424 ss[21] = '.';
8425 ss[22] = '\n';
8426 }
8427
8428 ss = s;
8429 while (*ss != 0)
8430 {
8431 if (*ss == '\\' && ss[1] == 'n')
8432 {
8433 *ss++ = ' ';
8434 *ss++ = '\n';
8435 }
8436 else ss++;
8437 }
8438 moan_tell_someone(freeze_tell, addr_defer, US"Message frozen",
8439 "Message %s has been frozen%s.\nThe sender is <%s>.\n", message_id,
8440 s, sender_address);
8441 }
8442
8443 /* Log freezing just before we update the -H file, to minimize the chance
8444 of a race problem. */
8445
8446 deliver_msglog("*** Frozen%s\n", frozen_info);
8447 log_write(0, LOG_MAIN, "Frozen%s", frozen_info);
8448 }
8449
8450 /* If there have been any updates to the non-recipients list, or other things
8451 that get written to the spool, we must now update the spool header file so
8452 that it has the right information for the next delivery attempt. If there
8453 was more than one address being delivered, the header_change update is done
8454 earlier, in case one succeeds and then something crashes. */
8455
8456 DEBUG(D_deliver)
8457 debug_printf("delivery deferred: update_spool=%d header_rewritten=%d\n",
8768d548 8458 update_spool, f.header_rewritten);
059ec3d9 8459
8768d548 8460 if (update_spool || f.header_rewritten)
059ec3d9
PH
8461 /* Panic-dies on error */
8462 (void)spool_write_header(message_id, SW_DELIVERING, NULL);
8463 }
8464
8465/* Finished with the message log. If the message is complete, it will have
8466been unlinked or renamed above. */
8467
f1e894f3 8468if (message_logs) (void)fclose(message_log);
059ec3d9
PH
8469
8470/* Now we can close and remove the journal file. Its only purpose is to record
8471successfully completed deliveries asap so that this information doesn't get
8472lost if Exim (or the machine) crashes. Forgetting about a failed delivery is
8473not serious, as trying it again is not harmful. The journal might not be open
8474if all addresses were deferred at routing or directing. Nevertheless, we must
8475remove it if it exists (may have been lying around from a crash during the
8476previous delivery attempt). We don't remove the journal if a delivery
8477subprocess failed to pass back delivery information; this is controlled by
8478the remove_journal flag. When the journal is left, we also don't move the
8479message off the main spool if frozen and the option is set. It should get moved
8480at the next attempt, after the journal has been inspected. */
8481
f1e894f3 8482if (journal_fd >= 0) (void)close(journal_fd);
059ec3d9
PH
8483
8484if (remove_journal)
8485 {
41313d92
JH
8486 uschar * fname = spool_fname(US"input", message_subdir, id, US"-J");
8487
8488 if (Uunlink(fname) < 0 && errno != ENOENT)
8489 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s", fname,
059ec3d9
PH
8490 strerror(errno));
8491
4c04137d 8492 /* Move the message off the spool if requested */
059ec3d9 8493
a7538db1 8494#ifdef SUPPORT_MOVE_FROZEN_MESSAGES
8768d548 8495 if (f.deliver_freeze && move_frozen_messages)
059ec3d9 8496 (void)spool_move_message(id, message_subdir, US"", US"F");
a7538db1 8497#endif
059ec3d9
PH
8498 }
8499
8500/* Closing the data file frees the lock; if the file has been unlinked it
8501will go away. Otherwise the message becomes available for another process
8502to try delivery. */
8503
f1e894f3 8504(void)close(deliver_datafile);
059ec3d9
PH
8505deliver_datafile = -1;
8506DEBUG(D_deliver) debug_printf("end delivery of %s\n", id);
8507
8508/* It is unlikely that there will be any cached resources, since they are
8509released after routing, and in the delivery subprocesses. However, it's
8510possible for an expansion for something afterwards (for example,
8511expand_check_condition) to do a lookup. We must therefore be sure everything is
8512released. */
8513
8514search_tidyup();
faa05a93 8515acl_where = ACL_WHERE_UNKNOWN;
059ec3d9
PH
8516return final_yield;
8517}
8518
476be7e2
JH
8519
8520
8521void
8522deliver_init(void)
8523{
10ac8d7f
JH
8524#ifdef EXIM_TFO_PROBE
8525tfo_probe();
8526#else
8768d548 8527f.tcp_fastopen_ok = TRUE;
10ac8d7f
JH
8528#endif
8529
8530
476be7e2
JH
8531if (!regex_PIPELINING) regex_PIPELINING =
8532 regex_must_compile(US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)", FALSE, TRUE);
8533
8534if (!regex_SIZE) regex_SIZE =
8535 regex_must_compile(US"\\n250[\\s\\-]SIZE(\\s|\\n|$)", FALSE, TRUE);
8536
8537if (!regex_AUTH) regex_AUTH =
ee8b8090 8538 regex_must_compile(AUTHS_REGEX, FALSE, TRUE);
476be7e2
JH
8539
8540#ifdef SUPPORT_TLS
8541if (!regex_STARTTLS) regex_STARTTLS =
8542 regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE);
8ac90765
JH
8543
8544# ifdef EXPERIMENTAL_REQUIRETLS
8545if (!regex_REQUIRETLS) regex_REQUIRETLS =
8546 regex_must_compile(US"\\n250[\\s\\-]REQUIRETLS(\\s|\\n|$)", FALSE, TRUE);
8547# endif
476be7e2
JH
8548#endif
8549
f98442df
JH
8550if (!regex_CHUNKING) regex_CHUNKING =
8551 regex_must_compile(US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)", FALSE, TRUE);
8552
476be7e2
JH
8553#ifndef DISABLE_PRDR
8554if (!regex_PRDR) regex_PRDR =
8555 regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE);
8556#endif
8557
8c5d388a 8558#ifdef SUPPORT_I18N
7ade712c
JH
8559if (!regex_UTF8) regex_UTF8 =
8560 regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE);
8561#endif
8562
476be7e2
JH
8563if (!regex_DSN) regex_DSN =
8564 regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE);
8565
8566if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA =
8567 regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE);
ee8b8090
JH
8568
8569#ifdef EXPERIMENTAL_PIPE_CONNECT
8570if (!regex_EARLY_PIPE) regex_EARLY_PIPE =
8571 regex_must_compile(US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)", FALSE, TRUE);
8572#endif
476be7e2
JH
8573}
8574
8575
a39bd74d
JB
8576uschar *
8577deliver_get_sender_address (uschar * id)
8578{
dadff1d4
HSHR
8579int rc;
8580uschar * new_sender_address,
8581 * save_sender_address;
8768d548 8582BOOL save_qr = f.queue_running;
41313d92 8583uschar * spoolname;
dadff1d4 8584
789f8a4f
JH
8585/* make spool_open_datafile non-noisy on fail */
8586
8768d548 8587f.queue_running = TRUE;
789f8a4f
JH
8588
8589/* Side effect: message_subdir is set for the (possibly split) spool directory */
8590
8591deliver_datafile = spool_open_datafile(id);
8768d548 8592f.queue_running = save_qr;
789f8a4f 8593if (deliver_datafile < 0)
a39bd74d
JB
8594 return NULL;
8595
dadff1d4
HSHR
8596/* Save and restore the global sender_address. I'm not sure if we should
8597not save/restore all the other global variables too, because
8598spool_read_header() may change all of them. But OTOH, when this
8599deliver_get_sender_address() gets called, the current message is done
8600already and nobody needs the globals anymore. (HS12, 2015-08-21) */
8601
41313d92 8602spoolname = string_sprintf("%s-H", id);
dadff1d4
HSHR
8603save_sender_address = sender_address;
8604
8605rc = spool_read_header(spoolname, TRUE, TRUE);
8606
8607new_sender_address = sender_address;
8608sender_address = save_sender_address;
8609
8610if (rc != spool_read_OK)
a39bd74d
JB
8611 return NULL;
8612
dadff1d4
HSHR
8613assert(new_sender_address);
8614
a39bd74d
JB
8615(void)close(deliver_datafile);
8616deliver_datafile = -1;
8617
dadff1d4 8618return new_sender_address;
a39bd74d
JB
8619}
8620
57cc2785
JH
8621
8622
8623void
8624delivery_re_exec(int exec_type)
8625{
cab0c277 8626uschar * where;
57cc2785 8627
74f1a423 8628if (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only)
57cc2785 8629 {
48334568 8630 int channel_fd = cutthrough.cctx.sock;
57cc2785
JH
8631
8632 smtp_peer_options = cutthrough.peer_options;
8633 continue_sequence = 0;
8634
8635#ifdef SUPPORT_TLS
8636 if (cutthrough.is_tls)
8637 {
48334568
JH
8638 int pfd[2], pid;
8639
14de8063 8640 smtp_peer_options |= OPTION_TLS;
57cc2785
JH
8641 sending_ip_address = cutthrough.snd_ip;
8642 sending_port = cutthrough.snd_port;
8643
cab0c277 8644 where = US"socketpair";
57cc2785
JH
8645 if (socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) != 0)
8646 goto fail;
8647
cab0c277 8648 where = US"fork";
57cc2785
JH
8649 if ((pid = fork()) < 0)
8650 goto fail;
8651
6c454556 8652 else if (pid == 0) /* child: fork again to totally disconnect */
5facbeaa 8653 {
8768d548 8654 if (f.running_in_test_harness) millisleep(100); /* let parent debug out */
d097cc73 8655 /* does not return */
74f1a423
JH
8656 smtp_proxy_tls(cutthrough.cctx.tls_ctx, big_buffer, big_buffer_size,
8657 pfd, 5*60);
5facbeaa 8658 }
57cc2785 8659
d097cc73 8660 DEBUG(D_transport) debug_printf("proxy-proc inter-pid %d\n", pid);
e47252f5 8661 close(pfd[0]);
60272099 8662 waitpid(pid, NULL, 0);
57cc2785
JH
8663 (void) close(channel_fd); /* release the client socket */
8664 channel_fd = pfd[1];
8665 }
8666#endif
8667
8668 transport_do_pass_socket(cutthrough.transport, cutthrough.host.name,
8669 cutthrough.host.address, message_id, channel_fd);
8670 }
8671else
8672 {
ef698bf6 8673 cancel_cutthrough_connection(TRUE, US"non-continued delivery");
57cc2785
JH
8674 (void) child_exec_exim(exec_type, FALSE, NULL, FALSE, 2, US"-Mc", message_id);
8675 }
cab0c277 8676return; /* compiler quietening; control does not reach here. */
57cc2785 8677
13c7874e 8678#ifdef SUPPORT_TLS
57cc2785
JH
8679fail:
8680 log_write(0,
8681 LOG_MAIN | (exec_type == CEE_EXEC_EXIT ? LOG_PANIC : LOG_PANIC_DIE),
cab0c277 8682 "delivery re-exec %s failed: %s", where, strerror(errno));
57cc2785
JH
8683
8684 /* Get here if exec_type == CEE_EXEC_EXIT.
8685 Note: this must be _exit(), not exit(). */
8686
8687 _exit(EX_EXECFAILED);
13c7874e 8688#endif
57cc2785
JH
8689}
8690
67d81c10
JH
8691/* vi: aw ai sw=2
8692*/
059ec3d9 8693/* End of deliver.c */