1748c798511ffad9fb929b41cd6091f6efb8ef89
[exim.git] / src / src / deliver.c
1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
4
5 /* Copyright (c) University of Cambridge 1995 - 2018 */
6 /* See the file NOTICE for conditions of use and distribution. */
7
8 /* The main code for delivering a message. */
9
10
11 #include "exim.h"
12 #include "transports/smtp.h"
13 #include <sys/uio.h>
14 #include <assert.h>
15
16
17 /* Data block for keeping track of subprocesses for parallel remote
18 delivery. */
19
20 typedef struct pardata {
21 address_item *addrlist; /* chain of addresses */
22 address_item *addr; /* next address data expected for */
23 pid_t pid; /* subprocess pid */
24 int fd; /* pipe fd for getting result from subprocess */
25 int transport_count; /* returned transport count value */
26 BOOL done; /* no more data needed */
27 uschar *msg; /* error message */
28 uschar *return_path; /* return_path for these addresses */
29 } pardata;
30
31 /* Values for the process_recipients variable */
32
33 enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
34 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
35 RECIP_FAIL_LOOP};
36
37 /* Mutually recursive functions for marking addresses done. */
38
39 static void child_done(address_item *, uschar *);
40 static void address_done(address_item *, uschar *);
41
42 /* Table for turning base-62 numbers into binary */
43
44 static uschar tab62[] =
45 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
46 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
47 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
48 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
49 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
50 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
51 59,60,61}; /* x-z */
52
53
54 /*************************************************
55 * Local static variables *
56 *************************************************/
57
58 /* addr_duplicate is global because it needs to be seen from the Envelope-To
59 writing code. */
60
61 static address_item *addr_defer = NULL;
62 static address_item *addr_failed = NULL;
63 static address_item *addr_fallback = NULL;
64 static address_item *addr_local = NULL;
65 static address_item *addr_new = NULL;
66 static address_item *addr_remote = NULL;
67 static address_item *addr_route = NULL;
68 static address_item *addr_succeed = NULL;
69 static address_item *addr_senddsn = NULL;
70
71 static FILE *message_log = NULL;
72 static BOOL update_spool;
73 static BOOL remove_journal;
74 static int parcount = 0;
75 static pardata *parlist = NULL;
76 static int return_count;
77 static uschar *frozen_info = US"";
78 static uschar *used_return_path = NULL;
79
80
81
82 /*************************************************
83 * read as much as requested *
84 *************************************************/
85
86 /* The syscall read(2) doesn't always returns as much as we want. For
87 several reasons it might get less. (Not talking about signals, as syscalls
88 are restartable). When reading from a network or pipe connection the sender
89 might send in smaller chunks, with delays between these chunks. The read(2)
90 may return such a chunk.
91
92 The more the writer writes and the smaller the pipe between write and read is,
93 the more we get the chance of reading leass than requested. (See bug 2130)
94
95 This function read(2)s until we got all the data we *requested*.
96
97 Note: This function may block. Use it only if you're sure about the
98 amount of data you will get.
99
100 Argument:
101 fd the file descriptor to read from
102 buffer pointer to a buffer of size len
103 len the requested(!) amount of bytes
104
105 Returns: the amount of bytes read
106 */
107 static ssize_t
108 readn(int fd, void * buffer, size_t len)
109 {
110 uschar * next = buffer;
111 uschar * end = next + len;
112
113 while (next < end)
114 {
115 ssize_t got = read(fd, next, end - next);
116
117 /* I'm not sure if there are signals that can interrupt us,
118 for now I assume the worst */
119 if (got == -1 && errno == EINTR) continue;
120 if (got <= 0) return next - US buffer;
121 next += got;
122 }
123
124 return len;
125 }
126
127
128 /*************************************************
129 * Make a new address item *
130 *************************************************/
131
132 /* This function gets the store and initializes with default values. The
133 transport_return value defaults to DEFER, so that any unexpected failure to
134 deliver does not wipe out the message. The default unique string is set to a
135 copy of the address, so that its domain can be lowercased.
136
137 Argument:
138 address the RFC822 address string
139 copy force a copy of the address
140
141 Returns: a pointer to an initialized address_item
142 */
143
144 address_item *
145 deliver_make_addr(uschar *address, BOOL copy)
146 {
147 address_item *addr = store_get(sizeof(address_item), FALSE);
148 *addr = address_defaults;
149 if (copy) address = string_copy(address);
150 addr->address = address;
151 addr->unique = string_copy(address);
152 return addr;
153 }
154
155
156
157
158 /*************************************************
159 * Set expansion values for an address *
160 *************************************************/
161
162 /* Certain expansion variables are valid only when handling an address or
163 address list. This function sets them up or clears the values, according to its
164 argument.
165
166 Arguments:
167 addr the address in question, or NULL to clear values
168 Returns: nothing
169 */
170
171 void
172 deliver_set_expansions(address_item *addr)
173 {
174 if (!addr)
175 {
176 const uschar ***p = address_expansions;
177 while (*p) **p++ = NULL;
178 return;
179 }
180
181 /* Exactly what gets set depends on whether there is one or more addresses, and
182 what they contain. These first ones are always set, taking their values from
183 the first address. */
184
185 if (!addr->host_list)
186 {
187 deliver_host = deliver_host_address = US"";
188 deliver_host_port = 0;
189 }
190 else
191 {
192 deliver_host = addr->host_list->name;
193 deliver_host_address = addr->host_list->address;
194 deliver_host_port = addr->host_list->port;
195 }
196
197 deliver_recipients = addr;
198 deliver_address_data = addr->prop.address_data;
199 deliver_domain_data = addr->prop.domain_data;
200 deliver_localpart_data = addr->prop.localpart_data;
201 router_var = addr->prop.variables;
202
203 /* These may be unset for multiple addresses */
204
205 deliver_domain = addr->domain;
206 self_hostname = addr->self_hostname;
207
208 #ifdef EXPERIMENTAL_BRIGHTMAIL
209 bmi_deliver = 1; /* deliver by default */
210 bmi_alt_location = NULL;
211 bmi_base64_verdict = NULL;
212 bmi_base64_tracker_verdict = NULL;
213 #endif
214
215 /* If there's only one address we can set everything. */
216
217 if (!addr->next)
218 {
219 address_item *addr_orig;
220
221 deliver_localpart = addr->local_part;
222 deliver_localpart_prefix = addr->prefix;
223 deliver_localpart_prefix_v = addr->prefix_v;
224 deliver_localpart_suffix = addr->suffix;
225 deliver_localpart_suffix_v = addr->suffix_v;
226
227 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
228 deliver_domain_orig = addr_orig->domain;
229
230 /* Re-instate any prefix and suffix in the original local part. In all
231 normal cases, the address will have a router associated with it, and we can
232 choose the caseful or caseless version accordingly. However, when a system
233 filter sets up a pipe, file, or autoreply delivery, no router is involved.
234 In this case, though, there won't be any prefix or suffix to worry about. */
235
236 deliver_localpart_orig = !addr_orig->router
237 ? addr_orig->local_part
238 : addr_orig->router->caseful_local_part
239 ? addr_orig->cc_local_part
240 : addr_orig->lc_local_part;
241
242 /* If there's a parent, make its domain and local part available, and if
243 delivering to a pipe or file, or sending an autoreply, get the local
244 part from the parent. For pipes and files, put the pipe or file string
245 into address_pipe and address_file. */
246
247 if (addr->parent)
248 {
249 deliver_domain_parent = addr->parent->domain;
250 deliver_localpart_parent = !addr->parent->router
251 ? addr->parent->local_part
252 : addr->parent->router->caseful_local_part
253 ? addr->parent->cc_local_part
254 : addr->parent->lc_local_part;
255
256 /* File deliveries have their own flag because they need to be picked out
257 as special more often. */
258
259 if (testflag(addr, af_pfr))
260 {
261 if (testflag(addr, af_file)) address_file = addr->local_part;
262 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
263 deliver_localpart = addr->parent->local_part;
264 deliver_localpart_prefix = addr->parent->prefix;
265 deliver_localpart_prefix_v = addr->parent->prefix_v;
266 deliver_localpart_suffix = addr->parent->suffix;
267 deliver_localpart_suffix_v = addr->parent->suffix_v;
268 }
269 }
270
271 #ifdef EXPERIMENTAL_BRIGHTMAIL
272 /* Set expansion variables related to Brightmail AntiSpam */
273 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
274 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
275 /* get message delivery status (0 - don't deliver | 1 - deliver) */
276 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
277 /* if message is to be delivered, get eventual alternate location */
278 if (bmi_deliver == 1)
279 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
280 #endif
281
282 }
283
284 /* For multiple addresses, don't set local part, and leave the domain and
285 self_hostname set only if it is the same for all of them. It is possible to
286 have multiple pipe and file addresses, but only when all addresses have routed
287 to the same pipe or file. */
288
289 else
290 {
291 if (testflag(addr, af_pfr))
292 {
293 if (testflag(addr, af_file)) address_file = addr->local_part;
294 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
295 }
296 for (address_item * addr2 = addr->next; addr2; addr2 = addr2->next)
297 {
298 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
299 deliver_domain = NULL;
300 if ( self_hostname
301 && ( !addr2->self_hostname
302 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
303 ) )
304 self_hostname = NULL;
305 if (!deliver_domain && !self_hostname) break;
306 }
307 }
308 }
309
310
311
312
313 /*************************************************
314 * Open a msglog file *
315 *************************************************/
316
317 /* This function is used both for normal message logs, and for files in the
318 msglog directory that are used to catch output from pipes. Try to create the
319 directory if it does not exist. From release 4.21, normal message logs should
320 be created when the message is received.
321
322 Called from deliver_message(), can be operating as root.
323
324 Argument:
325 filename the file name
326 mode the mode required
327 error used for saying what failed
328
329 Returns: a file descriptor, or -1 (with errno set)
330 */
331
332 static int
333 open_msglog_file(uschar *filename, int mode, uschar **error)
334 {
335 if (Ustrstr(filename, US"/../"))
336 log_write(0, LOG_MAIN|LOG_PANIC,
337 "Attempt to open msglog file path with upward-traversal: '%s'\n", filename);
338
339 for (int i = 2; i > 0; i--)
340 {
341 int fd = Uopen(filename,
342 #ifdef O_CLOEXEC
343 O_CLOEXEC |
344 #endif
345 #ifdef O_NOFOLLOW
346 O_NOFOLLOW |
347 #endif
348 O_WRONLY|O_APPEND|O_CREAT, mode);
349 if (fd >= 0)
350 {
351 /* Set the close-on-exec flag and change the owner to the exim uid/gid (this
352 function is called as root). Double check the mode, because the group setting
353 doesn't always get set automatically. */
354
355 #ifndef O_CLOEXEC
356 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
357 #endif
358 if (exim_fchown(fd, exim_uid, exim_gid, filename) < 0)
359 {
360 *error = US"chown";
361 return -1;
362 }
363 if (fchmod(fd, mode) < 0)
364 {
365 *error = US"chmod";
366 return -1;
367 }
368 return fd;
369 }
370 if (errno != ENOENT)
371 break;
372
373 (void)directory_make(spool_directory,
374 spool_sname(US"msglog", message_subdir),
375 MSGLOG_DIRECTORY_MODE, TRUE);
376 }
377
378 *error = US"create or open";
379 return -1;
380 }
381
382
383
384
385 /*************************************************
386 * Write to msglog if required *
387 *************************************************/
388
389 /* Write to the message log, if configured. This function may also be called
390 from transports.
391
392 Arguments:
393 format a string format
394
395 Returns: nothing
396 */
397
398 void
399 deliver_msglog(const char *format, ...)
400 {
401 va_list ap;
402 if (!message_logs) return;
403 va_start(ap, format);
404 vfprintf(message_log, format, ap);
405 fflush(message_log);
406 va_end(ap);
407 }
408
409
410
411
412 /*************************************************
413 * Replicate status for batch *
414 *************************************************/
415
416 /* When a transport handles a batch of addresses, it may treat them
417 individually, or it may just put the status in the first one, and return FALSE,
418 requesting that the status be copied to all the others externally. This is the
419 replication function. As well as the status, it copies the transport pointer,
420 which may have changed if appendfile passed the addresses on to a different
421 transport.
422
423 Argument: pointer to the first address in a chain
424 Returns: nothing
425 */
426
427 static void
428 replicate_status(address_item *addr)
429 {
430 for (address_item * addr2 = addr->next; addr2; addr2 = addr2->next)
431 {
432 addr2->transport = addr->transport;
433 addr2->transport_return = addr->transport_return;
434 addr2->basic_errno = addr->basic_errno;
435 addr2->more_errno = addr->more_errno;
436 addr2->delivery_time = addr->delivery_time;
437 addr2->special_action = addr->special_action;
438 addr2->message = addr->message;
439 addr2->user_message = addr->user_message;
440 }
441 }
442
443
444
445 /*************************************************
446 * Compare lists of hosts *
447 *************************************************/
448
449 /* This function is given two pointers to chains of host items, and it yields
450 TRUE if the lists refer to the same hosts in the same order, except that
451
452 (1) Multiple hosts with the same non-negative MX values are permitted to appear
453 in different orders. Round-robinning nameservers can cause this to happen.
454
455 (2) Multiple hosts with the same negative MX values less than MX_NONE are also
456 permitted to appear in different orders. This is caused by randomizing
457 hosts lists.
458
459 This enables Exim to use a single SMTP transaction for sending to two entirely
460 different domains that happen to end up pointing at the same hosts.
461
462 Arguments:
463 one points to the first host list
464 two points to the second host list
465
466 Returns: TRUE if the lists refer to the same host set
467 */
468
469 static BOOL
470 same_hosts(host_item *one, host_item *two)
471 {
472 while (one && two)
473 {
474 if (Ustrcmp(one->name, two->name) != 0)
475 {
476 int mx = one->mx;
477 host_item *end_one = one;
478 host_item *end_two = two;
479
480 /* Batch up only if there was no MX and the list was not randomized */
481
482 if (mx == MX_NONE) return FALSE;
483
484 /* Find the ends of the shortest sequence of identical MX values */
485
486 while ( end_one->next && end_one->next->mx == mx
487 && end_two->next && end_two->next->mx == mx)
488 {
489 end_one = end_one->next;
490 end_two = end_two->next;
491 }
492
493 /* If there aren't any duplicates, there's no match. */
494
495 if (end_one == one) return FALSE;
496
497 /* For each host in the 'one' sequence, check that it appears in the 'two'
498 sequence, returning FALSE if not. */
499
500 for (;;)
501 {
502 host_item *hi;
503 for (hi = two; hi != end_two->next; hi = hi->next)
504 if (Ustrcmp(one->name, hi->name) == 0) break;
505 if (hi == end_two->next) return FALSE;
506 if (one == end_one) break;
507 one = one->next;
508 }
509
510 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
511 Ensure both are pointing at the last host, and carry on as for equality. */
512
513 two = end_two;
514 }
515
516 /* if the names matched but ports do not, mismatch */
517 else if (one->port != two->port)
518 return FALSE;
519
520 /* Hosts matched */
521
522 one = one->next;
523 two = two->next;
524 }
525
526 /* True if both are NULL */
527
528 return (one == two);
529 }
530
531
532
533 /*************************************************
534 * Compare header lines *
535 *************************************************/
536
537 /* This function is given two pointers to chains of header items, and it yields
538 TRUE if they are the same header texts in the same order.
539
540 Arguments:
541 one points to the first header list
542 two points to the second header list
543
544 Returns: TRUE if the lists refer to the same header set
545 */
546
547 static BOOL
548 same_headers(header_line *one, header_line *two)
549 {
550 for (;; one = one->next, two = two->next)
551 {
552 if (one == two) return TRUE; /* Includes the case where both NULL */
553 if (!one || !two) return FALSE;
554 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
555 }
556 }
557
558
559
560 /*************************************************
561 * Compare string settings *
562 *************************************************/
563
564 /* This function is given two pointers to strings, and it returns
565 TRUE if they are the same pointer, or if the two strings are the same.
566
567 Arguments:
568 one points to the first string
569 two points to the second string
570
571 Returns: TRUE or FALSE
572 */
573
574 static BOOL
575 same_strings(uschar *one, uschar *two)
576 {
577 if (one == two) return TRUE; /* Includes the case where both NULL */
578 if (!one || !two) return FALSE;
579 return (Ustrcmp(one, two) == 0);
580 }
581
582
583
584 /*************************************************
585 * Compare uid/gid for addresses *
586 *************************************************/
587
588 /* This function is given a transport and two addresses. It yields TRUE if the
589 uid/gid/initgroups settings for the two addresses are going to be the same when
590 they are delivered.
591
592 Arguments:
593 tp the transort
594 addr1 the first address
595 addr2 the second address
596
597 Returns: TRUE or FALSE
598 */
599
600 static BOOL
601 same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
602 {
603 if ( !tp->uid_set && !tp->expand_uid
604 && !tp->deliver_as_creator
605 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
606 || ( testflag(addr1, af_uid_set)
607 && ( addr1->uid != addr2->uid
608 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
609 ) ) ) )
610 return FALSE;
611
612 if ( !tp->gid_set && !tp->expand_gid
613 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
614 || ( testflag(addr1, af_gid_set)
615 && addr1->gid != addr2->gid
616 ) ) )
617 return FALSE;
618
619 return TRUE;
620 }
621
622
623
624
625 /*************************************************
626 * Record that an address is complete *
627 *************************************************/
628
629 /* This function records that an address is complete. This is straightforward
630 for most addresses, where the unique address is just the full address with the
631 domain lower cased. For homonyms (addresses that are the same as one of their
632 ancestors) their are complications. Their unique addresses have \x\ prepended
633 (where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
634 cousins.
635
636 Exim used to record the unique addresses of homonyms as "complete". This,
637 however, fails when the pattern of redirection varies over time (e.g. if taking
638 unseen copies at only some times of day) because the prepended numbers may vary
639 from one delivery run to the next. This problem is solved by never recording
640 prepended unique addresses as complete. Instead, when a homonymic address has
641 actually been delivered via a transport, we record its basic unique address
642 followed by the name of the transport. This is checked in subsequent delivery
643 runs whenever an address is routed to a transport.
644
645 If the completed address is a top-level one (has no parent, which means it
646 cannot be homonymic) we also add the original address to the non-recipients
647 tree, so that it gets recorded in the spool file and therefore appears as
648 "done" in any spool listings. The original address may differ from the unique
649 address in the case of the domain.
650
651 Finally, this function scans the list of duplicates, marks as done any that
652 match this address, and calls child_done() for their ancestors.
653
654 Arguments:
655 addr address item that has been completed
656 now current time as a string
657
658 Returns: nothing
659 */
660
661 static void
662 address_done(address_item *addr, uschar *now)
663 {
664 update_spool = TRUE; /* Ensure spool gets updated */
665
666 /* Top-level address */
667
668 if (!addr->parent)
669 {
670 tree_add_nonrecipient(addr->unique);
671 tree_add_nonrecipient(addr->address);
672 }
673
674 /* Homonymous child address */
675
676 else if (testflag(addr, af_homonym))
677 {
678 if (addr->transport)
679 tree_add_nonrecipient(
680 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
681 }
682
683 /* Non-homonymous child address */
684
685 else tree_add_nonrecipient(addr->unique);
686
687 /* Check the list of duplicate addresses and ensure they are now marked
688 done as well. */
689
690 for (address_item * dup = addr_duplicate; dup; dup = dup->next)
691 if (Ustrcmp(addr->unique, dup->unique) == 0)
692 {
693 tree_add_nonrecipient(dup->unique);
694 child_done(dup, now);
695 }
696 }
697
698
699
700
701 /*************************************************
702 * Decrease counts in parents and mark done *
703 *************************************************/
704
705 /* This function is called when an address is complete. If there is a parent
706 address, its count of children is decremented. If there are still other
707 children outstanding, the function exits. Otherwise, if the count has become
708 zero, address_done() is called to mark the parent and its duplicates complete.
709 Then loop for any earlier ancestors.
710
711 Arguments:
712 addr points to the completed address item
713 now the current time as a string, for writing to the message log
714
715 Returns: nothing
716 */
717
718 static void
719 child_done(address_item *addr, uschar *now)
720 {
721 while (addr->parent)
722 {
723 address_item *aa;
724
725 addr = addr->parent;
726 if (--addr->child_count > 0) return; /* Incomplete parent */
727 address_done(addr, now);
728
729 /* Log the completion of all descendents only when there is no ancestor with
730 the same original address. */
731
732 for (aa = addr->parent; aa; aa = aa->parent)
733 if (Ustrcmp(aa->address, addr->address) == 0) break;
734 if (aa) continue;
735
736 deliver_msglog("%s %s: children all complete\n", now, addr->address);
737 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
738 }
739 }
740
741
742
743 /*************************************************
744 * Delivery logging support functions *
745 *************************************************/
746
747 /* The LOGGING() checks in d_log_interface() are complicated for backwards
748 compatibility. When outgoing interface logging was originally added, it was
749 conditional on just incoming_interface (which is off by default). The
750 outgoing_interface option is on by default to preserve this behaviour, but
751 you can enable incoming_interface and disable outgoing_interface to get I=
752 fields on incoming lines only.
753
754 Arguments:
755 g The log line
756 addr The address to be logged
757
758 Returns: New value for s
759 */
760
761 static gstring *
762 d_log_interface(gstring * g)
763 {
764 if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
765 && sending_ip_address)
766 {
767 g = string_fmt_append(g, " I=[%s]", sending_ip_address);
768 if (LOGGING(outgoing_port))
769 g = string_fmt_append(g, ":%d", sending_port);
770 }
771 return g;
772 }
773
774
775
776 static gstring *
777 d_hostlog(gstring * g, address_item * addr)
778 {
779 host_item * h = addr->host_used;
780
781 g = string_append(g, 2, US" H=", h->name);
782
783 if (LOGGING(dnssec) && h->dnssec == DS_YES)
784 g = string_catn(g, US" DS", 3);
785
786 g = string_append(g, 3, US" [", h->address, US"]");
787
788 if (LOGGING(outgoing_port))
789 g = string_fmt_append(g, ":%d", h->port);
790
791 #ifdef SUPPORT_SOCKS
792 if (LOGGING(proxy) && proxy_local_address)
793 {
794 g = string_append(g, 3, US" PRX=[", proxy_local_address, US"]");
795 if (LOGGING(outgoing_port))
796 g = string_fmt_append(g, ":%d", proxy_local_port);
797 }
798 #endif
799
800 g = d_log_interface(g);
801
802 if (testflag(addr, af_tcp_fastopen))
803 g = string_catn(g, US" TFO*", testflag(addr, af_tcp_fastopen_data) ? 5 : 4);
804
805 return g;
806 }
807
808
809
810
811
812 #ifndef DISABLE_TLS
813 static gstring *
814 d_tlslog(gstring * g, address_item * addr)
815 {
816 if (LOGGING(tls_cipher) && addr->cipher)
817 {
818 g = string_append(g, 2, US" X=", addr->cipher);
819 #ifdef EXPERIMENTAL_TLS_RESUME
820 if (LOGGING(tls_resumption) && testflag(addr, af_tls_resume))
821 g = string_catn(g, US"*", 1);
822 #endif
823 }
824 if (LOGGING(tls_certificate_verified) && addr->cipher)
825 g = string_append(g, 2, US" CV=",
826 testflag(addr, af_cert_verified)
827 ?
828 #ifdef SUPPORT_DANE
829 testflag(addr, af_dane_verified)
830 ? "dane"
831 :
832 #endif
833 "yes"
834 : "no");
835 if (LOGGING(tls_peerdn) && addr->peerdn)
836 g = string_append(g, 3, US" DN=\"", string_printing(addr->peerdn), US"\"");
837 return g;
838 }
839 #endif
840
841
842
843
844 #ifndef DISABLE_EVENT
845 uschar *
846 event_raise(uschar * action, const uschar * event, uschar * ev_data)
847 {
848 uschar * s;
849 if (action)
850 {
851 DEBUG(D_deliver)
852 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
853 event,
854 action, deliver_host_address);
855
856 event_name = event;
857 event_data = ev_data;
858
859 if (!(s = expand_string(action)) && *expand_string_message)
860 log_write(0, LOG_MAIN|LOG_PANIC,
861 "failed to expand event_action %s in %s: %s\n",
862 event, transport_name ? transport_name : US"main", expand_string_message);
863
864 event_name = event_data = NULL;
865
866 /* If the expansion returns anything but an empty string, flag for
867 the caller to modify his normal processing
868 */
869 if (s && *s)
870 {
871 DEBUG(D_deliver)
872 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
873 errno = ERRNO_EVENT;
874 return s;
875 }
876 }
877 return NULL;
878 }
879
880 void
881 msg_event_raise(const uschar * event, const address_item * addr)
882 {
883 const uschar * save_domain = deliver_domain;
884 uschar * save_local = deliver_localpart;
885 const uschar * save_host = deliver_host;
886 const uschar * save_address = deliver_host_address;
887 const int save_port = deliver_host_port;
888
889 router_name = addr->router ? addr->router->name : NULL;
890 deliver_domain = addr->domain;
891 deliver_localpart = addr->local_part;
892 deliver_host = addr->host_used ? addr->host_used->name : NULL;
893
894 if (!addr->transport)
895 {
896 if (Ustrcmp(event, "msg:fail:delivery") == 0)
897 {
898 /* An address failed with no transport involved. This happens when
899 a filter was used which triggered a fail command (in such a case
900 a transport isn't needed). Convert it to an internal fail event. */
901
902 (void) event_raise(event_action, US"msg:fail:internal", addr->message);
903 }
904 }
905 else
906 {
907 transport_name = addr->transport->name;
908
909 (void) event_raise(addr->transport->event_action, event,
910 addr->host_used
911 || Ustrcmp(addr->transport->driver_name, "smtp") == 0
912 || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
913 || Ustrcmp(addr->transport->driver_name, "autoreply") == 0
914 ? addr->message : NULL);
915 }
916
917 deliver_host_port = save_port;
918 deliver_host_address = save_address;
919 deliver_host = save_host;
920 deliver_localpart = save_local;
921 deliver_domain = save_domain;
922 router_name = transport_name = NULL;
923 }
924 #endif /*DISABLE_EVENT*/
925
926
927
928 /******************************************************************************/
929
930
931 /*************************************************
932 * Generate local prt for logging *
933 *************************************************/
934
935 /* This function is a subroutine for use in string_log_address() below.
936
937 Arguments:
938 addr the address being logged
939 yield the current dynamic buffer pointer
940
941 Returns: the new value of the buffer pointer
942 */
943
944 static gstring *
945 string_get_localpart(address_item * addr, gstring * yield)
946 {
947 uschar * s;
948
949 s = addr->prefix;
950 if (testflag(addr, af_include_affixes) && s)
951 {
952 #ifdef SUPPORT_I18N
953 if (testflag(addr, af_utf8_downcvt))
954 s = string_localpart_utf8_to_alabel(s, NULL);
955 #endif
956 yield = string_cat(yield, s);
957 }
958
959 s = addr->local_part;
960 #ifdef SUPPORT_I18N
961 if (testflag(addr, af_utf8_downcvt))
962 s = string_localpart_utf8_to_alabel(s, NULL);
963 #endif
964 yield = string_cat(yield, s);
965
966 s = addr->suffix;
967 if (testflag(addr, af_include_affixes) && s)
968 {
969 #ifdef SUPPORT_I18N
970 if (testflag(addr, af_utf8_downcvt))
971 s = string_localpart_utf8_to_alabel(s, NULL);
972 #endif
973 yield = string_cat(yield, s);
974 }
975
976 return yield;
977 }
978
979
980 /*************************************************
981 * Generate log address list *
982 *************************************************/
983
984 /* This function generates a list consisting of an address and its parents, for
985 use in logging lines. For saved onetime aliased addresses, the onetime parent
986 field is used. If the address was delivered by a transport with rcpt_include_
987 affixes set, the af_include_affixes bit will be set in the address. In that
988 case, we include the affixes here too.
989
990 Arguments:
991 g points to growing-string struct
992 addr bottom (ultimate) address
993 all_parents if TRUE, include all parents
994 success TRUE for successful delivery
995
996 Returns: a growable string in dynamic store
997 */
998
999 static gstring *
1000 string_log_address(gstring * g,
1001 address_item *addr, BOOL all_parents, BOOL success)
1002 {
1003 BOOL add_topaddr = TRUE;
1004 address_item *topaddr;
1005
1006 /* Find the ultimate parent */
1007
1008 for (topaddr = addr; topaddr->parent; topaddr = topaddr->parent) ;
1009
1010 /* We start with just the local part for pipe, file, and reply deliveries, and
1011 for successful local deliveries from routers that have the log_as_local flag
1012 set. File deliveries from filters can be specified as non-absolute paths in
1013 cases where the transport is going to complete the path. If there is an error
1014 before this happens (expansion failure) the local part will not be updated, and
1015 so won't necessarily look like a path. Add extra text for this case. */
1016
1017 if ( testflag(addr, af_pfr)
1018 || ( success
1019 && addr->router && addr->router->log_as_local
1020 && addr->transport && addr->transport->info->local
1021 ) )
1022 {
1023 if (testflag(addr, af_file) && addr->local_part[0] != '/')
1024 g = string_catn(g, CUS"save ", 5);
1025 g = string_get_localpart(addr, g);
1026 }
1027
1028 /* Other deliveries start with the full address. It we have split it into local
1029 part and domain, use those fields. Some early failures can happen before the
1030 splitting is done; in those cases use the original field. */
1031
1032 else
1033 {
1034 uschar * cmp;
1035 int off = g->ptr; /* start of the "full address" */
1036
1037 if (addr->local_part)
1038 {
1039 const uschar * s;
1040 g = string_get_localpart(addr, g);
1041 g = string_catn(g, US"@", 1);
1042 s = addr->domain;
1043 #ifdef SUPPORT_I18N
1044 if (testflag(addr, af_utf8_downcvt))
1045 s = string_localpart_utf8_to_alabel(s, NULL);
1046 #endif
1047 g = string_cat(g, s);
1048 }
1049 else
1050 g = string_cat(g, addr->address);
1051
1052 /* If the address we are going to print is the same as the top address,
1053 and all parents are not being included, don't add on the top address. First
1054 of all, do a caseless comparison; if this succeeds, do a caseful comparison
1055 on the local parts. */
1056
1057 cmp = g->s + off; /* only now, as rebuffer likely done */
1058 string_from_gstring(g); /* ensure nul-terminated */
1059 if ( strcmpic(cmp, topaddr->address) == 0
1060 && Ustrncmp(cmp, topaddr->address, Ustrchr(cmp, '@') - cmp) == 0
1061 && !addr->onetime_parent
1062 && (!all_parents || !addr->parent || addr->parent == topaddr)
1063 )
1064 add_topaddr = FALSE;
1065 }
1066
1067 /* If all parents are requested, or this is a local pipe/file/reply, and
1068 there is at least one intermediate parent, show it in brackets, and continue
1069 with all of them if all are wanted. */
1070
1071 if ( (all_parents || testflag(addr, af_pfr))
1072 && addr->parent
1073 && addr->parent != topaddr)
1074 {
1075 uschar *s = US" (";
1076 for (address_item * addr2 = addr->parent; addr2 != topaddr; addr2 = addr2->parent)
1077 {
1078 g = string_catn(g, s, 2);
1079 g = string_cat (g, addr2->address);
1080 if (!all_parents) break;
1081 s = US", ";
1082 }
1083 g = string_catn(g, US")", 1);
1084 }
1085
1086 /* Add the top address if it is required */
1087
1088 if (add_topaddr)
1089 g = string_append(g, 3,
1090 US" <",
1091 addr->onetime_parent ? addr->onetime_parent : topaddr->address,
1092 US">");
1093
1094 return g;
1095 }
1096
1097
1098
1099 /******************************************************************************/
1100
1101
1102
1103 /* If msg is NULL this is a delivery log and logchar is used. Otherwise
1104 this is a nonstandard call; no two-character delivery flag is written
1105 but sender-host and sender are prefixed and "msg" is inserted in the log line.
1106
1107 Arguments:
1108 flags passed to log_write()
1109 */
1110 void
1111 delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
1112 {
1113 gstring * g; /* Used for a temporary, expanding buffer, for building log lines */
1114 rmark reset_point;
1115
1116 /* Log the delivery on the main log. We use an extensible string to build up
1117 the log line, and reset the store afterwards. Remote deliveries should always
1118 have a pointer to the host item that succeeded; local deliveries can have a
1119 pointer to a single host item in their host list, for use by the transport. */
1120
1121 #ifndef DISABLE_EVENT
1122 /* presume no successful remote delivery */
1123 lookup_dnssec_authenticated = NULL;
1124 #endif
1125
1126 reset_point = store_mark();
1127 g = string_get_tainted(256, TRUE); /* addrs will be tainted, so avoid copy */
1128
1129 if (msg)
1130 g = string_append(g, 2, host_and_ident(TRUE), US" ");
1131 else
1132 {
1133 g->s[0] = logchar; g->ptr = 1;
1134 g = string_catn(g, US"> ", 2);
1135 }
1136 g = string_log_address(g, addr, LOGGING(all_parents), TRUE);
1137
1138 if (LOGGING(sender_on_delivery) || msg)
1139 g = string_append(g, 3, US" F=<",
1140 #ifdef SUPPORT_I18N
1141 testflag(addr, af_utf8_downcvt)
1142 ? string_address_utf8_to_alabel(sender_address, NULL)
1143 :
1144 #endif
1145 sender_address,
1146 US">");
1147
1148 if (*queue_name)
1149 g = string_append(g, 2, US" Q=", queue_name);
1150
1151 #ifdef EXPERIMENTAL_SRS
1152 if(addr->prop.srs_sender)
1153 g = string_append(g, 3, US" SRS=<", addr->prop.srs_sender, US">");
1154 #endif
1155
1156 /* You might think that the return path must always be set for a successful
1157 delivery; indeed, I did for some time, until this statement crashed. The case
1158 when it is not set is for a delivery to /dev/null which is optimised by not
1159 being run at all. */
1160
1161 if (used_return_path && LOGGING(return_path_on_delivery))
1162 g = string_append(g, 3, US" P=<", used_return_path, US">");
1163
1164 if (msg)
1165 g = string_append(g, 2, US" ", msg);
1166
1167 /* For a delivery from a system filter, there may not be a router */
1168 if (addr->router)
1169 g = string_append(g, 2, US" R=", addr->router->name);
1170
1171 g = string_append(g, 2, US" T=", addr->transport->name);
1172
1173 if (LOGGING(delivery_size))
1174 g = string_fmt_append(g, " S=%d", transport_count);
1175
1176 /* Local delivery */
1177
1178 if (addr->transport->info->local)
1179 {
1180 if (addr->host_list)
1181 g = string_append(g, 2, US" H=", addr->host_list->name);
1182 g = d_log_interface(g);
1183 if (addr->shadow_message)
1184 g = string_cat(g, addr->shadow_message);
1185 }
1186
1187 /* Remote delivery */
1188
1189 else
1190 {
1191 if (addr->host_used)
1192 {
1193 g = d_hostlog(g, addr);
1194 if (continue_sequence > 1)
1195 g = string_catn(g, US"*", 1);
1196
1197 #ifndef DISABLE_EVENT
1198 deliver_host_address = addr->host_used->address;
1199 deliver_host_port = addr->host_used->port;
1200 deliver_host = addr->host_used->name;
1201
1202 /* DNS lookup status */
1203 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
1204 : addr->host_used->dnssec==DS_NO ? US"no"
1205 : NULL;
1206 #endif
1207 }
1208
1209 #ifndef DISABLE_TLS
1210 g = d_tlslog(g, addr);
1211 #endif
1212
1213 if (addr->authenticator)
1214 {
1215 g = string_append(g, 2, US" A=", addr->authenticator);
1216 if (addr->auth_id)
1217 {
1218 g = string_append(g, 2, US":", addr->auth_id);
1219 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
1220 g = string_append(g, 2, US":", addr->auth_sndr);
1221 }
1222 }
1223
1224 if (LOGGING(pipelining))
1225 {
1226 if (testflag(addr, af_pipelining))
1227 g = string_catn(g, US" L", 2);
1228 #ifndef DISABLE_PIPE_CONNECT
1229 if (testflag(addr, af_early_pipe))
1230 g = string_catn(g, US"*", 1);
1231 #endif
1232 }
1233
1234 #ifndef DISABLE_PRDR
1235 if (testflag(addr, af_prdr_used))
1236 g = string_catn(g, US" PRDR", 5);
1237 #endif
1238
1239 if (testflag(addr, af_chunking_used))
1240 g = string_catn(g, US" K", 2);
1241 }
1242
1243 /* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
1244
1245 if ( LOGGING(smtp_confirmation)
1246 && addr->message
1247 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
1248 )
1249 {
1250 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
1251 uschar *p = big_buffer;
1252 uschar *ss = addr->message;
1253 *p++ = '\"';
1254 for (int i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
1255 {
1256 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
1257 *p++ = ss[i];
1258 }
1259 *p++ = '\"';
1260 *p = 0;
1261 g = string_append(g, 2, US" C=", big_buffer);
1262 }
1263
1264 /* Time on queue and actual time taken to deliver */
1265
1266 if (LOGGING(queue_time))
1267 g = string_append(g, 2, US" QT=",
1268 string_timesince(&received_time));
1269
1270 if (LOGGING(deliver_time))
1271 g = string_append(g, 2, US" DT=", string_timediff(&addr->delivery_time));
1272
1273 /* string_cat() always leaves room for the terminator. Release the
1274 store we used to build the line after writing it. */
1275
1276 log_write(0, flags, "%s", string_from_gstring(g));
1277
1278 #ifndef DISABLE_EVENT
1279 if (!msg) msg_event_raise(US"msg:delivery", addr);
1280 #endif
1281
1282 store_reset(reset_point);
1283 return;
1284 }
1285
1286
1287
1288 static void
1289 deferral_log(address_item * addr, uschar * now,
1290 int logflags, uschar * driver_name, uschar * driver_kind)
1291 {
1292 rmark reset_point = store_mark();
1293 gstring * g = string_get(256);
1294
1295 /* Build up the line that is used for both the message log and the main
1296 log. */
1297
1298 /* Create the address string for logging. Must not do this earlier, because
1299 an OK result may be changed to FAIL when a pipe returns text. */
1300
1301 g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
1302
1303 if (*queue_name)
1304 g = string_append(g, 2, US" Q=", queue_name);
1305
1306 /* Either driver_name contains something and driver_kind contains
1307 " router" or " transport" (note the leading space), or driver_name is
1308 a null string and driver_kind contains "routing" without the leading
1309 space, if all routing has been deferred. When a domain has been held,
1310 so nothing has been done at all, both variables contain null strings. */
1311
1312 if (driver_name)
1313 {
1314 if (driver_kind[1] == 't' && addr->router)
1315 g = string_append(g, 2, US" R=", addr->router->name);
1316 g = string_fmt_append(g, " %c=%s", toupper(driver_kind[1]), driver_name);
1317 }
1318 else if (driver_kind)
1319 g = string_append(g, 2, US" ", driver_kind);
1320
1321 g = string_fmt_append(g, " defer (%d)", addr->basic_errno);
1322
1323 if (addr->basic_errno > 0)
1324 g = string_append(g, 2, US": ",
1325 US strerror(addr->basic_errno));
1326
1327 if (addr->host_used)
1328 {
1329 g = string_append(g, 5,
1330 US" H=", addr->host_used->name,
1331 US" [", addr->host_used->address, US"]");
1332 if (LOGGING(outgoing_port))
1333 {
1334 int port = addr->host_used->port;
1335 g = string_fmt_append(g, ":%d", port == PORT_NONE ? 25 : port);
1336 }
1337 }
1338
1339 if (LOGGING(deliver_time))
1340 g = string_append(g, 2, US" DT=", string_timediff(&addr->delivery_time));
1341
1342 if (addr->message)
1343 g = string_append(g, 2, US": ", addr->message);
1344
1345 (void) string_from_gstring(g);
1346
1347 /* Log the deferment in the message log, but don't clutter it
1348 up with retry-time defers after the first delivery attempt. */
1349
1350 if (f.deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1351 deliver_msglog("%s %s\n", now, g->s);
1352
1353 /* Write the main log and reset the store.
1354 For errors of the type "retry time not reached" (also remotes skipped
1355 on queue run), logging is controlled by L_retry_defer. Note that this kind
1356 of error number is negative, and all the retry ones are less than any
1357 others. */
1358
1359
1360 log_write(addr->basic_errno <= ERRNO_RETRY_BASE ? L_retry_defer : 0, logflags,
1361 "== %s", g->s);
1362
1363 store_reset(reset_point);
1364 return;
1365 }
1366
1367
1368
1369 static void
1370 failure_log(address_item * addr, uschar * driver_kind, uschar * now)
1371 {
1372 rmark reset_point = store_mark();
1373 gstring * g = string_get(256);
1374
1375 #ifndef DISABLE_EVENT
1376 /* Message failures for which we will send a DSN get their event raised
1377 later so avoid doing it here. */
1378
1379 if ( !addr->prop.ignore_error
1380 && !(addr->dsn_flags & (rf_dsnflags & ~rf_notify_failure))
1381 )
1382 msg_event_raise(US"msg:fail:delivery", addr);
1383 #endif
1384
1385 /* Build up the log line for the message and main logs */
1386
1387 /* Create the address string for logging. Must not do this earlier, because
1388 an OK result may be changed to FAIL when a pipe returns text. */
1389
1390 g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
1391
1392 if (LOGGING(sender_on_delivery))
1393 g = string_append(g, 3, US" F=<", sender_address, US">");
1394
1395 if (*queue_name)
1396 g = string_append(g, 2, US" Q=", queue_name);
1397
1398 /* Return path may not be set if no delivery actually happened */
1399
1400 if (used_return_path && LOGGING(return_path_on_delivery))
1401 g = string_append(g, 3, US" P=<", used_return_path, US">");
1402
1403 if (addr->router)
1404 g = string_append(g, 2, US" R=", addr->router->name);
1405 if (addr->transport)
1406 g = string_append(g, 2, US" T=", addr->transport->name);
1407
1408 if (addr->host_used)
1409 g = d_hostlog(g, addr);
1410
1411 #ifndef DISABLE_TLS
1412 g = d_tlslog(g, addr);
1413 #endif
1414
1415 if (addr->basic_errno > 0)
1416 g = string_append(g, 2, US": ", US strerror(addr->basic_errno));
1417
1418 if (addr->message)
1419 g = string_append(g, 2, US": ", addr->message);
1420
1421 if (LOGGING(deliver_time))
1422 g = string_append(g, 2, US" DT=", string_timediff(&addr->delivery_time));
1423
1424 (void) string_from_gstring(g);
1425
1426 /* Do the logging. For the message log, "routing failed" for those cases,
1427 just to make it clearer. */
1428
1429 if (driver_kind)
1430 deliver_msglog("%s %s failed for %s\n", now, driver_kind, g->s);
1431 else
1432 deliver_msglog("%s %s\n", now, g->s);
1433
1434 log_write(0, LOG_MAIN, "** %s", g->s);
1435
1436 store_reset(reset_point);
1437 return;
1438 }
1439
1440
1441
1442 /*************************************************
1443 * Actions at the end of handling an address *
1444 *************************************************/
1445
1446 /* This is a function for processing a single address when all that can be done
1447 with it has been done.
1448
1449 Arguments:
1450 addr points to the address block
1451 result the result of the delivery attempt
1452 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1453 driver_type indicates which type of driver (transport, or router) was last
1454 to process the address
1455 logchar '=' or '-' for use when logging deliveries with => or ->
1456
1457 Returns: nothing
1458 */
1459
1460 static void
1461 post_process_one(address_item *addr, int result, int logflags, int driver_type,
1462 int logchar)
1463 {
1464 uschar *now = tod_stamp(tod_log);
1465 uschar *driver_kind = NULL;
1466 uschar *driver_name = NULL;
1467
1468 DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1469
1470 /* Set up driver kind and name for logging. Disable logging if the router or
1471 transport has disabled it. */
1472
1473 if (driver_type == EXIM_DTYPE_TRANSPORT)
1474 {
1475 if (addr->transport)
1476 {
1477 driver_name = addr->transport->name;
1478 driver_kind = US" transport";
1479 f.disable_logging = addr->transport->disable_logging;
1480 }
1481 else driver_kind = US"transporting";
1482 }
1483 else if (driver_type == EXIM_DTYPE_ROUTER)
1484 {
1485 if (addr->router)
1486 {
1487 driver_name = addr->router->name;
1488 driver_kind = US" router";
1489 f.disable_logging = addr->router->disable_logging;
1490 }
1491 else driver_kind = US"routing";
1492 }
1493
1494 /* If there's an error message set, ensure that it contains only printing
1495 characters - it should, but occasionally things slip in and this at least
1496 stops the log format from getting wrecked. We also scan the message for an LDAP
1497 expansion item that has a password setting, and flatten the password. This is a
1498 fudge, but I don't know a cleaner way of doing this. (If the item is badly
1499 malformed, it won't ever have gone near LDAP.) */
1500
1501 if (addr->message)
1502 {
1503 const uschar * s = string_printing(addr->message);
1504
1505 /* deconst cast ok as string_printing known to have alloc'n'copied */
1506 addr->message = expand_hide_passwords(US s);
1507 }
1508
1509 /* If we used a transport that has one of the "return_output" options set, and
1510 if it did in fact generate some output, then for return_output we treat the
1511 message as failed if it was not already set that way, so that the output gets
1512 returned to the sender, provided there is a sender to send it to. For
1513 return_fail_output, do this only if the delivery failed. Otherwise we just
1514 unlink the file, and remove the name so that if the delivery failed, we don't
1515 try to send back an empty or unwanted file. The log_output options operate only
1516 on a non-empty file.
1517
1518 In any case, we close the message file, because we cannot afford to leave a
1519 file-descriptor for one address while processing (maybe very many) others. */
1520
1521 if (addr->return_file >= 0 && addr->return_filename)
1522 {
1523 BOOL return_output = FALSE;
1524 struct stat statbuf;
1525 (void)EXIMfsync(addr->return_file);
1526
1527 /* If there is no output, do nothing. */
1528
1529 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1530 {
1531 transport_instance *tb = addr->transport;
1532
1533 /* Handle logging options */
1534
1535 if ( tb->log_output
1536 || result == FAIL && tb->log_fail_output
1537 || result == DEFER && tb->log_defer_output
1538 )
1539 {
1540 uschar *s;
1541 FILE *f = Ufopen(addr->return_filename, "rb");
1542 if (!f)
1543 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1544 "from %s transport: %s", addr->return_filename, tb->name,
1545 strerror(errno));
1546 else
1547 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
1548 {
1549 uschar *p = big_buffer + Ustrlen(big_buffer);
1550 const uschar * sp;
1551 while (p > big_buffer && isspace(p[-1])) p--;
1552 *p = 0;
1553 sp = string_printing(big_buffer);
1554 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
1555 addr->address, tb->name, sp);
1556 }
1557 (void)fclose(f);
1558 }
1559
1560 /* Handle returning options, but only if there is an address to return
1561 the text to. */
1562
1563 if (sender_address[0] != 0 || addr->prop.errors_address)
1564 if (tb->return_output)
1565 {
1566 addr->transport_return = result = FAIL;
1567 if (addr->basic_errno == 0 && !addr->message)
1568 addr->message = US"return message generated";
1569 return_output = TRUE;
1570 }
1571 else
1572 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
1573 }
1574
1575 /* Get rid of the file unless it might be returned, but close it in
1576 all cases. */
1577
1578 if (!return_output)
1579 {
1580 Uunlink(addr->return_filename);
1581 addr->return_filename = NULL;
1582 addr->return_file = -1;
1583 }
1584
1585 (void)close(addr->return_file);
1586 }
1587
1588 /* The success case happens only after delivery by a transport. */
1589
1590 if (result == OK)
1591 {
1592 addr->next = addr_succeed;
1593 addr_succeed = addr;
1594
1595 /* Call address_done() to ensure that we don't deliver to this address again,
1596 and write appropriate things to the message log. If it is a child address, we
1597 call child_done() to scan the ancestors and mark them complete if this is the
1598 last child to complete. */
1599
1600 address_done(addr, now);
1601 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1602
1603 if (!addr->parent)
1604 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1605 driver_name, driver_kind);
1606 else
1607 {
1608 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1609 addr->parent->address, driver_name, driver_kind);
1610 child_done(addr, now);
1611 }
1612
1613 /* Certificates for logging (via events) */
1614 #ifndef DISABLE_TLS
1615 tls_out.ourcert = addr->ourcert;
1616 addr->ourcert = NULL;
1617 tls_out.peercert = addr->peercert;
1618 addr->peercert = NULL;
1619
1620 tls_out.ver = addr->tlsver;
1621 tls_out.cipher = addr->cipher;
1622 tls_out.peerdn = addr->peerdn;
1623 tls_out.ocsp = addr->ocsp;
1624 # ifdef SUPPORT_DANE
1625 tls_out.dane_verified = testflag(addr, af_dane_verified);
1626 # endif
1627 #endif
1628
1629 delivery_log(LOG_MAIN, addr, logchar, NULL);
1630
1631 #ifndef DISABLE_TLS
1632 tls_free_cert(&tls_out.ourcert);
1633 tls_free_cert(&tls_out.peercert);
1634 tls_out.ver = NULL;
1635 tls_out.cipher = NULL;
1636 tls_out.peerdn = NULL;
1637 tls_out.ocsp = OCSP_NOT_REQ;
1638 # ifdef SUPPORT_DANE
1639 tls_out.dane_verified = FALSE;
1640 # endif
1641 #endif
1642 }
1643
1644
1645 /* Soft failure, or local delivery process failed; freezing may be
1646 requested. */
1647
1648 else if (result == DEFER || result == PANIC)
1649 {
1650 if (result == PANIC) logflags |= LOG_PANIC;
1651
1652 /* This puts them on the chain in reverse order. Do not change this, because
1653 the code for handling retries assumes that the one with the retry
1654 information is last. */
1655
1656 addr->next = addr_defer;
1657 addr_defer = addr;
1658
1659 /* The only currently implemented special action is to freeze the
1660 message. Logging of this is done later, just before the -H file is
1661 updated. */
1662
1663 if (addr->special_action == SPECIAL_FREEZE)
1664 {
1665 f.deliver_freeze = TRUE;
1666 deliver_frozen_at = time(NULL);
1667 update_spool = TRUE;
1668 }
1669
1670 /* If doing a 2-stage queue run, we skip writing to either the message
1671 log or the main log for SMTP defers. */
1672
1673 if (!f.queue_2stage || addr->basic_errno != 0)
1674 deferral_log(addr, now, logflags, driver_name, driver_kind);
1675 }
1676
1677
1678 /* Hard failure. If there is an address to which an error message can be sent,
1679 put this address on the failed list. If not, put it on the deferred list and
1680 freeze the mail message for human attention. The latter action can also be
1681 explicitly requested by a router or transport. */
1682
1683 else
1684 {
1685 /* If this is a delivery error, or a message for which no replies are
1686 wanted, and the message's age is greater than ignore_bounce_errors_after,
1687 force the af_ignore_error flag. This will cause the address to be discarded
1688 later (with a log entry). */
1689
1690 if (!*sender_address && message_age >= ignore_bounce_errors_after)
1691 addr->prop.ignore_error = TRUE;
1692
1693 /* Freeze the message if requested, or if this is a bounce message (or other
1694 message with null sender) and this address does not have its own errors
1695 address. However, don't freeze if errors are being ignored. The actual code
1696 to ignore occurs later, instead of sending a message. Logging of freezing
1697 occurs later, just before writing the -H file. */
1698
1699 if ( !addr->prop.ignore_error
1700 && ( addr->special_action == SPECIAL_FREEZE
1701 || (sender_address[0] == 0 && !addr->prop.errors_address)
1702 ) )
1703 {
1704 frozen_info = addr->special_action == SPECIAL_FREEZE
1705 ? US""
1706 : f.sender_local && !f.local_error_message
1707 ? US" (message created with -f <>)"
1708 : US" (delivery error message)";
1709 f.deliver_freeze = TRUE;
1710 deliver_frozen_at = time(NULL);
1711 update_spool = TRUE;
1712
1713 /* The address is put on the defer rather than the failed queue, because
1714 the message is being retained. */
1715
1716 addr->next = addr_defer;
1717 addr_defer = addr;
1718 }
1719
1720 /* Don't put the address on the nonrecipients tree yet; wait until an
1721 error message has been successfully sent. */
1722
1723 else
1724 {
1725 addr->next = addr_failed;
1726 addr_failed = addr;
1727 }
1728
1729 failure_log(addr, driver_name ? NULL : driver_kind, now);
1730 }
1731
1732 /* Ensure logging is turned on again in all cases */
1733
1734 f.disable_logging = FALSE;
1735 }
1736
1737
1738
1739
1740 /*************************************************
1741 * Address-independent error *
1742 *************************************************/
1743
1744 /* This function is called when there's an error that is not dependent on a
1745 particular address, such as an expansion string failure. It puts the error into
1746 all the addresses in a batch, logs the incident on the main and panic logs, and
1747 clears the expansions. It is mostly called from local_deliver(), but can be
1748 called for a remote delivery via findugid().
1749
1750 Arguments:
1751 logit TRUE if (MAIN+PANIC) logging required
1752 addr the first of the chain of addresses
1753 code the error code
1754 format format string for error message, or NULL if already set in addr
1755 ... arguments for the format
1756
1757 Returns: nothing
1758 */
1759
1760 static void
1761 common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1762 {
1763 addr->basic_errno = code;
1764
1765 if (format)
1766 {
1767 va_list ap;
1768 gstring * g;
1769
1770 va_start(ap, format);
1771 g = string_vformat(NULL, SVFMT_EXTEND|SVFMT_REBUFFER, CS format, ap);
1772 va_end(ap);
1773 addr->message = string_from_gstring(g);
1774 }
1775
1776 for (address_item * addr2 = addr->next; addr2; addr2 = addr2->next)
1777 {
1778 addr2->basic_errno = code;
1779 addr2->message = addr->message;
1780 }
1781
1782 if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1783 deliver_set_expansions(NULL);
1784 }
1785
1786
1787
1788
1789 /*************************************************
1790 * Check a "never users" list *
1791 *************************************************/
1792
1793 /* This function is called to check whether a uid is on one of the two "never
1794 users" lists.
1795
1796 Arguments:
1797 uid the uid to be checked
1798 nusers the list to be scanned; the first item in the list is the count
1799
1800 Returns: TRUE if the uid is on the list
1801 */
1802
1803 static BOOL
1804 check_never_users(uid_t uid, uid_t *nusers)
1805 {
1806 if (!nusers) return FALSE;
1807 for (int i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1808 return FALSE;
1809 }
1810
1811
1812
1813 /*************************************************
1814 * Find uid and gid for a transport *
1815 *************************************************/
1816
1817 /* This function is called for both local and remote deliveries, to find the
1818 uid/gid under which to run the delivery. The values are taken preferentially
1819 from the transport (either explicit or deliver_as_creator), then from the
1820 address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1821 the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1822 panic error is logged, and the function fails (which normally leads to delivery
1823 deferral).
1824
1825 Arguments:
1826 addr the address (possibly a chain)
1827 tp the transport
1828 uidp pointer to uid field
1829 gidp pointer to gid field
1830 igfp pointer to the use_initgroups field
1831
1832 Returns: FALSE if failed - error has been set in address(es)
1833 */
1834
1835 static BOOL
1836 findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1837 BOOL *igfp)
1838 {
1839 uschar *nuname;
1840 BOOL gid_set = FALSE;
1841
1842 /* Default initgroups flag comes from the transport */
1843
1844 *igfp = tp->initgroups;
1845
1846 /* First see if there's a gid on the transport, either fixed or expandable.
1847 The expanding function always logs failure itself. */
1848
1849 if (tp->gid_set)
1850 {
1851 *gidp = tp->gid;
1852 gid_set = TRUE;
1853 }
1854 else if (tp->expand_gid)
1855 {
1856 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1857 &(addr->message)))
1858 {
1859 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1860 return FALSE;
1861 }
1862 gid_set = TRUE;
1863 }
1864
1865 /* If the transport did not set a group, see if the router did. */
1866
1867 if (!gid_set && testflag(addr, af_gid_set))
1868 {
1869 *gidp = addr->gid;
1870 gid_set = TRUE;
1871 }
1872
1873 /* Pick up a uid from the transport if one is set. */
1874
1875 if (tp->uid_set) *uidp = tp->uid;
1876
1877 /* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1878 it does not provide a passwd value from which a gid can be taken. */
1879
1880 else if (tp->expand_uid)
1881 {
1882 struct passwd *pw;
1883 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1884 uidp, &(addr->message)))
1885 {
1886 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1887 return FALSE;
1888 }
1889 if (!gid_set && pw)
1890 {
1891 *gidp = pw->pw_gid;
1892 gid_set = TRUE;
1893 }
1894 }
1895
1896 /* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1897
1898 else if (tp->deliver_as_creator)
1899 {
1900 *uidp = originator_uid;
1901 if (!gid_set)
1902 {
1903 *gidp = originator_gid;
1904 gid_set = TRUE;
1905 }
1906 }
1907
1908 /* Otherwise see if the address specifies the uid and if so, take it and its
1909 initgroups flag. */
1910
1911 else if (testflag(addr, af_uid_set))
1912 {
1913 *uidp = addr->uid;
1914 *igfp = testflag(addr, af_initgroups);
1915 }
1916
1917 /* Nothing has specified the uid - default to the Exim user, and group if the
1918 gid is not set. */
1919
1920 else
1921 {
1922 *uidp = exim_uid;
1923 if (!gid_set)
1924 {
1925 *gidp = exim_gid;
1926 gid_set = TRUE;
1927 }
1928 }
1929
1930 /* If no gid is set, it is a disaster. We default to the Exim gid only if
1931 defaulting to the Exim uid. In other words, if the configuration has specified
1932 a uid, it must also provide a gid. */
1933
1934 if (!gid_set)
1935 {
1936 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1937 "%s transport", tp->name);
1938 return FALSE;
1939 }
1940
1941 /* Check that the uid is not on the lists of banned uids that may not be used
1942 for delivery processes. */
1943
1944 nuname = check_never_users(*uidp, never_users)
1945 ? US"never_users"
1946 : check_never_users(*uidp, fixed_never_users)
1947 ? US"fixed_never_users"
1948 : NULL;
1949 if (nuname)
1950 {
1951 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1952 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1953 return FALSE;
1954 }
1955
1956 /* All is well */
1957
1958 return TRUE;
1959 }
1960
1961
1962
1963
1964 /*************************************************
1965 * Check the size of a message for a transport *
1966 *************************************************/
1967
1968 /* Checks that the message isn't too big for the selected transport.
1969 This is called only when it is known that the limit is set.
1970
1971 Arguments:
1972 tp the transport
1973 addr the (first) address being delivered
1974
1975 Returns: OK
1976 DEFER expansion failed or did not yield an integer
1977 FAIL message too big
1978 */
1979
1980 int
1981 check_message_size(transport_instance *tp, address_item *addr)
1982 {
1983 int rc = OK;
1984 int size_limit;
1985
1986 deliver_set_expansions(addr);
1987 size_limit = expand_string_integer(tp->message_size_limit, TRUE);
1988 deliver_set_expansions(NULL);
1989
1990 if (expand_string_message)
1991 {
1992 rc = DEFER;
1993 addr->message = size_limit == -1
1994 ? string_sprintf("failed to expand message_size_limit "
1995 "in %s transport: %s", tp->name, expand_string_message)
1996 : string_sprintf("invalid message_size_limit "
1997 "in %s transport: %s", tp->name, expand_string_message);
1998 }
1999 else if (size_limit > 0 && message_size > size_limit)
2000 {
2001 rc = FAIL;
2002 addr->message =
2003 string_sprintf("message is too big (transport limit = %d)",
2004 size_limit);
2005 }
2006
2007 return rc;
2008 }
2009
2010
2011
2012 /*************************************************
2013 * Transport-time check for a previous delivery *
2014 *************************************************/
2015
2016 /* Check that this base address hasn't previously been delivered to its routed
2017 transport. If it has been delivered, mark it done. The check is necessary at
2018 delivery time in order to handle homonymic addresses correctly in cases where
2019 the pattern of redirection changes between delivery attempts (so the unique
2020 fields change). Non-homonymic previous delivery is detected earlier, at routing
2021 time (which saves unnecessary routing).
2022
2023 Arguments:
2024 addr the address item
2025 testing TRUE if testing wanted only, without side effects
2026
2027 Returns: TRUE if previously delivered by the transport
2028 */
2029
2030 static BOOL
2031 previously_transported(address_item *addr, BOOL testing)
2032 {
2033 uschar * s = string_sprintf("%s/%s",
2034 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
2035
2036 if (tree_search(tree_nonrecipients, s) != 0)
2037 {
2038 DEBUG(D_deliver|D_route|D_transport)
2039 debug_printf("%s was previously delivered (%s transport): discarded\n",
2040 addr->address, addr->transport->name);
2041 if (!testing) child_done(addr, tod_stamp(tod_log));
2042 return TRUE;
2043 }
2044
2045 return FALSE;
2046 }
2047
2048
2049
2050 /******************************************************
2051 * Check for a given header in a header string *
2052 ******************************************************/
2053
2054 /* This function is used when generating quota warnings. The configuration may
2055 specify any header lines it likes in quota_warn_message. If certain of them are
2056 missing, defaults are inserted, so we need to be able to test for the presence
2057 of a given header.
2058
2059 Arguments:
2060 hdr the required header name
2061 hstring the header string
2062
2063 Returns: TRUE the header is in the string
2064 FALSE the header is not in the string
2065 */
2066
2067 static BOOL
2068 contains_header(uschar *hdr, uschar *hstring)
2069 {
2070 int len = Ustrlen(hdr);
2071 uschar *p = hstring;
2072 while (*p != 0)
2073 {
2074 if (strncmpic(p, hdr, len) == 0)
2075 {
2076 p += len;
2077 while (*p == ' ' || *p == '\t') p++;
2078 if (*p == ':') return TRUE;
2079 }
2080 while (*p != 0 && *p != '\n') p++;
2081 if (*p == '\n') p++;
2082 }
2083 return FALSE;
2084 }
2085
2086
2087
2088
2089 /*************************************************
2090 * Perform a local delivery *
2091 *************************************************/
2092
2093 /* Each local delivery is performed in a separate process which sets its
2094 uid and gid as specified. This is a safer way than simply changing and
2095 restoring using seteuid(); there is a body of opinion that seteuid() cannot be
2096 used safely. From release 4, Exim no longer makes any use of it. Besides, not
2097 all systems have seteuid().
2098
2099 If the uid/gid are specified in the transport_instance, they are used; the
2100 transport initialization must ensure that either both or neither are set.
2101 Otherwise, the values associated with the address are used. If neither are set,
2102 it is a configuration error.
2103
2104 The transport or the address may specify a home directory (transport over-
2105 rides), and if they do, this is set as $home. If neither have set a working
2106 directory, this value is used for that as well. Otherwise $home is left unset
2107 and the cwd is set to "/" - a directory that should be accessible to all users.
2108
2109 Using a separate process makes it more complicated to get error information
2110 back. We use a pipe to pass the return code and also an error code and error
2111 text string back to the parent process.
2112
2113 Arguments:
2114 addr points to an address block for this delivery; for "normal" local
2115 deliveries this is the only address to be delivered, but for
2116 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
2117 a number of addresses can be handled simultaneously, and in this
2118 case addr will point to a chain of addresses with the same
2119 characteristics.
2120
2121 shadowing TRUE if running a shadow transport; this causes output from pipes
2122 to be ignored.
2123
2124 Returns: nothing
2125 */
2126
2127 static void
2128 deliver_local(address_item *addr, BOOL shadowing)
2129 {
2130 BOOL use_initgroups;
2131 uid_t uid;
2132 gid_t gid;
2133 int status, len, rc;
2134 int pfd[2];
2135 pid_t pid;
2136 uschar *working_directory;
2137 address_item *addr2;
2138 transport_instance *tp = addr->transport;
2139
2140 /* Set up the return path from the errors or sender address. If the transport
2141 has its own return path setting, expand it and replace the existing value. */
2142
2143 if(addr->prop.errors_address)
2144 return_path = addr->prop.errors_address;
2145 #ifdef EXPERIMENTAL_SRS
2146 else if (addr->prop.srs_sender)
2147 return_path = addr->prop.srs_sender;
2148 #endif
2149 else
2150 return_path = sender_address;
2151
2152 if (tp->return_path)
2153 {
2154 uschar *new_return_path = expand_string(tp->return_path);
2155 if (!new_return_path)
2156 {
2157 if (!f.expand_string_forcedfail)
2158 {
2159 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
2160 US"Failed to expand return path \"%s\" in %s transport: %s",
2161 tp->return_path, tp->name, expand_string_message);
2162 return;
2163 }
2164 }
2165 else return_path = new_return_path;
2166 }
2167
2168 /* For local deliveries, one at a time, the value used for logging can just be
2169 set directly, once and for all. */
2170
2171 used_return_path = return_path;
2172
2173 /* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
2174 gets put into the address(es), and the expansions are unset, so we can just
2175 return. */
2176
2177 if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
2178
2179 /* See if either the transport or the address specifies a home directory. A
2180 home directory set in the address may already be expanded; a flag is set to
2181 indicate that. In other cases we must expand it. */
2182
2183 if ( (deliver_home = tp->home_dir) /* Set in transport, or */
2184 || ( (deliver_home = addr->home_dir) /* Set in address and */
2185 && !testflag(addr, af_home_expanded) /* not expanded */
2186 ) )
2187 {
2188 uschar *rawhome = deliver_home;
2189 deliver_home = NULL; /* in case it contains $home */
2190 if (!(deliver_home = expand_string(rawhome)))
2191 {
2192 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
2193 "to expand for %s transport: %s", rawhome, tp->name,
2194 expand_string_message);
2195 return;
2196 }
2197 if (*deliver_home != '/')
2198 {
2199 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
2200 "is not absolute for %s transport", deliver_home, tp->name);
2201 return;
2202 }
2203 }
2204
2205 /* See if either the transport or the address specifies a current directory,
2206 and if so, expand it. If nothing is set, use the home directory, unless it is
2207 also unset in which case use "/", which is assumed to be a directory to which
2208 all users have access. It is necessary to be in a visible directory for some
2209 operating systems when running pipes, as some commands (e.g. "rm" under Solaris
2210 2.5) require this. */
2211
2212 working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
2213 if (working_directory)
2214 {
2215 uschar *raw = working_directory;
2216 if (!(working_directory = expand_string(raw)))
2217 {
2218 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
2219 "failed to expand for %s transport: %s", raw, tp->name,
2220 expand_string_message);
2221 return;
2222 }
2223 if (*working_directory != '/')
2224 {
2225 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
2226 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
2227 return;
2228 }
2229 }
2230 else working_directory = deliver_home ? deliver_home : US"/";
2231
2232 /* If one of the return_output flags is set on the transport, create and open a
2233 file in the message log directory for the transport to write its output onto.
2234 This is mainly used by pipe transports. The file needs to be unique to the
2235 address. This feature is not available for shadow transports. */
2236
2237 if ( !shadowing
2238 && ( tp->return_output || tp->return_fail_output
2239 || tp->log_output || tp->log_fail_output || tp->log_defer_output
2240 ) )
2241 {
2242 uschar * error;
2243
2244 addr->return_filename =
2245 spool_fname(US"msglog", message_subdir, message_id,
2246 string_sprintf("-%d-%d", getpid(), return_count++));
2247
2248 if ((addr->return_file = open_msglog_file(addr->return_filename, 0400, &error)) < 0)
2249 {
2250 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
2251 "to return message: %s", error, tp->name, strerror(errno));
2252 return;
2253 }
2254 }
2255
2256 /* Create the pipe for inter-process communication. */
2257
2258 if (pipe(pfd) != 0)
2259 {
2260 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
2261 strerror(errno));
2262 return;
2263 }
2264
2265 /* Now fork the process to do the real work in the subprocess, but first
2266 ensure that all cached resources are freed so that the subprocess starts with
2267 a clean slate and doesn't interfere with the parent process. */
2268
2269 search_tidyup();
2270
2271 if ((pid = exim_fork(US"delivery-local")) == 0)
2272 {
2273 BOOL replicate = TRUE;
2274
2275 /* Prevent core dumps, as we don't want them in users' home directories.
2276 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
2277 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
2278 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
2279 complain if the error is "not supported".
2280
2281 There are two scenarios where changing the max limit has an effect. In one,
2282 the user is using a .forward and invoking a command of their choice via pipe;
2283 for these, we do need the max limit to be 0 unless the admin chooses to
2284 permit an increased limit. In the other, the command is invoked directly by
2285 the transport and is under administrator control, thus being able to raise
2286 the limit aids in debugging. So there's no general always-right answer.
2287
2288 Thus we inhibit core-dumps completely but let individual transports, while
2289 still root, re-raise the limits back up to aid debugging. We make the
2290 default be no core-dumps -- few enough people can use core dumps in
2291 diagnosis that it's reasonable to make them something that has to be explicitly requested.
2292 */
2293
2294 #ifdef RLIMIT_CORE
2295 struct rlimit rl;
2296 rl.rlim_cur = 0;
2297 rl.rlim_max = 0;
2298 if (setrlimit(RLIMIT_CORE, &rl) < 0)
2299 {
2300 # ifdef SETRLIMIT_NOT_SUPPORTED
2301 if (errno != ENOSYS && errno != ENOTSUP)
2302 # endif
2303 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
2304 strerror(errno));
2305 }
2306 #endif
2307
2308 /* Reset the random number generator, so different processes don't all
2309 have the same sequence. */
2310
2311 random_seed = 0;
2312
2313 /* If the transport has a setup entry, call this first, while still
2314 privileged. (Appendfile uses this to expand quota, for example, while
2315 able to read private files.) */
2316
2317 if (addr->transport->setup)
2318 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
2319 &(addr->message)))
2320 {
2321 case DEFER:
2322 addr->transport_return = DEFER;
2323 goto PASS_BACK;
2324
2325 case FAIL:
2326 addr->transport_return = PANIC;
2327 goto PASS_BACK;
2328 }
2329
2330 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2331 when the process becomes unprivileged, it won't be able to write to the
2332 process log. SIGHUP is ignored throughout exim, except when it is being
2333 run as a daemon. */
2334
2335 signal(SIGINT, SIG_IGN);
2336 signal(SIGTERM, SIG_IGN);
2337 signal(SIGUSR1, SIG_IGN);
2338
2339 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2340 half - for transports that exec things (e.g. pipe). Then set the required
2341 gid/uid. */
2342
2343 (void)close(pfd[pipe_read]);
2344 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
2345 FD_CLOEXEC);
2346 exim_setugid(uid, gid, use_initgroups,
2347 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2348 addr->address, addr->transport->name));
2349
2350 DEBUG(D_deliver)
2351 {
2352 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
2353 for (address_item * batched = addr->next; batched; batched = batched->next)
2354 debug_printf("additional batched address: %s\n", batched->address);
2355 }
2356
2357 /* Set an appropriate working directory. */
2358
2359 if (Uchdir(working_directory) < 0)
2360 {
2361 addr->transport_return = DEFER;
2362 addr->basic_errno = errno;
2363 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2364 }
2365
2366 /* If successful, call the transport */
2367
2368 else
2369 {
2370 BOOL ok = TRUE;
2371 set_process_info("delivering %s to %s using %s", message_id,
2372 addr->local_part, addr->transport->name);
2373
2374 /* Setting this global in the subprocess means we need never clear it */
2375 transport_name = addr->transport->name;
2376
2377 /* If a transport filter has been specified, set up its argument list.
2378 Any errors will get put into the address, and FALSE yielded. */
2379
2380 if (addr->transport->filter_command)
2381 {
2382 ok = transport_set_up_command(&transport_filter_argv,
2383 addr->transport->filter_command,
2384 TRUE, PANIC, addr, US"transport filter", NULL);
2385 transport_filter_timeout = addr->transport->filter_timeout;
2386 }
2387 else transport_filter_argv = NULL;
2388
2389 if (ok)
2390 {
2391 debug_print_string(addr->transport->debug_string);
2392 replicate = !(addr->transport->info->code)(addr->transport, addr);
2393 }
2394 }
2395
2396 /* Pass the results back down the pipe. If necessary, first replicate the
2397 status in the top address to the others in the batch. The label is the
2398 subject of a goto when a call to the transport's setup function fails. We
2399 pass the pointer to the transport back in case it got changed as a result of
2400 file_format in appendfile. */
2401
2402 PASS_BACK:
2403
2404 if (replicate) replicate_status(addr);
2405 for (addr2 = addr; addr2; addr2 = addr2->next)
2406 {
2407 int i;
2408 int local_part_length = Ustrlen(addr2->local_part);
2409 uschar *s;
2410 int ret;
2411
2412 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2413 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2414 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2415 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2416 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
2417 || (ret = write(pfd[pipe_write], &addr2->delivery_time, sizeof(struct timeval))) != sizeof(struct timeval)
2418 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2419 || (ret = write(pfd[pipe_write], &addr2->transport,
2420 sizeof(transport_instance *))) != sizeof(transport_instance *)
2421
2422 /* For a file delivery, pass back the local part, in case the original
2423 was only part of the final delivery path. This gives more complete
2424 logging. */
2425
2426 || (testflag(addr2, af_file)
2427 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
2428 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2429 )
2430 )
2431 )
2432 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
2433 ret == -1 ? strerror(errno) : "short write");
2434
2435 /* Now any messages */
2436
2437 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2438 {
2439 int message_length = s ? Ustrlen(s) + 1 : 0;
2440 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2441 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
2442 )
2443 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
2444 ret == -1 ? strerror(errno) : "short write");
2445 }
2446 }
2447
2448 /* OK, this process is now done. Free any cached resources that it opened,
2449 and close the pipe we were writing down before exiting. */
2450
2451 (void)close(pfd[pipe_write]);
2452 search_tidyup();
2453 exit(EXIT_SUCCESS);
2454 }
2455
2456 /* Back in the main process: panic if the fork did not succeed. This seems
2457 better than returning an error - if forking is failing it is probably best
2458 not to try other deliveries for this message. */
2459
2460 if (pid < 0)
2461 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2462 addr->address);
2463
2464 /* Read the pipe to get the delivery status codes and error messages. Our copy
2465 of the writing end must be closed first, as otherwise read() won't return zero
2466 on an empty pipe. We check that a status exists for each address before
2467 overwriting the address structure. If data is missing, the default DEFER status
2468 will remain. Afterwards, close the reading end. */
2469
2470 (void)close(pfd[pipe_write]);
2471
2472 for (addr2 = addr; addr2; addr2 = addr2->next)
2473 {
2474 if ((len = read(pfd[pipe_read], &status, sizeof(int))) > 0)
2475 {
2476 int i;
2477 uschar **sptr;
2478
2479 addr2->transport_return = status;
2480 len = read(pfd[pipe_read], &transport_count,
2481 sizeof(transport_count));
2482 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2483 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2484 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
2485 len = read(pfd[pipe_read], &addr2->delivery_time, sizeof(struct timeval));
2486 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2487 len = read(pfd[pipe_read], &addr2->transport,
2488 sizeof(transport_instance *));
2489
2490 if (testflag(addr2, af_file))
2491 {
2492 int llen;
2493 if ( read(pfd[pipe_read], &llen, sizeof(int)) != sizeof(int)
2494 || llen > 64*4 /* limit from rfc 5821, times I18N factor */
2495 )
2496 {
2497 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part length read"
2498 " from delivery subprocess");
2499 break;
2500 }
2501 /* sanity-checked llen so disable the Coverity error */
2502 /* coverity[tainted_data] */
2503 if (read(pfd[pipe_read], big_buffer, llen) != llen)
2504 {
2505 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part read"
2506 " from delivery subprocess");
2507 break;
2508 }
2509 big_buffer[llen] = 0;
2510 addr2->local_part = string_copy(big_buffer);
2511 }
2512
2513 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
2514 {
2515 int message_length;
2516 len = read(pfd[pipe_read], &message_length, sizeof(int));
2517 if (message_length > 0)
2518 {
2519 len = read(pfd[pipe_read], big_buffer, message_length);
2520 big_buffer[big_buffer_size-1] = '\0'; /* guard byte */
2521 if (len > 0) *sptr = string_copy(big_buffer);
2522 }
2523 }
2524 }
2525
2526 else
2527 {
2528 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2529 "from delivery subprocess", addr2->unique);
2530 break;
2531 }
2532 }
2533
2534 (void)close(pfd[pipe_read]);
2535
2536 /* Unless shadowing, write all successful addresses immediately to the journal
2537 file, to ensure they are recorded asap. For homonymic addresses, use the base
2538 address plus the transport name. Failure to write the journal is panic-worthy,
2539 but don't stop, as it may prove possible subsequently to update the spool file
2540 in order to record the delivery. */
2541
2542 if (!shadowing)
2543 {
2544 for (addr2 = addr; addr2; addr2 = addr2->next)
2545 if (addr2->transport_return == OK)
2546 {
2547 if (testflag(addr2, af_homonym))
2548 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2549 else
2550 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
2551
2552 /* In the test harness, wait just a bit to let the subprocess finish off
2553 any debug output etc first. */
2554
2555 testharness_pause_ms(300);
2556
2557 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2558 len = Ustrlen(big_buffer);
2559 if (write(journal_fd, big_buffer, len) != len)
2560 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2561 big_buffer, strerror(errno));
2562 }
2563
2564 /* Ensure the journal file is pushed out to disk. */
2565
2566 if (EXIMfsync(journal_fd) < 0)
2567 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2568 strerror(errno));
2569 }
2570
2571 /* Wait for the process to finish. If it terminates with a non-zero code,
2572 freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2573 status values of all the addresses as they are. Take care to handle the case
2574 when the subprocess doesn't seem to exist. This has been seen on one system
2575 when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2576 happens, wait() doesn't recognize the termination of child processes. Exim now
2577 resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2578
2579 while ((rc = wait(&status)) != pid)
2580 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2581 {
2582 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2583 addr->transport->driver_name);
2584 status = 0;
2585 break;
2586 }
2587
2588 if ((status & 0xffff) != 0)
2589 {
2590 int msb = (status >> 8) & 255;
2591 int lsb = status & 255;
2592 int code = (msb == 0)? (lsb & 0x7f) : msb;
2593 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2594 addr->special_action = SPECIAL_FREEZE;
2595 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2596 "status 0x%04x: %s %d",
2597 addr->transport->driver_name,
2598 status,
2599 msb == 0 ? "terminated by signal" : "exit code",
2600 code);
2601 }
2602
2603 /* If SPECIAL_WARN is set in the top address, send a warning message. */
2604
2605 if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
2606 {
2607 int fd;
2608 uschar *warn_message;
2609 pid_t pid;
2610
2611 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2612
2613 if (!(warn_message = expand_string(addr->transport->warn_message)))
2614 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2615 "message for %s transport): %s", addr->transport->warn_message,
2616 addr->transport->name, expand_string_message);
2617
2618 else if ((pid = child_open_exim(&fd, US"tpt-warning-message")) > 0)
2619 {
2620 FILE *f = fdopen(fd, "wb");
2621 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2622 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2623 fprintf(f, "Auto-Submitted: auto-replied\n");
2624 if (!contains_header(US"From", warn_message))
2625 moan_write_from(f);
2626 fprintf(f, "%s", CS warn_message);
2627
2628 /* Close and wait for child process to complete, without a timeout. */
2629
2630 (void)fclose(f);
2631 (void)child_close(pid, 0);
2632 }
2633
2634 addr->special_action = SPECIAL_NONE;
2635 }
2636 }
2637
2638
2639
2640
2641 /* Check transport for the given concurrency limit. Return TRUE if over
2642 the limit (or an expansion failure), else FALSE and if there was a limit,
2643 the key for the hints database used for the concurrency count. */
2644
2645 static BOOL
2646 tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key)
2647 {
2648 unsigned max_parallel;
2649
2650 if (!tp->max_parallel) return FALSE;
2651
2652 max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE);
2653 if (expand_string_message)
2654 {
2655 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2656 "in %s transport (%s): %s", tp->name, addr->address,
2657 expand_string_message);
2658 return TRUE;
2659 }
2660
2661 if (max_parallel > 0)
2662 {
2663 uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name);
2664 if (!enq_start(serialize_key, max_parallel))
2665 {
2666 address_item * next;
2667 DEBUG(D_transport)
2668 debug_printf("skipping tpt %s because concurrency limit %u reached\n",
2669 tp->name, max_parallel);
2670 do
2671 {
2672 next = addr->next;
2673 addr->message = US"concurrency limit reached for transport";
2674 addr->basic_errno = ERRNO_TRETRY;
2675 post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_TRANSPORT, 0);
2676 } while ((addr = next));
2677 return TRUE;
2678 }
2679 *key = serialize_key;
2680 }
2681 return FALSE;
2682 }
2683
2684
2685
2686 /*************************************************
2687 * Do local deliveries *
2688 *************************************************/
2689
2690 /* This function processes the list of addresses in addr_local. True local
2691 deliveries are always done one address at a time. However, local deliveries can
2692 be batched up in some cases. Typically this is when writing batched SMTP output
2693 files for use by some external transport mechanism, or when running local
2694 deliveries over LMTP.
2695
2696 Arguments: None
2697 Returns: Nothing
2698 */
2699
2700 static void
2701 do_local_deliveries(void)
2702 {
2703 open_db dbblock;
2704 open_db *dbm_file = NULL;
2705 time_t now = time(NULL);
2706
2707 /* Loop until we have exhausted the supply of local deliveries */
2708
2709 while (addr_local)
2710 {
2711 struct timeval delivery_start;
2712 struct timeval deliver_time;
2713 address_item *addr2, *addr3, *nextaddr;
2714 int logflags = LOG_MAIN;
2715 int logchar = f.dont_deliver? '*' : '=';
2716 transport_instance *tp;
2717 uschar * serialize_key = NULL;
2718
2719 /* Pick the first undelivered address off the chain */
2720
2721 address_item *addr = addr_local;
2722 addr_local = addr->next;
2723 addr->next = NULL;
2724
2725 DEBUG(D_deliver|D_transport)
2726 debug_printf("--------> %s <--------\n", addr->address);
2727
2728 /* An internal disaster if there is no transport. Should not occur! */
2729
2730 if (!(tp = addr->transport))
2731 {
2732 logflags |= LOG_PANIC;
2733 f.disable_logging = FALSE; /* Jic */
2734 addr->message = addr->router
2735 ? string_sprintf("No transport set by %s router", addr->router->name)
2736 : US"No transport set by system filter";
2737 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
2738 continue;
2739 }
2740
2741 /* Check that this base address hasn't previously been delivered to this
2742 transport. The check is necessary at this point to handle homonymic addresses
2743 correctly in cases where the pattern of redirection changes between delivery
2744 attempts. Non-homonymic previous delivery is detected earlier, at routing
2745 time. */
2746
2747 if (previously_transported(addr, FALSE)) continue;
2748
2749 /* There are weird cases where logging is disabled */
2750
2751 f.disable_logging = tp->disable_logging;
2752
2753 /* Check for batched addresses and possible amalgamation. Skip all the work
2754 if either batch_max <= 1 or there aren't any other addresses for local
2755 delivery. */
2756
2757 if (tp->batch_max > 1 && addr_local)
2758 {
2759 int batch_count = 1;
2760 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
2761 BOOL uses_lp = ( testflag(addr, af_pfr)
2762 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2763 )
2764 || readconf_depends((driver_instance *)tp, US"local_part");
2765 uschar *batch_id = NULL;
2766 address_item **anchor = &addr_local;
2767 address_item *last = addr;
2768 address_item *next;
2769
2770 /* Expand the batch_id string for comparison with other addresses.
2771 Expansion failure suppresses batching. */
2772
2773 if (tp->batch_id)
2774 {
2775 deliver_set_expansions(addr);
2776 batch_id = expand_string(tp->batch_id);
2777 deliver_set_expansions(NULL);
2778 if (!batch_id)
2779 {
2780 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2781 "in %s transport (%s): %s", tp->name, addr->address,
2782 expand_string_message);
2783 batch_count = tp->batch_max;
2784 }
2785 }
2786
2787 /* Until we reach the batch_max limit, pick off addresses which have the
2788 same characteristics. These are:
2789
2790 same transport
2791 not previously delivered (see comment about 50 lines above)
2792 same local part if the transport's configuration contains $local_part
2793 or if this is a file or pipe delivery from a redirection
2794 same domain if the transport's configuration contains $domain
2795 same errors address
2796 same additional headers
2797 same headers to be removed
2798 same uid/gid for running the transport
2799 same first host if a host list is set
2800 */
2801
2802 while ((next = *anchor) && batch_count < tp->batch_max)
2803 {
2804 BOOL ok =
2805 tp == next->transport
2806 && !previously_transported(next, TRUE)
2807 && testflag(addr, af_pfr) == testflag(next, af_pfr)
2808 && testflag(addr, af_file) == testflag(next, af_file)
2809 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2810 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2811 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2812 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2813 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2814 && same_ugid(tp, addr, next)
2815 && ( !addr->host_list && !next->host_list
2816 || addr->host_list
2817 && next->host_list
2818 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2819 );
2820
2821 /* If the transport has a batch_id setting, batch_id will be non-NULL
2822 from the expansion outside the loop. Expand for this address and compare.
2823 Expansion failure makes this address ineligible for batching. */
2824
2825 if (ok && batch_id)
2826 {
2827 uschar *bid;
2828 address_item *save_nextnext = next->next;
2829 next->next = NULL; /* Expansion for a single address */
2830 deliver_set_expansions(next);
2831 next->next = save_nextnext;
2832 bid = expand_string(tp->batch_id);
2833 deliver_set_expansions(NULL);
2834 if (!bid)
2835 {
2836 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2837 "in %s transport (%s): %s", tp->name, next->address,
2838 expand_string_message);
2839 ok = FALSE;
2840 }
2841 else ok = (Ustrcmp(batch_id, bid) == 0);
2842 }
2843
2844 /* Take address into batch if OK. */
2845
2846 if (ok)
2847 {
2848 *anchor = next->next; /* Include the address */
2849 next->next = NULL;
2850 last->next = next;
2851 last = next;
2852 batch_count++;
2853 }
2854 else anchor = &next->next; /* Skip the address */
2855 }
2856 }
2857
2858 /* We now have one or more addresses that can be delivered in a batch. Check
2859 whether the transport is prepared to accept a message of this size. If not,
2860 fail them all forthwith. If the expansion fails, or does not yield an
2861 integer, defer delivery. */
2862
2863 if (tp->message_size_limit)
2864 {
2865 int rc = check_message_size(tp, addr);
2866 if (rc != OK)
2867 {
2868 replicate_status(addr);
2869 while (addr)
2870 {
2871 addr2 = addr->next;
2872 post_process_one(addr, rc, logflags, EXIM_DTYPE_TRANSPORT, 0);
2873 addr = addr2;
2874 }
2875 continue; /* With next batch of addresses */
2876 }
2877 }
2878
2879 /* If we are not running the queue, or if forcing, all deliveries will be
2880 attempted. Otherwise, we must respect the retry times for each address. Even
2881 when not doing this, we need to set up the retry key string, and determine
2882 whether a retry record exists, because after a successful delivery, a delete
2883 retry item must be set up. Keep the retry database open only for the duration
2884 of these checks, rather than for all local deliveries, because some local
2885 deliveries (e.g. to pipes) can take a substantial time. */
2886
2887 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE)))
2888 {
2889 DEBUG(D_deliver|D_retry|D_hints_lookup)
2890 debug_printf("no retry data available\n");
2891 }
2892
2893 addr2 = addr;
2894 addr3 = NULL;
2895 while (addr2)
2896 {
2897 BOOL ok = TRUE; /* to deliver this address */
2898 uschar *retry_key;
2899
2900 /* Set up the retry key to include the domain or not, and change its
2901 leading character from "R" to "T". Must make a copy before doing this,
2902 because the old key may be pointed to from a "delete" retry item after
2903 a routing delay. */
2904
2905 retry_key = string_copy(
2906 tp->retry_use_local_part ? addr2->address_retry_key :
2907 addr2->domain_retry_key);
2908 *retry_key = 'T';
2909
2910 /* Inspect the retry data. If there is no hints file, delivery happens. */
2911
2912 if (dbm_file)
2913 {
2914 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2915
2916 /* If there is no retry record, delivery happens. If there is,
2917 remember it exists so it can be deleted after a successful delivery. */
2918
2919 if (retry_record)
2920 {
2921 setflag(addr2, af_lt_retry_exists);
2922
2923 /* A retry record exists for this address. If queue running and not
2924 forcing, inspect its contents. If the record is too old, or if its
2925 retry time has come, or if it has passed its cutoff time, delivery
2926 will go ahead. */
2927
2928 DEBUG(D_retry)
2929 {
2930 debug_printf("retry record exists: age=%s ",
2931 readconf_printtime(now - retry_record->time_stamp));
2932 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2933 debug_printf(" time to retry = %s expired = %d\n",
2934 readconf_printtime(retry_record->next_try - now),
2935 retry_record->expired);
2936 }
2937
2938 if (f.queue_running && !f.deliver_force)
2939 {
2940 ok = (now - retry_record->time_stamp > retry_data_expire)
2941 || (now >= retry_record->next_try)
2942 || retry_record->expired;
2943
2944 /* If we haven't reached the retry time, there is one more check
2945 to do, which is for the ultimate address timeout. */
2946
2947 if (!ok)
2948 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2949 retry_record, now);
2950 }
2951 }
2952 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2953 }
2954
2955 /* This address is to be delivered. Leave it on the chain. */
2956
2957 if (ok)
2958 {
2959 addr3 = addr2;
2960 addr2 = addr2->next;
2961 }
2962
2963 /* This address is to be deferred. Take it out of the chain, and
2964 post-process it as complete. Must take it out of the chain first,
2965 because post processing puts it on another chain. */
2966
2967 else
2968 {
2969 address_item *this = addr2;
2970 this->message = US"Retry time not yet reached";
2971 this->basic_errno = ERRNO_LRETRY;
2972 addr2 = addr3 ? (addr3->next = addr2->next)
2973 : (addr = addr2->next);
2974 post_process_one(this, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
2975 }
2976 }
2977
2978 if (dbm_file) dbfn_close(dbm_file);
2979
2980 /* If there are no addresses left on the chain, they all deferred. Loop
2981 for the next set of addresses. */
2982
2983 if (!addr) continue;
2984
2985 /* If the transport is limited for parallellism, enforce that here.
2986 We use a hints DB entry, incremented here and decremented after
2987 the transport (and any shadow transport) completes. */
2988
2989 if (tpt_parallel_check(tp, addr, &serialize_key))
2990 {
2991 if (expand_string_message)
2992 {
2993 logflags |= LOG_PANIC;
2994 do
2995 {
2996 addr = addr->next;
2997 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
2998 } while ((addr = addr2));
2999 }
3000 continue; /* Loop for the next set of addresses. */
3001 }
3002
3003
3004 /* So, finally, we do have some addresses that can be passed to the
3005 transport. Before doing so, set up variables that are relevant to a
3006 single delivery. */
3007
3008 deliver_set_expansions(addr);
3009
3010 gettimeofday(&delivery_start, NULL);
3011 deliver_local(addr, FALSE);
3012 timesince(&deliver_time, &delivery_start);
3013
3014 /* If a shadow transport (which must perforce be another local transport), is
3015 defined, and its condition is met, we must pass the message to the shadow
3016 too, but only those addresses that succeeded. We do this by making a new
3017 chain of addresses - also to keep the original chain uncontaminated. We must
3018 use a chain rather than doing it one by one, because the shadow transport may
3019 batch.
3020
3021 NOTE: if the condition fails because of a lookup defer, there is nothing we
3022 can do! */
3023
3024 if ( tp->shadow
3025 && ( !tp->shadow_condition
3026 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
3027 ) )
3028 {
3029 transport_instance *stp;
3030 address_item *shadow_addr = NULL;
3031 address_item **last = &shadow_addr;
3032
3033 for (stp = transports; stp; stp = stp->next)
3034 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
3035
3036 if (!stp)
3037 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
3038 tp->shadow);
3039
3040 /* Pick off the addresses that have succeeded, and make clones. Put into
3041 the shadow_message field a pointer to the shadow_message field of the real
3042 address. */
3043
3044 else for (addr2 = addr; addr2; addr2 = addr2->next)
3045 if (addr2->transport_return == OK)
3046 {
3047 addr3 = store_get(sizeof(address_item), FALSE);
3048 *addr3 = *addr2;
3049 addr3->next = NULL;
3050 addr3->shadow_message = US &addr2->shadow_message;
3051 addr3->transport = stp;
3052 addr3->transport_return = DEFER;
3053 addr3->return_filename = NULL;
3054 addr3->return_file = -1;
3055 *last = addr3;
3056 last = &addr3->next;
3057 }
3058
3059 /* If we found any addresses to shadow, run the delivery, and stick any
3060 message back into the shadow_message field in the original. */
3061
3062 if (shadow_addr)
3063 {
3064 int save_count = transport_count;
3065
3066 DEBUG(D_deliver|D_transport)
3067 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
3068 deliver_local(shadow_addr, TRUE);
3069
3070 for(; shadow_addr; shadow_addr = shadow_addr->next)
3071 {
3072 int sresult = shadow_addr->transport_return;
3073 *(uschar **)shadow_addr->shadow_message =
3074 sresult == OK
3075 ? string_sprintf(" ST=%s", stp->name)
3076 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
3077 shadow_addr->basic_errno <= 0
3078 ? US""
3079 : US strerror(shadow_addr->basic_errno),
3080 shadow_addr->basic_errno <= 0 || !shadow_addr->message
3081 ? US""
3082 : US": ",
3083 shadow_addr->message
3084 ? shadow_addr->message
3085 : shadow_addr->basic_errno <= 0
3086 ? US"unknown error"
3087 : US"");
3088
3089 DEBUG(D_deliver|D_transport)
3090 debug_printf("%s shadow transport returned %s for %s\n",
3091 stp->name, rc_to_string(sresult), shadow_addr->address);
3092 }
3093
3094 DEBUG(D_deliver|D_transport)
3095 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
3096
3097 transport_count = save_count; /* Restore original transport count */
3098 }
3099 }
3100
3101 /* Cancel the expansions that were set up for the delivery. */
3102
3103 deliver_set_expansions(NULL);
3104
3105 /* If the transport was parallelism-limited, decrement the hints DB record. */
3106
3107 if (serialize_key) enq_end(serialize_key);
3108
3109 /* Now we can process the results of the real transport. We must take each
3110 address off the chain first, because post_process_one() puts it on another
3111 chain. */
3112
3113 for (addr2 = addr; addr2; addr2 = nextaddr)
3114 {
3115 int result = addr2->transport_return;
3116 nextaddr = addr2->next;
3117
3118 DEBUG(D_deliver|D_transport)
3119 debug_printf("%s transport returned %s for %s\n",
3120 tp->name, rc_to_string(result), addr2->address);
3121
3122 /* If there is a retry_record, or if delivery is deferred, build a retry
3123 item for setting a new retry time or deleting the old retry record from
3124 the database. These items are handled all together after all addresses
3125 have been handled (so the database is open just for a short time for
3126 updating). */
3127
3128 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
3129 {
3130 int flags = result == DEFER ? 0 : rf_delete;
3131 uschar *retry_key = string_copy(tp->retry_use_local_part
3132 ? addr2->address_retry_key : addr2->domain_retry_key);
3133 *retry_key = 'T';
3134 retry_add_item(addr2, retry_key, flags);
3135 }
3136
3137 /* Done with this address */
3138
3139 addr2->delivery_time = deliver_time;
3140 post_process_one(addr2, result, logflags, EXIM_DTYPE_TRANSPORT, logchar);
3141
3142 /* If a pipe delivery generated text to be sent back, the result may be
3143 changed to FAIL, and we must copy this for subsequent addresses in the
3144 batch. */
3145
3146 if (addr2->transport_return != result)
3147 {
3148 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
3149 {
3150 addr3->transport_return = addr2->transport_return;
3151 addr3->basic_errno = addr2->basic_errno;
3152 addr3->message = addr2->message;
3153 }
3154 result = addr2->transport_return;
3155 }
3156
3157 /* Whether or not the result was changed to FAIL, we need to copy the
3158 return_file value from the first address into all the addresses of the
3159 batch, so they are all listed in the error message. */
3160
3161 addr2->return_file = addr->return_file;
3162
3163 /* Change log character for recording successful deliveries. */
3164
3165 if (result == OK) logchar = '-';
3166 }
3167 } /* Loop back for next batch of addresses */
3168 }
3169
3170
3171
3172
3173 /*************************************************
3174 * Sort remote deliveries *
3175 *************************************************/
3176
3177 /* This function is called if remote_sort_domains is set. It arranges that the
3178 chain of addresses for remote deliveries is ordered according to the strings
3179 specified. Try to make this shuffling reasonably efficient by handling
3180 sequences of addresses rather than just single ones.
3181
3182 Arguments: None
3183 Returns: Nothing
3184 */
3185
3186 static void
3187 sort_remote_deliveries(void)
3188 {
3189 int sep = 0;
3190 address_item **aptr = &addr_remote;
3191 const uschar *listptr = remote_sort_domains;
3192 uschar *pattern;
3193 uschar patbuf[256];
3194
3195 while ( *aptr
3196 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
3197 )
3198 {
3199 address_item *moved = NULL;
3200 address_item **bptr = &moved;
3201
3202 while (*aptr)
3203 {
3204 address_item **next;
3205 deliver_domain = (*aptr)->domain; /* set $domain */
3206 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
3207 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
3208 {
3209 aptr = &(*aptr)->next;
3210 continue;
3211 }
3212
3213 next = &(*aptr)->next;
3214 while ( *next
3215 && (deliver_domain = (*next)->domain, /* Set $domain */
3216 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
3217 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
3218 )
3219 next = &(*next)->next;
3220
3221 /* If the batch of non-matchers is at the end, add on any that were
3222 extracted further up the chain, and end this iteration. Otherwise,
3223 extract them from the chain and hang on the moved chain. */
3224
3225 if (!*next)
3226 {
3227 *next = moved;
3228 break;
3229 }
3230
3231 *bptr = *aptr;
3232 *aptr = *next;
3233 *next = NULL;
3234 bptr = next;
3235 aptr = &(*aptr)->next;
3236 }
3237
3238 /* If the loop ended because the final address matched, *aptr will
3239 be NULL. Add on to the end any extracted non-matching addresses. If
3240 *aptr is not NULL, the loop ended via "break" when *next is null, that
3241 is, there was a string of non-matching addresses at the end. In this
3242 case the extracted addresses have already been added on the end. */
3243
3244 if (!*aptr) *aptr = moved;
3245 }
3246
3247 DEBUG(D_deliver)
3248 {
3249 debug_printf("remote addresses after sorting:\n");
3250 for (address_item * addr = addr_remote; addr; addr = addr->next)
3251 debug_printf(" %s\n", addr->address);
3252 }
3253 }
3254
3255
3256
3257 /*************************************************
3258 * Read from pipe for remote delivery subprocess *
3259 *************************************************/
3260
3261 /* This function is called when the subprocess is complete, but can also be
3262 called before it is complete, in order to empty a pipe that is full (to prevent
3263 deadlock). It must therefore keep track of its progress in the parlist data
3264 block.
3265
3266 We read the pipe to get the delivery status codes and a possible error message
3267 for each address, optionally preceded by unusability data for the hosts and
3268 also by optional retry data.
3269
3270 Read in large chunks into the big buffer and then scan through, interpreting
3271 the data therein. In most cases, only a single read will be necessary. No
3272 individual item will ever be anywhere near 2500 bytes in length, so by ensuring
3273 that we read the next chunk when there is less than 2500 bytes left in the
3274 non-final chunk, we can assume each item is complete in the buffer before
3275 handling it. Each item is written using a single write(), which is atomic for
3276 small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
3277 often bigger) so even if we are reading while the subprocess is still going, we
3278 should never have only a partial item in the buffer.
3279
3280 hs12: This assumption is not true anymore, since we get quite large items (certificate
3281 information and such).
3282
3283 Argument:
3284 poffset the offset of the parlist item
3285 eop TRUE if the process has completed
3286
3287 Returns: TRUE if the terminating 'Z' item has been read,
3288 or there has been a disaster (i.e. no more data needed);
3289 FALSE otherwise
3290 */
3291
3292 static BOOL
3293 par_read_pipe(int poffset, BOOL eop)
3294 {
3295 host_item *h;
3296 pardata *p = parlist + poffset;
3297 address_item *addrlist = p->addrlist;
3298 address_item *addr = p->addr;
3299 pid_t pid = p->pid;
3300 int fd = p->fd;
3301
3302 uschar *msg = p->msg;
3303 BOOL done = p->done;
3304
3305 /* Loop through all items, reading from the pipe when necessary. The pipe
3306 used to be non-blocking. But I do not see a reason for using non-blocking I/O
3307 here, as the preceding select() tells us, if data is available for reading.
3308
3309 A read() on a "selected" handle should never block, but(!) it may return
3310 less data then we expected. (The buffer size we pass to read() shouldn't be
3311 understood as a "request", but as a "limit".)
3312
3313 Each separate item is written to the pipe in a timely manner. But, especially for
3314 larger items, the read(2) may already return partial data from the write(2).
3315
3316 The write is atomic mostly (depending on the amount written), but atomic does
3317 not imply "all or noting", it just is "not intermixed" with other writes on the
3318 same channel (pipe).
3319
3320 */
3321
3322 DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
3323 (int)p->pid, eop? "ended" : "not ended yet");
3324
3325 while (!done)
3326 {
3327 retry_item *r, **rp;
3328 uschar pipeheader[PIPE_HEADER_SIZE+1];
3329 uschar *id = &pipeheader[0];
3330 uschar *subid = &pipeheader[1];
3331 uschar *ptr = big_buffer;
3332 size_t required = PIPE_HEADER_SIZE; /* first the pipehaeder, later the data */
3333 ssize_t got;
3334
3335 DEBUG(D_deliver) debug_printf(
3336 "expect %lu bytes (pipeheader) from tpt process %d\n", (u_long)required, pid);
3337
3338 /* We require(!) all the PIPE_HEADER_SIZE bytes here, as we know,
3339 they're written in a timely manner, so waiting for the write shouldn't hurt a lot.
3340 If we get less, we can assume the subprocess do be done and do not expect any further
3341 information from it. */
3342
3343 if ((got = readn(fd, pipeheader, required)) != required)
3344 {
3345 msg = string_sprintf("got " SSIZE_T_FMT " of %d bytes (pipeheader) "
3346 "from transport process %d for transport %s",
3347 got, PIPE_HEADER_SIZE, pid, addr->transport->driver_name);
3348 done = TRUE;
3349 break;
3350 }
3351
3352 pipeheader[PIPE_HEADER_SIZE] = '\0';
3353 DEBUG(D_deliver)
3354 debug_printf("got %ld bytes (pipeheader) from transport process %d\n",
3355 (long) got, pid);
3356
3357 {
3358 /* If we can't decode the pipeheader, the subprocess seems to have a
3359 problem, we do not expect any furher information from it. */
3360 char *endc;
3361 required = Ustrtol(pipeheader+2, &endc, 10);
3362 if (*endc)
3363 {
3364 msg = string_sprintf("failed to read pipe "
3365 "from transport process %d for transport %s: error decoding size from header",
3366 pid, addr->transport->driver_name);
3367 done = TRUE;
3368 break;
3369 }
3370 }
3371
3372 DEBUG(D_deliver)
3373 debug_printf("expect %lu bytes (pipedata) from transport process %d\n",
3374 (u_long)required, pid);
3375
3376 /* Same as above, the transport process will write the bytes announced
3377 in a timely manner, so we can just wait for the bytes, getting less than expected
3378 is considered a problem of the subprocess, we do not expect anything else from it. */
3379 if ((got = readn(fd, big_buffer, required)) != required)
3380 {
3381 msg = string_sprintf("got only " SSIZE_T_FMT " of " SIZE_T_FMT
3382 " bytes (pipedata) from transport process %d for transport %s",
3383 got, required, pid, addr->transport->driver_name);
3384 done = TRUE;
3385 break;
3386 }
3387
3388 /* Handle each possible type of item, assuming the complete item is
3389 available in store. */
3390
3391 switch (*id)
3392 {
3393 /* Host items exist only if any hosts were marked unusable. Match
3394 up by checking the IP address. */
3395
3396 case 'H':
3397 for (h = addrlist->host_list; h; h = h->next)
3398 {
3399 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
3400 h->status = ptr[0];
3401 h->why = ptr[1];
3402 }
3403 ptr += 2;
3404 while (*ptr++);
3405 break;
3406
3407 /* Retry items are sent in a preceding R item for each address. This is
3408 kept separate to keep each message short enough to guarantee it won't
3409 be split in the pipe. Hopefully, in the majority of cases, there won't in
3410 fact be any retry items at all.
3411
3412 The complete set of retry items might include an item to delete a
3413 routing retry if there was a previous routing delay. However, routing
3414 retries are also used when a remote transport identifies an address error.
3415 In that case, there may also be an "add" item for the same key. Arrange
3416 that a "delete" item is dropped in favour of an "add" item. */
3417
3418 case 'R':
3419 if (!addr) goto ADDR_MISMATCH;
3420
3421 DEBUG(D_deliver|D_retry)
3422 debug_printf("reading retry information for %s from subprocess\n",
3423 ptr+1);
3424
3425 /* Cut out any "delete" items on the list. */
3426
3427 for (rp = &addr->retries; (r = *rp); rp = &r->next)
3428 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3429 {
3430 if (!(r->flags & rf_delete)) break; /* It was not "delete" */
3431 *rp = r->next; /* Excise a delete item */
3432 DEBUG(D_deliver|D_retry)
3433 debug_printf(" existing delete item dropped\n");
3434 }
3435
3436 /* We want to add a delete item only if there is no non-delete item;
3437 however we still have to step ptr through the data. */
3438
3439 if (!r || !(*ptr & rf_delete))
3440 {
3441 r = store_get(sizeof(retry_item), FALSE);
3442 r->next = addr->retries;
3443 addr->retries = r;
3444 r->flags = *ptr++;
3445 r->key = string_copy(ptr);
3446 while (*ptr++);
3447 memcpy(&r->basic_errno, ptr, sizeof(r->basic_errno));
3448 ptr += sizeof(r->basic_errno);
3449 memcpy(&r->more_errno, ptr, sizeof(r->more_errno));
3450 ptr += sizeof(r->more_errno);
3451 r->message = *ptr ? string_copy(ptr) : NULL;
3452 DEBUG(D_deliver|D_retry) debug_printf(" added %s item\n",
3453 r->flags & rf_delete ? "delete" : "retry");
3454 }
3455
3456 else
3457 {
3458 DEBUG(D_deliver|D_retry)
3459 debug_printf(" delete item not added: non-delete item exists\n");
3460 ptr++;
3461 while(*ptr++);
3462 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3463 }
3464
3465 while(*ptr++);
3466 break;
3467
3468 /* Put the amount of data written into the parlist block */
3469
3470 case 'S':
3471 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3472 ptr += sizeof(transport_count);
3473 break;
3474
3475 /* Address items are in the order of items on the address chain. We
3476 remember the current address value in case this function is called
3477 several times to empty the pipe in stages. Information about delivery
3478 over TLS is sent in a preceding X item for each address. We don't put
3479 it in with the other info, in order to keep each message short enough to
3480 guarantee it won't be split in the pipe. */
3481
3482 #ifndef DISABLE_TLS
3483 case 'X':
3484 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
3485 switch (*subid)
3486 {
3487 case '1':
3488 addr->tlsver = addr->cipher = addr->peerdn = NULL;
3489
3490 if (*ptr)
3491 {
3492 addr->cipher = string_copy(ptr);
3493 addr->tlsver = string_copyn(ptr, Ustrchr(ptr, ':') - ptr);
3494 }
3495 while (*ptr++);
3496 if (*ptr)
3497 addr->peerdn = string_copy(ptr);
3498 break;
3499
3500 case '2':
3501 if (*ptr)
3502 (void) tls_import_cert(ptr, &addr->peercert);
3503 else
3504 addr->peercert = NULL;
3505 break;
3506
3507 case '3':
3508 if (*ptr)
3509 (void) tls_import_cert(ptr, &addr->ourcert);
3510 else
3511 addr->ourcert = NULL;
3512 break;
3513
3514 # ifndef DISABLE_OCSP
3515 case '4':
3516 addr->ocsp = *ptr ? *ptr - '0' : OCSP_NOT_REQ;
3517 break;
3518 # endif
3519 }
3520 while (*ptr++);
3521 break;
3522 #endif /*DISABLE_TLS*/
3523
3524 case 'C': /* client authenticator information */
3525 switch (*subid)
3526 {
3527 case '1': addr->authenticator = *ptr ? string_copy(ptr) : NULL; break;
3528 case '2': addr->auth_id = *ptr ? string_copy(ptr) : NULL; break;
3529 case '3': addr->auth_sndr = *ptr ? string_copy(ptr) : NULL; break;
3530 }
3531 while (*ptr++);
3532 break;
3533
3534 #ifndef DISABLE_PRDR
3535 case 'P':
3536 setflag(addr, af_prdr_used);
3537 break;
3538 #endif
3539
3540 case 'L':
3541 switch (*subid)
3542 {
3543 #ifndef DISABLE_PIPE_CONNECT
3544 case 2: setflag(addr, af_early_pipe); /*FALLTHROUGH*/
3545 #endif
3546 case 1: setflag(addr, af_pipelining); break;
3547 }
3548 break;
3549
3550 case 'K':
3551 setflag(addr, af_chunking_used);
3552 break;
3553
3554 case 'T':
3555 setflag(addr, af_tcp_fastopen_conn);
3556 if (*subid > '0') setflag(addr, af_tcp_fastopen);
3557 if (*subid > '1') setflag(addr, af_tcp_fastopen_data);
3558 break;
3559
3560 case 'D':
3561 if (!addr) goto ADDR_MISMATCH;
3562 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3563 ptr += sizeof(addr->dsn_aware);
3564 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
3565 break;
3566
3567 case 'A':
3568 if (!addr)
3569 {
3570 ADDR_MISMATCH:
3571 msg = string_sprintf("address count mismatch for data read from pipe "
3572 "for transport process %d for transport %s", pid,
3573 addrlist->transport->driver_name);
3574 done = TRUE;
3575 break;
3576 }
3577
3578 switch (*subid)
3579 {
3580 #ifdef SUPPORT_SOCKS
3581 case '2': /* proxy information; must arrive before A0 and applies to that addr XXX oops*/
3582 proxy_session = TRUE; /*XXX should this be cleared somewhere? */
3583 if (*ptr == 0)
3584 ptr++;
3585 else
3586 {
3587 proxy_local_address = string_copy(ptr);
3588 while(*ptr++);
3589 memcpy(&proxy_local_port, ptr, sizeof(proxy_local_port));
3590 ptr += sizeof(proxy_local_port);
3591 }
3592 break;
3593 #endif
3594
3595 #ifdef EXPERIMENTAL_DSN_INFO
3596 case '1': /* must arrive before A0, and applies to that addr */
3597 /* Two strings: smtp_greeting and helo_response */
3598 addr->smtp_greeting = string_copy(ptr);
3599 while(*ptr++);
3600 addr->helo_response = string_copy(ptr);
3601 while(*ptr++);
3602 break;
3603 #endif
3604
3605 case '0':
3606 DEBUG(D_deliver) debug_printf("A0 %s tret %d\n", addr->address, *ptr);
3607 addr->transport_return = *ptr++;
3608 addr->special_action = *ptr++;
3609 memcpy(&addr->basic_errno, ptr, sizeof(addr->basic_errno));
3610 ptr += sizeof(addr->basic_errno);
3611 memcpy(&addr->more_errno, ptr, sizeof(addr->more_errno));
3612 ptr += sizeof(addr->more_errno);
3613 memcpy(&addr->delivery_time, ptr, sizeof(addr->delivery_time));
3614 ptr += sizeof(addr->delivery_time);
3615 memcpy(&addr->flags, ptr, sizeof(addr->flags));
3616 ptr += sizeof(addr->flags);
3617 addr->message = *ptr ? string_copy(ptr) : NULL;
3618 while(*ptr++);
3619 addr->user_message = *ptr ? string_copy(ptr) : NULL;
3620 while(*ptr++);
3621
3622 /* Always two strings for host information, followed by the port number and DNSSEC mark */
3623
3624 if (*ptr)
3625 {
3626 h = store_get(sizeof(host_item), FALSE);
3627 h->name = string_copy(ptr);
3628 while (*ptr++);
3629 h->address = string_copy(ptr);
3630 while(*ptr++);
3631 memcpy(&h->port, ptr, sizeof(h->port));
3632 ptr += sizeof(h->port);
3633 h->dnssec = *ptr == '2' ? DS_YES
3634 : *ptr == '1' ? DS_NO
3635 : DS_UNK;
3636 ptr++;
3637 addr->host_used = h;
3638 }
3639 else ptr++;
3640
3641 /* Finished with this address */
3642
3643 addr = addr->next;
3644 break;
3645 }
3646 break;
3647
3648 /* Local interface address/port */
3649 case 'I':
3650 if (*ptr) sending_ip_address = string_copy(ptr);
3651 while (*ptr++) ;
3652 if (*ptr) sending_port = atoi(CS ptr);
3653 while (*ptr++) ;
3654 break;
3655
3656 /* Z marks the logical end of the data. It is followed by '0' if
3657 continue_transport was NULL at the end of transporting, otherwise '1'.
3658 We need to know when it becomes NULL during a delivery down a passed SMTP
3659 channel so that we don't try to pass anything more down it. Of course, for
3660 most normal messages it will remain NULL all the time. */
3661
3662 case 'Z':
3663 if (*ptr == '0')
3664 {
3665 continue_transport = NULL;
3666 continue_hostname = NULL;
3667 }
3668 done = TRUE;
3669 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
3670 break;
3671
3672 /* Anything else is a disaster. */
3673
3674 default:
3675 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3676 "process %d for transport %s", ptr[-1], pid,
3677 addr->transport->driver_name);
3678 done = TRUE;
3679 break;
3680 }
3681 }
3682
3683 /* The done flag is inspected externally, to determine whether or not to
3684 call the function again when the process finishes. */
3685
3686 p->done = done;
3687
3688 /* If the process hadn't finished, and we haven't seen the end of the data
3689 or if we suffered a disaster, update the rest of the state, and return FALSE to
3690 indicate "not finished". */
3691
3692 if (!eop && !done)
3693 {
3694 p->addr = addr;
3695 p->msg = msg;
3696 return FALSE;
3697 }
3698
3699 /* Close our end of the pipe, to prevent deadlock if the far end is still
3700 pushing stuff into it. */
3701
3702 (void)close(fd);
3703 p->fd = -1;
3704
3705 /* If we have finished without error, but haven't had data for every address,
3706 something is wrong. */
3707
3708 if (!msg && addr)
3709 msg = string_sprintf("insufficient address data read from pipe "
3710 "for transport process %d for transport %s", pid,
3711 addr->transport->driver_name);
3712
3713 /* If an error message is set, something has gone wrong in getting back
3714 the delivery data. Put the message into each address and freeze it. */
3715
3716 if (msg)
3717 for (addr = addrlist; addr; addr = addr->next)
3718 {
3719 addr->transport_return = DEFER;
3720 addr->special_action = SPECIAL_FREEZE;
3721 addr->message = msg;
3722 log_write(0, LOG_MAIN|LOG_PANIC, "Delivery status for %s: %s\n", addr->address, addr->message);
3723 }
3724
3725 /* Return TRUE to indicate we have got all we need from this process, even
3726 if it hasn't actually finished yet. */
3727
3728 return TRUE;
3729 }
3730
3731
3732
3733 /*************************************************
3734 * Post-process a set of remote addresses *
3735 *************************************************/
3736
3737 /* Do what has to be done immediately after a remote delivery for each set of
3738 addresses, then re-write the spool if necessary. Note that post_process_one
3739 puts the address on an appropriate queue; hence we must fish off the next
3740 one first. This function is also called if there is a problem with setting
3741 up a subprocess to do a remote delivery in parallel. In this case, the final
3742 argument contains a message, and the action must be forced to DEFER.
3743
3744 Argument:
3745 addr pointer to chain of address items
3746 logflags flags for logging
3747 msg NULL for normal cases; -> error message for unexpected problems
3748 fallback TRUE if processing fallback hosts
3749
3750 Returns: nothing
3751 */
3752
3753 static void
3754 remote_post_process(address_item *addr, int logflags, uschar *msg,
3755 BOOL fallback)
3756 {
3757 /* If any host addresses were found to be unusable, add them to the unusable
3758 tree so that subsequent deliveries don't try them. */
3759
3760 for (host_item * h = addr->host_list; h; h = h->next)
3761 if (h->address)
3762 if (h->status >= hstatus_unusable) tree_add_unusable(h);
3763
3764 /* Now handle each address on the chain. The transport has placed '=' or '-'
3765 into the special_action field for each successful delivery. */
3766
3767 while (addr)
3768 {
3769 address_item *next = addr->next;
3770
3771 /* If msg == NULL (normal processing) and the result is DEFER and we are
3772 processing the main hosts and there are fallback hosts available, put the
3773 address on the list for fallback delivery. */
3774
3775 if ( addr->transport_return == DEFER
3776 && addr->fallback_hosts
3777 && !fallback
3778 && !msg
3779 )
3780 {
3781 addr->host_list = addr->fallback_hosts;
3782 addr->next = addr_fallback;
3783 addr_fallback = addr;
3784 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3785 }
3786
3787 /* If msg is set (=> unexpected problem), set it in the address before
3788 doing the ordinary post processing. */
3789
3790 else
3791 {
3792 if (msg)
3793 {
3794 addr->message = msg;
3795 addr->transport_return = DEFER;
3796 }
3797 (void)post_process_one(addr, addr->transport_return, logflags,
3798 EXIM_DTYPE_TRANSPORT, addr->special_action);
3799 }
3800
3801 /* Next address */
3802
3803 addr = next;
3804 }
3805
3806 /* If we have just delivered down a passed SMTP channel, and that was
3807 the last address, the channel will have been closed down. Now that
3808 we have logged that delivery, set continue_sequence to 1 so that
3809 any subsequent deliveries don't get "*" incorrectly logged. */
3810
3811 if (!continue_transport) continue_sequence = 1;
3812 }
3813
3814
3815
3816 /*************************************************
3817 * Wait for one remote delivery subprocess *
3818 *************************************************/
3819
3820 /* This function is called while doing remote deliveries when either the
3821 maximum number of processes exist and we need one to complete so that another
3822 can be created, or when waiting for the last ones to complete. It must wait for
3823 the completion of one subprocess, empty the control block slot, and return a
3824 pointer to the address chain.
3825
3826 Arguments: none
3827 Returns: pointer to the chain of addresses handled by the process;
3828 NULL if no subprocess found - this is an unexpected error
3829 */
3830
3831 static address_item *
3832 par_wait(void)
3833 {
3834 int poffset, status;
3835 address_item *addr, *addrlist;
3836 pid_t pid;
3837
3838 set_process_info("delivering %s: waiting for a remote delivery subprocess "
3839 "to finish", message_id);
3840
3841 /* Loop until either a subprocess completes, or there are no subprocesses in
3842 existence - in which case give an error return. We cannot proceed just by
3843 waiting for a completion, because a subprocess may have filled up its pipe, and
3844 be waiting for it to be emptied. Therefore, if no processes have finished, we
3845 wait for one of the pipes to acquire some data by calling select(), with a
3846 timeout just in case.
3847
3848 The simple approach is just to iterate after reading data from a ready pipe.
3849 This leads to non-ideal behaviour when the subprocess has written its final Z
3850 item, closed the pipe, and is in the process of exiting (the common case). A
3851 call to waitpid() yields nothing completed, but select() shows the pipe ready -
3852 reading it yields EOF, so you end up with busy-waiting until the subprocess has
3853 actually finished.
3854
3855 To avoid this, if all the data that is needed has been read from a subprocess
3856 after select(), an explicit wait() for it is done. We know that all it is doing
3857 is writing to the pipe and then exiting, so the wait should not be long.
3858
3859 The non-blocking waitpid() is to some extent just insurance; if we could
3860 reliably detect end-of-file on the pipe, we could always know when to do a
3861 blocking wait() for a completed process. However, because some systems use
3862 NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3863 use code that functions without the need to recognize EOF.
3864
3865 There's a double loop here just in case we end up with a process that is not in
3866 the list of remote delivery processes. Something has obviously gone wrong if
3867 this is the case. (For example, a process that is incorrectly left over from
3868 routing or local deliveries might be found.) The damage can be minimized by
3869 looping back and looking for another process. If there aren't any, the error
3870 return will happen. */
3871
3872 for (;;) /* Normally we do not repeat this loop */
3873 {
3874 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3875 {
3876 struct timeval tv;
3877 fd_set select_pipes;
3878 int maxpipe, readycount;
3879
3880 /* A return value of -1 can mean several things. If errno != ECHILD, it
3881 either means invalid options (which we discount), or that this process was
3882 interrupted by a signal. Just loop to try the waitpid() again.
3883
3884 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3885 in existence. This should never happen, and is an unexpected error.
3886 However, there is a nasty complication when running under Linux. If "strace
3887 -f" is being used under Linux to trace this process and its children,
3888 subprocesses are "stolen" from their parents and become the children of the
3889 tracing process. A general wait such as the one we've just obeyed returns
3890 as if there are no children while subprocesses are running. Once a
3891 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3892 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3893 palliative.
3894
3895 This does not happen using "truss" on Solaris, nor (I think) with other
3896 tracing facilities on other OS. It seems to be specific to Linux.
3897
3898 What we do to get round this is to use kill() to see if any of our
3899 subprocesses are still in existence. If kill() gives an OK return, we know
3900 it must be for one of our processes - it can't be for a re-use of the pid,
3901 because if our process had finished, waitpid() would have found it. If any
3902 of our subprocesses are in existence, we proceed to use select() as if
3903 waitpid() had returned zero. I think this is safe. */
3904
3905 if (pid < 0)
3906 {
3907 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3908
3909 DEBUG(D_deliver)
3910 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3911 "for process existence\n");
3912
3913 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3914 {
3915 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3916 {
3917 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3918 "stolen by strace\n", (int)pid);
3919 break; /* With poffset set */
3920 }
3921 }
3922
3923 if (poffset >= remote_max_parallel)
3924 {
3925 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3926 return NULL; /* This is the error return */
3927 }
3928 }
3929
3930 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3931 been handled above. A return value of zero means that there is at least one
3932 subprocess, but there are no completed subprocesses. See if any pipes are
3933 ready with any data for reading. */
3934
3935 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3936
3937 maxpipe = 0;
3938 FD_ZERO(&select_pipes);
3939 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3940 if (parlist[poffset].pid != 0)
3941 {
3942 int fd = parlist[poffset].fd;
3943 FD_SET(fd, &select_pipes);
3944 if (fd > maxpipe) maxpipe = fd;
3945 }
3946
3947 /* Stick in a 60-second timeout, just in case. */
3948
3949 tv.tv_sec = 60;
3950 tv.tv_usec = 0;
3951
3952 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3953 NULL, NULL, &tv);
3954
3955 /* Scan through the pipes and read any that are ready; use the count
3956 returned by select() to stop when there are no more. Select() can return
3957 with no processes (e.g. if interrupted). This shouldn't matter.
3958
3959 If par_read_pipe() returns TRUE, it means that either the terminating Z was
3960 read, or there was a disaster. In either case, we are finished with this
3961 process. Do an explicit wait() for the process and break the main loop if
3962 it succeeds.
3963
3964 It turns out that we have to deal with the case of an interrupted system
3965 call, which can happen on some operating systems if the signal handling is
3966 set up to do that by default. */
3967
3968 for (poffset = 0;
3969 readycount > 0 && poffset < remote_max_parallel;
3970 poffset++)
3971 {
3972 if ( (pid = parlist[poffset].pid) != 0
3973 && FD_ISSET(parlist[poffset].fd, &select_pipes)
3974 )
3975 {
3976 readycount--;
3977 if (par_read_pipe(poffset, FALSE)) /* Finished with this pipe */
3978 for (;;) /* Loop for signals */
3979 {
3980 pid_t endedpid = waitpid(pid, &status, 0);
3981 if (endedpid == pid) goto PROCESS_DONE;
3982 if (endedpid != (pid_t)(-1) || errno != EINTR)
3983 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return "
3984 "%d (errno = %d) from waitpid() for process %d",
3985 (int)endedpid, errno, (int)pid);
3986 }
3987 }
3988 }
3989
3990 /* Now go back and look for a completed subprocess again. */
3991 }
3992
3993 /* A completed process was detected by the non-blocking waitpid(). Find the
3994 data block that corresponds to this subprocess. */
3995
3996 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3997 if (pid == parlist[poffset].pid) break;
3998
3999 /* Found the data block; this is a known remote delivery process. We don't
4000 need to repeat the outer loop. This should be what normally happens. */
4001
4002 if (poffset < remote_max_parallel) break;
4003
4004 /* This situation is an error, but it's probably better to carry on looking
4005 for another process than to give up (as we used to do). */
4006
4007 log_write(0, LOG_MAIN|LOG_PANIC, "Process %d finished: not found in remote "
4008 "transport process list", pid);
4009 } /* End of the "for" loop */
4010
4011 /* Come here when all the data was completely read after a select(), and
4012 the process in pid has been wait()ed for. */
4013
4014 PROCESS_DONE:
4015
4016 DEBUG(D_deliver)
4017 {
4018 if (status == 0)
4019 debug_printf("remote delivery process %d ended\n", (int)pid);
4020 else
4021 debug_printf("remote delivery process %d ended: status=%04x\n", (int)pid,
4022 status);
4023 }
4024
4025 set_process_info("delivering %s", message_id);
4026
4027 /* Get the chain of processed addresses */
4028
4029 addrlist = parlist[poffset].addrlist;
4030
4031 /* If the process did not finish cleanly, record an error and freeze (except
4032 for SIGTERM, SIGKILL and SIGQUIT), and also ensure the journal is not removed,
4033 in case the delivery did actually happen. */
4034
4035 if ((status & 0xffff) != 0)
4036 {
4037 uschar *msg;
4038 int msb = (status >> 8) & 255;
4039 int lsb = status & 255;
4040 int code = (msb == 0)? (lsb & 0x7f) : msb;
4041
4042 msg = string_sprintf("%s transport process returned non-zero status 0x%04x: "
4043 "%s %d",
4044 addrlist->transport->driver_name,
4045 status,