9f292fc535ca7124f363ad9ad2fa6d88b6b3be91
[exim.git] / src / src / deliver.c
1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
4
5 /* Copyright (c) University of Cambridge 1995 - 2018 */
6 /* See the file NOTICE for conditions of use and distribution. */
7
8 /* The main code for delivering a message. */
9
10
11 #include "exim.h"
12 #include "transports/smtp.h"
13 #include <sys/uio.h>
14 #include <assert.h>
15
16
17 /* Data block for keeping track of subprocesses for parallel remote
18 delivery. */
19
20 typedef struct pardata {
21 address_item *addrlist; /* chain of addresses */
22 address_item *addr; /* next address data expected for */
23 pid_t pid; /* subprocess pid */
24 int fd; /* pipe fd for getting result from subprocess */
25 int transport_count; /* returned transport count value */
26 BOOL done; /* no more data needed */
27 uschar *msg; /* error message */
28 uschar *return_path; /* return_path for these addresses */
29 } pardata;
30
31 /* Values for the process_recipients variable */
32
33 enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
34 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
35 RECIP_FAIL_LOOP};
36
37 /* Mutually recursive functions for marking addresses done. */
38
39 static void child_done(address_item *, uschar *);
40 static void address_done(address_item *, uschar *);
41
42 /* Table for turning base-62 numbers into binary */
43
44 static uschar tab62[] =
45 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
46 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
47 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
48 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
49 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
50 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
51 59,60,61}; /* x-z */
52
53
54 /*************************************************
55 * Local static variables *
56 *************************************************/
57
58 /* addr_duplicate is global because it needs to be seen from the Envelope-To
59 writing code. */
60
61 static address_item *addr_defer = NULL;
62 static address_item *addr_failed = NULL;
63 static address_item *addr_fallback = NULL;
64 static address_item *addr_local = NULL;
65 static address_item *addr_new = NULL;
66 static address_item *addr_remote = NULL;
67 static address_item *addr_route = NULL;
68 static address_item *addr_succeed = NULL;
69 static address_item *addr_dsntmp = NULL;
70 static address_item *addr_senddsn = NULL;
71
72 static FILE *message_log = NULL;
73 static BOOL update_spool;
74 static BOOL remove_journal;
75 static int parcount = 0;
76 static pardata *parlist = NULL;
77 static int return_count;
78 static uschar *frozen_info = US"";
79 static uschar *used_return_path = NULL;
80
81
82
83 /*************************************************
84 * read as much as requested *
85 *************************************************/
86
87 /* The syscall read(2) doesn't always returns as much as we want. For
88 several reasons it might get less. (Not talking about signals, as syscalls
89 are restartable). When reading from a network or pipe connection the sender
90 might send in smaller chunks, with delays between these chunks. The read(2)
91 may return such a chunk.
92
93 The more the writer writes and the smaller the pipe between write and read is,
94 the more we get the chance of reading leass than requested. (See bug 2130)
95
96 This function read(2)s until we got all the data we *requested*.
97
98 Note: This function may block. Use it only if you're sure about the
99 amount of data you will get.
100
101 Argument:
102 fd the file descriptor to read from
103 buffer pointer to a buffer of size len
104 len the requested(!) amount of bytes
105
106 Returns: the amount of bytes read
107 */
108 static ssize_t
109 readn(int fd, void * buffer, size_t len)
110 {
111 void * next = buffer;
112 void * end = buffer + len;
113
114 while (next < end)
115 {
116 ssize_t got = read(fd, next, end - next);
117
118 /* I'm not sure if there are signals that can interrupt us,
119 for now I assume the worst */
120 if (got == -1 && errno == EINTR) continue;
121 if (got <= 0) return next - buffer;
122 next += got;
123 }
124
125 return len;
126 }
127
128
129 /*************************************************
130 * Make a new address item *
131 *************************************************/
132
133 /* This function gets the store and initializes with default values. The
134 transport_return value defaults to DEFER, so that any unexpected failure to
135 deliver does not wipe out the message. The default unique string is set to a
136 copy of the address, so that its domain can be lowercased.
137
138 Argument:
139 address the RFC822 address string
140 copy force a copy of the address
141
142 Returns: a pointer to an initialized address_item
143 */
144
145 address_item *
146 deliver_make_addr(uschar *address, BOOL copy)
147 {
148 address_item *addr = store_get(sizeof(address_item));
149 *addr = address_defaults;
150 if (copy) address = string_copy(address);
151 addr->address = address;
152 addr->unique = string_copy(address);
153 return addr;
154 }
155
156
157
158
159 /*************************************************
160 * Set expansion values for an address *
161 *************************************************/
162
163 /* Certain expansion variables are valid only when handling an address or
164 address list. This function sets them up or clears the values, according to its
165 argument.
166
167 Arguments:
168 addr the address in question, or NULL to clear values
169 Returns: nothing
170 */
171
172 void
173 deliver_set_expansions(address_item *addr)
174 {
175 if (!addr)
176 {
177 const uschar ***p = address_expansions;
178 while (*p) **p++ = NULL;
179 return;
180 }
181
182 /* Exactly what gets set depends on whether there is one or more addresses, and
183 what they contain. These first ones are always set, taking their values from
184 the first address. */
185
186 if (!addr->host_list)
187 {
188 deliver_host = deliver_host_address = US"";
189 deliver_host_port = 0;
190 }
191 else
192 {
193 deliver_host = addr->host_list->name;
194 deliver_host_address = addr->host_list->address;
195 deliver_host_port = addr->host_list->port;
196 }
197
198 deliver_recipients = addr;
199 deliver_address_data = addr->prop.address_data;
200 deliver_domain_data = addr->prop.domain_data;
201 deliver_localpart_data = addr->prop.localpart_data;
202
203 /* These may be unset for multiple addresses */
204
205 deliver_domain = addr->domain;
206 self_hostname = addr->self_hostname;
207
208 #ifdef EXPERIMENTAL_BRIGHTMAIL
209 bmi_deliver = 1; /* deliver by default */
210 bmi_alt_location = NULL;
211 bmi_base64_verdict = NULL;
212 bmi_base64_tracker_verdict = NULL;
213 #endif
214
215 /* If there's only one address we can set everything. */
216
217 if (!addr->next)
218 {
219 address_item *addr_orig;
220
221 deliver_localpart = addr->local_part;
222 deliver_localpart_prefix = addr->prefix;
223 deliver_localpart_suffix = addr->suffix;
224
225 for (addr_orig = addr; addr_orig->parent; addr_orig = addr_orig->parent) ;
226 deliver_domain_orig = addr_orig->domain;
227
228 /* Re-instate any prefix and suffix in the original local part. In all
229 normal cases, the address will have a router associated with it, and we can
230 choose the caseful or caseless version accordingly. However, when a system
231 filter sets up a pipe, file, or autoreply delivery, no router is involved.
232 In this case, though, there won't be any prefix or suffix to worry about. */
233
234 deliver_localpart_orig = !addr_orig->router
235 ? addr_orig->local_part
236 : addr_orig->router->caseful_local_part
237 ? addr_orig->cc_local_part
238 : addr_orig->lc_local_part;
239
240 /* If there's a parent, make its domain and local part available, and if
241 delivering to a pipe or file, or sending an autoreply, get the local
242 part from the parent. For pipes and files, put the pipe or file string
243 into address_pipe and address_file. */
244
245 if (addr->parent)
246 {
247 deliver_domain_parent = addr->parent->domain;
248 deliver_localpart_parent = !addr->parent->router
249 ? addr->parent->local_part
250 : addr->parent->router->caseful_local_part
251 ? addr->parent->cc_local_part
252 : addr->parent->lc_local_part;
253
254 /* File deliveries have their own flag because they need to be picked out
255 as special more often. */
256
257 if (testflag(addr, af_pfr))
258 {
259 if (testflag(addr, af_file)) address_file = addr->local_part;
260 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
261 deliver_localpart = addr->parent->local_part;
262 deliver_localpart_prefix = addr->parent->prefix;
263 deliver_localpart_suffix = addr->parent->suffix;
264 }
265 }
266
267 #ifdef EXPERIMENTAL_BRIGHTMAIL
268 /* Set expansion variables related to Brightmail AntiSpam */
269 bmi_base64_verdict = bmi_get_base64_verdict(deliver_localpart_orig, deliver_domain_orig);
270 bmi_base64_tracker_verdict = bmi_get_base64_tracker_verdict(bmi_base64_verdict);
271 /* get message delivery status (0 - don't deliver | 1 - deliver) */
272 bmi_deliver = bmi_get_delivery_status(bmi_base64_verdict);
273 /* if message is to be delivered, get eventual alternate location */
274 if (bmi_deliver == 1)
275 bmi_alt_location = bmi_get_alt_location(bmi_base64_verdict);
276 #endif
277
278 }
279
280 /* For multiple addresses, don't set local part, and leave the domain and
281 self_hostname set only if it is the same for all of them. It is possible to
282 have multiple pipe and file addresses, but only when all addresses have routed
283 to the same pipe or file. */
284
285 else
286 {
287 address_item *addr2;
288 if (testflag(addr, af_pfr))
289 {
290 if (testflag(addr, af_file)) address_file = addr->local_part;
291 else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
292 }
293 for (addr2 = addr->next; addr2; addr2 = addr2->next)
294 {
295 if (deliver_domain && Ustrcmp(deliver_domain, addr2->domain) != 0)
296 deliver_domain = NULL;
297 if ( self_hostname
298 && ( !addr2->self_hostname
299 || Ustrcmp(self_hostname, addr2->self_hostname) != 0
300 ) )
301 self_hostname = NULL;
302 if (!deliver_domain && !self_hostname) break;
303 }
304 }
305 }
306
307
308
309
310 /*************************************************
311 * Open a msglog file *
312 *************************************************/
313
314 /* This function is used both for normal message logs, and for files in the
315 msglog directory that are used to catch output from pipes. Try to create the
316 directory if it does not exist. From release 4.21, normal message logs should
317 be created when the message is received.
318
319 Called from deliver_message(), can be operating as root.
320
321 Argument:
322 filename the file name
323 mode the mode required
324 error used for saying what failed
325
326 Returns: a file descriptor, or -1 (with errno set)
327 */
328
329 static int
330 open_msglog_file(uschar *filename, int mode, uschar **error)
331 {
332 int fd, i;
333
334 for (i = 2; i > 0; i--)
335 {
336 fd = Uopen(filename,
337 #ifdef O_CLOEXEC
338 O_CLOEXEC |
339 #endif
340 #ifdef O_NOFOLLOW
341 O_NOFOLLOW |
342 #endif
343 O_WRONLY|O_APPEND|O_CREAT, mode);
344 if (fd >= 0)
345 {
346 /* Set the close-on-exec flag and change the owner to the exim uid/gid (this
347 function is called as root). Double check the mode, because the group setting
348 doesn't always get set automatically. */
349
350 #ifndef O_CLOEXEC
351 (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
352 #endif
353 if (fchown(fd, exim_uid, exim_gid) < 0)
354 {
355 *error = US"chown";
356 return -1;
357 }
358 if (fchmod(fd, mode) < 0)
359 {
360 *error = US"chmod";
361 return -1;
362 }
363 return fd;
364 }
365 if (errno != ENOENT)
366 break;
367
368 (void)directory_make(spool_directory,
369 spool_sname(US"msglog", message_subdir),
370 MSGLOG_DIRECTORY_MODE, TRUE);
371 }
372
373 *error = US"create";
374 return -1;
375 }
376
377
378
379
380 /*************************************************
381 * Write to msglog if required *
382 *************************************************/
383
384 /* Write to the message log, if configured. This function may also be called
385 from transports.
386
387 Arguments:
388 format a string format
389
390 Returns: nothing
391 */
392
393 void
394 deliver_msglog(const char *format, ...)
395 {
396 va_list ap;
397 if (!message_logs) return;
398 va_start(ap, format);
399 vfprintf(message_log, format, ap);
400 fflush(message_log);
401 va_end(ap);
402 }
403
404
405
406
407 /*************************************************
408 * Replicate status for batch *
409 *************************************************/
410
411 /* When a transport handles a batch of addresses, it may treat them
412 individually, or it may just put the status in the first one, and return FALSE,
413 requesting that the status be copied to all the others externally. This is the
414 replication function. As well as the status, it copies the transport pointer,
415 which may have changed if appendfile passed the addresses on to a different
416 transport.
417
418 Argument: pointer to the first address in a chain
419 Returns: nothing
420 */
421
422 static void
423 replicate_status(address_item *addr)
424 {
425 address_item *addr2;
426 for (addr2 = addr->next; addr2; addr2 = addr2->next)
427 {
428 addr2->transport = addr->transport;
429 addr2->transport_return = addr->transport_return;
430 addr2->basic_errno = addr->basic_errno;
431 addr2->more_errno = addr->more_errno;
432 addr2->delivery_usec = addr->delivery_usec;
433 addr2->special_action = addr->special_action;
434 addr2->message = addr->message;
435 addr2->user_message = addr->user_message;
436 }
437 }
438
439
440
441 /*************************************************
442 * Compare lists of hosts *
443 *************************************************/
444
445 /* This function is given two pointers to chains of host items, and it yields
446 TRUE if the lists refer to the same hosts in the same order, except that
447
448 (1) Multiple hosts with the same non-negative MX values are permitted to appear
449 in different orders. Round-robinning nameservers can cause this to happen.
450
451 (2) Multiple hosts with the same negative MX values less than MX_NONE are also
452 permitted to appear in different orders. This is caused by randomizing
453 hosts lists.
454
455 This enables Exim to use a single SMTP transaction for sending to two entirely
456 different domains that happen to end up pointing at the same hosts.
457
458 Arguments:
459 one points to the first host list
460 two points to the second host list
461
462 Returns: TRUE if the lists refer to the same host set
463 */
464
465 static BOOL
466 same_hosts(host_item *one, host_item *two)
467 {
468 while (one && two)
469 {
470 if (Ustrcmp(one->name, two->name) != 0)
471 {
472 int mx = one->mx;
473 host_item *end_one = one;
474 host_item *end_two = two;
475
476 /* Batch up only if there was no MX and the list was not randomized */
477
478 if (mx == MX_NONE) return FALSE;
479
480 /* Find the ends of the shortest sequence of identical MX values */
481
482 while ( end_one->next && end_one->next->mx == mx
483 && end_two->next && end_two->next->mx == mx)
484 {
485 end_one = end_one->next;
486 end_two = end_two->next;
487 }
488
489 /* If there aren't any duplicates, there's no match. */
490
491 if (end_one == one) return FALSE;
492
493 /* For each host in the 'one' sequence, check that it appears in the 'two'
494 sequence, returning FALSE if not. */
495
496 for (;;)
497 {
498 host_item *hi;
499 for (hi = two; hi != end_two->next; hi = hi->next)
500 if (Ustrcmp(one->name, hi->name) == 0) break;
501 if (hi == end_two->next) return FALSE;
502 if (one == end_one) break;
503 one = one->next;
504 }
505
506 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
507 Ensure both are pointing at the last host, and carry on as for equality. */
508
509 two = end_two;
510 }
511
512 /* if the names matched but ports do not, mismatch */
513 else if (one->port != two->port)
514 return FALSE;
515
516 /* Hosts matched */
517
518 one = one->next;
519 two = two->next;
520 }
521
522 /* True if both are NULL */
523
524 return (one == two);
525 }
526
527
528
529 /*************************************************
530 * Compare header lines *
531 *************************************************/
532
533 /* This function is given two pointers to chains of header items, and it yields
534 TRUE if they are the same header texts in the same order.
535
536 Arguments:
537 one points to the first header list
538 two points to the second header list
539
540 Returns: TRUE if the lists refer to the same header set
541 */
542
543 static BOOL
544 same_headers(header_line *one, header_line *two)
545 {
546 for (;; one = one->next, two = two->next)
547 {
548 if (one == two) return TRUE; /* Includes the case where both NULL */
549 if (!one || !two) return FALSE;
550 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
551 }
552 }
553
554
555
556 /*************************************************
557 * Compare string settings *
558 *************************************************/
559
560 /* This function is given two pointers to strings, and it returns
561 TRUE if they are the same pointer, or if the two strings are the same.
562
563 Arguments:
564 one points to the first string
565 two points to the second string
566
567 Returns: TRUE or FALSE
568 */
569
570 static BOOL
571 same_strings(uschar *one, uschar *two)
572 {
573 if (one == two) return TRUE; /* Includes the case where both NULL */
574 if (!one || !two) return FALSE;
575 return (Ustrcmp(one, two) == 0);
576 }
577
578
579
580 /*************************************************
581 * Compare uid/gid for addresses *
582 *************************************************/
583
584 /* This function is given a transport and two addresses. It yields TRUE if the
585 uid/gid/initgroups settings for the two addresses are going to be the same when
586 they are delivered.
587
588 Arguments:
589 tp the transort
590 addr1 the first address
591 addr2 the second address
592
593 Returns: TRUE or FALSE
594 */
595
596 static BOOL
597 same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
598 {
599 if ( !tp->uid_set && !tp->expand_uid
600 && !tp->deliver_as_creator
601 && ( testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set)
602 || ( testflag(addr1, af_uid_set)
603 && ( addr1->uid != addr2->uid
604 || testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups)
605 ) ) ) )
606 return FALSE;
607
608 if ( !tp->gid_set && !tp->expand_gid
609 && ( testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set)
610 || ( testflag(addr1, af_gid_set)
611 && addr1->gid != addr2->gid
612 ) ) )
613 return FALSE;
614
615 return TRUE;
616 }
617
618
619
620
621 /*************************************************
622 * Record that an address is complete *
623 *************************************************/
624
625 /* This function records that an address is complete. This is straightforward
626 for most addresses, where the unique address is just the full address with the
627 domain lower cased. For homonyms (addresses that are the same as one of their
628 ancestors) their are complications. Their unique addresses have \x\ prepended
629 (where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
630 cousins.
631
632 Exim used to record the unique addresses of homonyms as "complete". This,
633 however, fails when the pattern of redirection varies over time (e.g. if taking
634 unseen copies at only some times of day) because the prepended numbers may vary
635 from one delivery run to the next. This problem is solved by never recording
636 prepended unique addresses as complete. Instead, when a homonymic address has
637 actually been delivered via a transport, we record its basic unique address
638 followed by the name of the transport. This is checked in subsequent delivery
639 runs whenever an address is routed to a transport.
640
641 If the completed address is a top-level one (has no parent, which means it
642 cannot be homonymic) we also add the original address to the non-recipients
643 tree, so that it gets recorded in the spool file and therefore appears as
644 "done" in any spool listings. The original address may differ from the unique
645 address in the case of the domain.
646
647 Finally, this function scans the list of duplicates, marks as done any that
648 match this address, and calls child_done() for their ancestors.
649
650 Arguments:
651 addr address item that has been completed
652 now current time as a string
653
654 Returns: nothing
655 */
656
657 static void
658 address_done(address_item *addr, uschar *now)
659 {
660 address_item *dup;
661
662 update_spool = TRUE; /* Ensure spool gets updated */
663
664 /* Top-level address */
665
666 if (!addr->parent)
667 {
668 tree_add_nonrecipient(addr->unique);
669 tree_add_nonrecipient(addr->address);
670 }
671
672 /* Homonymous child address */
673
674 else if (testflag(addr, af_homonym))
675 {
676 if (addr->transport)
677 tree_add_nonrecipient(
678 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
679 }
680
681 /* Non-homonymous child address */
682
683 else tree_add_nonrecipient(addr->unique);
684
685 /* Check the list of duplicate addresses and ensure they are now marked
686 done as well. */
687
688 for (dup = addr_duplicate; dup; dup = dup->next)
689 if (Ustrcmp(addr->unique, dup->unique) == 0)
690 {
691 tree_add_nonrecipient(dup->unique);
692 child_done(dup, now);
693 }
694 }
695
696
697
698
699 /*************************************************
700 * Decrease counts in parents and mark done *
701 *************************************************/
702
703 /* This function is called when an address is complete. If there is a parent
704 address, its count of children is decremented. If there are still other
705 children outstanding, the function exits. Otherwise, if the count has become
706 zero, address_done() is called to mark the parent and its duplicates complete.
707 Then loop for any earlier ancestors.
708
709 Arguments:
710 addr points to the completed address item
711 now the current time as a string, for writing to the message log
712
713 Returns: nothing
714 */
715
716 static void
717 child_done(address_item *addr, uschar *now)
718 {
719 address_item *aa;
720 while (addr->parent)
721 {
722 addr = addr->parent;
723 if (--addr->child_count > 0) return; /* Incomplete parent */
724 address_done(addr, now);
725
726 /* Log the completion of all descendents only when there is no ancestor with
727 the same original address. */
728
729 for (aa = addr->parent; aa; aa = aa->parent)
730 if (Ustrcmp(aa->address, addr->address) == 0) break;
731 if (aa) continue;
732
733 deliver_msglog("%s %s: children all complete\n", now, addr->address);
734 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
735 }
736 }
737
738
739
740 /*************************************************
741 * Delivery logging support functions *
742 *************************************************/
743
744 /* The LOGGING() checks in d_log_interface() are complicated for backwards
745 compatibility. When outgoing interface logging was originally added, it was
746 conditional on just incoming_interface (which is off by default). The
747 outgoing_interface option is on by default to preserve this behaviour, but
748 you can enable incoming_interface and disable outgoing_interface to get I=
749 fields on incoming lines only.
750
751 Arguments:
752 g The log line
753 addr The address to be logged
754
755 Returns: New value for s
756 */
757
758 static gstring *
759 d_log_interface(gstring * g)
760 {
761 if (LOGGING(incoming_interface) && LOGGING(outgoing_interface)
762 && sending_ip_address)
763 {
764 g = string_append(g, 2, US" I=[", sending_ip_address);
765 g = LOGGING(outgoing_port)
766 ? string_append(g, 2, US"]:", string_sprintf("%d", sending_port))
767 : string_catn(g, US"]", 1);
768 }
769 return g;
770 }
771
772
773
774 static gstring *
775 d_hostlog(gstring * g, address_item * addr)
776 {
777 host_item * h = addr->host_used;
778
779 g = string_append(g, 2, US" H=", h->name);
780
781 if (LOGGING(dnssec) && h->dnssec == DS_YES)
782 g = string_catn(g, US" DS", 3);
783
784 g = string_append(g, 3, US" [", h->address, US"]");
785
786 if (LOGGING(outgoing_port))
787 g = string_append(g, 2, US":", string_sprintf("%d", h->port));
788
789 #ifdef SUPPORT_SOCKS
790 if (LOGGING(proxy) && proxy_local_address)
791 {
792 g = string_append(g, 3, US" PRX=[", proxy_local_address, US"]");
793 if (LOGGING(outgoing_port))
794 g = string_append(g, 2, US":", string_sprintf("%d", proxy_local_port));
795 }
796 #endif
797
798 g = d_log_interface(g);
799
800 if (testflag(addr, af_tcp_fastopen))
801 g = string_catn(g, US" TFO*", testflag(addr, af_tcp_fastopen_data) ? 5 : 4);
802
803 return g;
804 }
805
806
807
808
809
810 #ifdef SUPPORT_TLS
811 static gstring *
812 d_tlslog(gstring * s, address_item * addr)
813 {
814 if (LOGGING(tls_cipher) && addr->cipher)
815 s = string_append(s, 2, US" X=", addr->cipher);
816 if (LOGGING(tls_certificate_verified) && addr->cipher)
817 s = string_append(s, 2, US" CV=",
818 testflag(addr, af_cert_verified)
819 ?
820 #ifdef SUPPORT_DANE
821 testflag(addr, af_dane_verified)
822 ? "dane"
823 :
824 #endif
825 "yes"
826 : "no");
827 if (LOGGING(tls_peerdn) && addr->peerdn)
828 s = string_append(s, 3, US" DN=\"", string_printing(addr->peerdn), US"\"");
829 return s;
830 }
831 #endif
832
833
834
835
836 #ifndef DISABLE_EVENT
837 uschar *
838 event_raise(uschar * action, const uschar * event, uschar * ev_data)
839 {
840 uschar * s;
841 if (action)
842 {
843 DEBUG(D_deliver)
844 debug_printf("Event(%s): event_action=|%s| delivery_IP=%s\n",
845 event,
846 action, deliver_host_address);
847
848 event_name = event;
849 event_data = ev_data;
850
851 if (!(s = expand_string(action)) && *expand_string_message)
852 log_write(0, LOG_MAIN|LOG_PANIC,
853 "failed to expand event_action %s in %s: %s\n",
854 event, transport_name ? transport_name : US"main", expand_string_message);
855
856 event_name = event_data = NULL;
857
858 /* If the expansion returns anything but an empty string, flag for
859 the caller to modify his normal processing
860 */
861 if (s && *s)
862 {
863 DEBUG(D_deliver)
864 debug_printf("Event(%s): event_action returned \"%s\"\n", event, s);
865 return s;
866 }
867 }
868 return NULL;
869 }
870
871 void
872 msg_event_raise(const uschar * event, const address_item * addr)
873 {
874 const uschar * save_domain = deliver_domain;
875 uschar * save_local = deliver_localpart;
876 const uschar * save_host = deliver_host;
877 const uschar * save_address = deliver_host_address;
878 const int save_port = deliver_host_port;
879
880 router_name = addr->router ? addr->router->name : NULL;
881 deliver_domain = addr->domain;
882 deliver_localpart = addr->local_part;
883 deliver_host = addr->host_used ? addr->host_used->name : NULL;
884
885 if (!addr->transport)
886 {
887 if (Ustrcmp(event, "msg:fail:delivery") == 0)
888 {
889 /* An address failed with no transport involved. This happens when
890 a filter was used which triggered a fail command (in such a case
891 a transport isn't needed). Convert it to an internal fail event. */
892
893 (void) event_raise(event_action, US"msg:fail:internal", addr->message);
894 }
895 }
896 else
897 {
898 transport_name = addr->transport->name;
899
900 (void) event_raise(addr->transport->event_action, event,
901 addr->host_used
902 || Ustrcmp(addr->transport->driver_name, "smtp") == 0
903 || Ustrcmp(addr->transport->driver_name, "lmtp") == 0
904 || Ustrcmp(addr->transport->driver_name, "autoreply") == 0
905 ? addr->message : NULL);
906 }
907
908 deliver_host_port = save_port;
909 deliver_host_address = save_address;
910 deliver_host = save_host;
911 deliver_localpart = save_local;
912 deliver_domain = save_domain;
913 router_name = transport_name = NULL;
914 }
915 #endif /*DISABLE_EVENT*/
916
917
918
919 /******************************************************************************/
920
921
922 /*************************************************
923 * Generate local prt for logging *
924 *************************************************/
925
926 /* This function is a subroutine for use in string_log_address() below.
927
928 Arguments:
929 addr the address being logged
930 yield the current dynamic buffer pointer
931
932 Returns: the new value of the buffer pointer
933 */
934
935 static gstring *
936 string_get_localpart(address_item * addr, gstring * yield)
937 {
938 uschar * s;
939
940 s = addr->prefix;
941 if (testflag(addr, af_include_affixes) && s)
942 {
943 #ifdef SUPPORT_I18N
944 if (testflag(addr, af_utf8_downcvt))
945 s = string_localpart_utf8_to_alabel(s, NULL);
946 #endif
947 yield = string_cat(yield, s);
948 }
949
950 s = addr->local_part;
951 #ifdef SUPPORT_I18N
952 if (testflag(addr, af_utf8_downcvt))
953 s = string_localpart_utf8_to_alabel(s, NULL);
954 #endif
955 yield = string_cat(yield, s);
956
957 s = addr->suffix;
958 if (testflag(addr, af_include_affixes) && s)
959 {
960 #ifdef SUPPORT_I18N
961 if (testflag(addr, af_utf8_downcvt))
962 s = string_localpart_utf8_to_alabel(s, NULL);
963 #endif
964 yield = string_cat(yield, s);
965 }
966
967 return yield;
968 }
969
970
971 /*************************************************
972 * Generate log address list *
973 *************************************************/
974
975 /* This function generates a list consisting of an address and its parents, for
976 use in logging lines. For saved onetime aliased addresses, the onetime parent
977 field is used. If the address was delivered by a transport with rcpt_include_
978 affixes set, the af_include_affixes bit will be set in the address. In that
979 case, we include the affixes here too.
980
981 Arguments:
982 g points to growing-string struct
983 addr bottom (ultimate) address
984 all_parents if TRUE, include all parents
985 success TRUE for successful delivery
986
987 Returns: a growable string in dynamic store
988 */
989
990 static gstring *
991 string_log_address(gstring * g,
992 address_item *addr, BOOL all_parents, BOOL success)
993 {
994 BOOL add_topaddr = TRUE;
995 address_item *topaddr;
996
997 /* Find the ultimate parent */
998
999 for (topaddr = addr; topaddr->parent; topaddr = topaddr->parent) ;
1000
1001 /* We start with just the local part for pipe, file, and reply deliveries, and
1002 for successful local deliveries from routers that have the log_as_local flag
1003 set. File deliveries from filters can be specified as non-absolute paths in
1004 cases where the transport is going to complete the path. If there is an error
1005 before this happens (expansion failure) the local part will not be updated, and
1006 so won't necessarily look like a path. Add extra text for this case. */
1007
1008 if ( testflag(addr, af_pfr)
1009 || ( success
1010 && addr->router && addr->router->log_as_local
1011 && addr->transport && addr->transport->info->local
1012 ) )
1013 {
1014 if (testflag(addr, af_file) && addr->local_part[0] != '/')
1015 g = string_catn(g, CUS"save ", 5);
1016 g = string_get_localpart(addr, g);
1017 }
1018
1019 /* Other deliveries start with the full address. It we have split it into local
1020 part and domain, use those fields. Some early failures can happen before the
1021 splitting is done; in those cases use the original field. */
1022
1023 else
1024 {
1025 uschar * cmp = g->s + g->ptr;
1026
1027 if (addr->local_part)
1028 {
1029 const uschar * s;
1030 g = string_get_localpart(addr, g);
1031 g = string_catn(g, US"@", 1);
1032 s = addr->domain;
1033 #ifdef SUPPORT_I18N
1034 if (testflag(addr, af_utf8_downcvt))
1035 s = string_localpart_utf8_to_alabel(s, NULL);
1036 #endif
1037 g = string_cat(g, s);
1038 }
1039 else
1040 g = string_cat(g, addr->address);
1041
1042 /* If the address we are going to print is the same as the top address,
1043 and all parents are not being included, don't add on the top address. First
1044 of all, do a caseless comparison; if this succeeds, do a caseful comparison
1045 on the local parts. */
1046
1047 string_from_gstring(g); /* ensure nul-terminated */
1048 if ( strcmpic(cmp, topaddr->address) == 0
1049 && Ustrncmp(cmp, topaddr->address, Ustrchr(cmp, '@') - cmp) == 0
1050 && !addr->onetime_parent
1051 && (!all_parents || !addr->parent || addr->parent == topaddr)
1052 )
1053 add_topaddr = FALSE;
1054 }
1055
1056 /* If all parents are requested, or this is a local pipe/file/reply, and
1057 there is at least one intermediate parent, show it in brackets, and continue
1058 with all of them if all are wanted. */
1059
1060 if ( (all_parents || testflag(addr, af_pfr))
1061 && addr->parent
1062 && addr->parent != topaddr)
1063 {
1064 uschar *s = US" (";
1065 address_item *addr2;
1066 for (addr2 = addr->parent; addr2 != topaddr; addr2 = addr2->parent)
1067 {
1068 g = string_catn(g, s, 2);
1069 g = string_cat (g, addr2->address);
1070 if (!all_parents) break;
1071 s = US", ";
1072 }
1073 g = string_catn(g, US")", 1);
1074 }
1075
1076 /* Add the top address if it is required */
1077
1078 if (add_topaddr)
1079 g = string_append(g, 3,
1080 US" <",
1081 addr->onetime_parent ? addr->onetime_parent : topaddr->address,
1082 US">");
1083
1084 return g;
1085 }
1086
1087
1088
1089 void
1090 timesince(struct timeval * diff, struct timeval * then)
1091 {
1092 gettimeofday(diff, NULL);
1093 diff->tv_sec -= then->tv_sec;
1094 if ((diff->tv_usec -= then->tv_usec) < 0)
1095 {
1096 diff->tv_sec--;
1097 diff->tv_usec += 1000*1000;
1098 }
1099 }
1100
1101
1102
1103 uschar *
1104 string_timediff(struct timeval * diff)
1105 {
1106 static uschar buf[sizeof("0.000s")];
1107
1108 if (diff->tv_sec >= 5 || !LOGGING(millisec))
1109 return readconf_printtime((int)diff->tv_sec);
1110
1111 sprintf(CS buf, "%u.%03us", (uint)diff->tv_sec, (uint)diff->tv_usec/1000);
1112 return buf;
1113 }
1114
1115
1116 uschar *
1117 string_timesince(struct timeval * then)
1118 {
1119 struct timeval diff;
1120
1121 timesince(&diff, then);
1122 return string_timediff(&diff);
1123 }
1124
1125 /******************************************************************************/
1126
1127
1128
1129 /* If msg is NULL this is a delivery log and logchar is used. Otherwise
1130 this is a nonstandard call; no two-character delivery flag is written
1131 but sender-host and sender are prefixed and "msg" is inserted in the log line.
1132
1133 Arguments:
1134 flags passed to log_write()
1135 */
1136 void
1137 delivery_log(int flags, address_item * addr, int logchar, uschar * msg)
1138 {
1139 gstring * g; /* Used for a temporary, expanding buffer, for building log lines */
1140 void * reset_point; /* released afterwards. */
1141
1142 /* Log the delivery on the main log. We use an extensible string to build up
1143 the log line, and reset the store afterwards. Remote deliveries should always
1144 have a pointer to the host item that succeeded; local deliveries can have a
1145 pointer to a single host item in their host list, for use by the transport. */
1146
1147 #ifndef DISABLE_EVENT
1148 /* presume no successful remote delivery */
1149 lookup_dnssec_authenticated = NULL;
1150 #endif
1151
1152 g = reset_point = string_get(256);
1153
1154 if (msg)
1155 g = string_append(g, 2, host_and_ident(TRUE), US" ");
1156 else
1157 {
1158 g->s[0] = logchar; g->ptr = 1;
1159 g = string_catn(g, US"> ", 2);
1160 }
1161 g = string_log_address(g, addr, LOGGING(all_parents), TRUE);
1162
1163 if (LOGGING(sender_on_delivery) || msg)
1164 g = string_append(g, 3, US" F=<",
1165 #ifdef SUPPORT_I18N
1166 testflag(addr, af_utf8_downcvt)
1167 ? string_address_utf8_to_alabel(sender_address, NULL)
1168 :
1169 #endif
1170 sender_address,
1171 US">");
1172
1173 if (*queue_name)
1174 g = string_append(g, 2, US" Q=", queue_name);
1175
1176 #ifdef EXPERIMENTAL_SRS
1177 if(addr->prop.srs_sender)
1178 g = string_append(g, 3, US" SRS=<", addr->prop.srs_sender, US">");
1179 #endif
1180
1181 /* You might think that the return path must always be set for a successful
1182 delivery; indeed, I did for some time, until this statement crashed. The case
1183 when it is not set is for a delivery to /dev/null which is optimised by not
1184 being run at all. */
1185
1186 if (used_return_path && LOGGING(return_path_on_delivery))
1187 g = string_append(g, 3, US" P=<", used_return_path, US">");
1188
1189 if (msg)
1190 g = string_append(g, 2, US" ", msg);
1191
1192 /* For a delivery from a system filter, there may not be a router */
1193 if (addr->router)
1194 g = string_append(g, 2, US" R=", addr->router->name);
1195
1196 g = string_append(g, 2, US" T=", addr->transport->name);
1197
1198 if (LOGGING(delivery_size))
1199 g = string_append(g, 2, US" S=",
1200 string_sprintf("%d", transport_count));
1201
1202 /* Local delivery */
1203
1204 if (addr->transport->info->local)
1205 {
1206 if (addr->host_list)
1207 g = string_append(g, 2, US" H=", addr->host_list->name);
1208 g = d_log_interface(g);
1209 if (addr->shadow_message)
1210 g = string_cat(g, addr->shadow_message);
1211 }
1212
1213 /* Remote delivery */
1214
1215 else
1216 {
1217 if (addr->host_used)
1218 {
1219 g = d_hostlog(g, addr);
1220 if (continue_sequence > 1)
1221 g = string_catn(g, US"*", 1);
1222
1223 #ifndef DISABLE_EVENT
1224 deliver_host_address = addr->host_used->address;
1225 deliver_host_port = addr->host_used->port;
1226 deliver_host = addr->host_used->name;
1227
1228 /* DNS lookup status */
1229 lookup_dnssec_authenticated = addr->host_used->dnssec==DS_YES ? US"yes"
1230 : addr->host_used->dnssec==DS_NO ? US"no"
1231 : NULL;
1232 #endif
1233 }
1234
1235 #ifdef SUPPORT_TLS
1236 g = d_tlslog(g, addr);
1237 #endif
1238
1239 if (addr->authenticator)
1240 {
1241 g = string_append(g, 2, US" A=", addr->authenticator);
1242 if (addr->auth_id)
1243 {
1244 g = string_append(g, 2, US":", addr->auth_id);
1245 if (LOGGING(smtp_mailauth) && addr->auth_sndr)
1246 g = string_append(g, 2, US":", addr->auth_sndr);
1247 }
1248 }
1249
1250 if (LOGGING(pipelining))
1251 {
1252 if (testflag(addr, af_pipelining))
1253 g = string_catn(g, US" L", 2);
1254 #ifdef EXPERIMENTAL_PIPE_CONNECT
1255 if (testflag(addr, af_early_pipe))
1256 g = string_catn(g, US"*", 1);
1257 #endif
1258 }
1259
1260 #ifndef DISABLE_PRDR
1261 if (testflag(addr, af_prdr_used))
1262 g = string_catn(g, US" PRDR", 5);
1263 #endif
1264
1265 if (testflag(addr, af_chunking_used))
1266 g = string_catn(g, US" K", 2);
1267 }
1268
1269 /* confirmation message (SMTP (host_used) and LMTP (driver_name)) */
1270
1271 if ( LOGGING(smtp_confirmation)
1272 && addr->message
1273 && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0)
1274 )
1275 {
1276 unsigned i;
1277 unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024;
1278 uschar *p = big_buffer;
1279 uschar *ss = addr->message;
1280 *p++ = '\"';
1281 for (i = 0; i < lim && ss[i] != 0; i++) /* limit logged amount */
1282 {
1283 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\'; /* quote \ and " */
1284 *p++ = ss[i];
1285 }
1286 *p++ = '\"';
1287 *p = 0;
1288 g = string_append(g, 2, US" C=", big_buffer);
1289 }
1290
1291 /* Time on queue and actual time taken to deliver */
1292
1293 if (LOGGING(queue_time))
1294 g = string_append(g, 2, US" QT=",
1295 string_timesince(&received_time));
1296
1297 if (LOGGING(deliver_time))
1298 {
1299 struct timeval diff = {.tv_sec = addr->more_errno, .tv_usec = addr->delivery_usec};
1300 g = string_append(g, 2, US" DT=", string_timediff(&diff));
1301 }
1302
1303 /* string_cat() always leaves room for the terminator. Release the
1304 store we used to build the line after writing it. */
1305
1306 log_write(0, flags, "%s", string_from_gstring(g));
1307
1308 #ifndef DISABLE_EVENT
1309 if (!msg) msg_event_raise(US"msg:delivery", addr);
1310 #endif
1311
1312 store_reset(reset_point);
1313 return;
1314 }
1315
1316
1317
1318 static void
1319 deferral_log(address_item * addr, uschar * now,
1320 int logflags, uschar * driver_name, uschar * driver_kind)
1321 {
1322 gstring * g;
1323 void * reset_point;
1324
1325 /* Build up the line that is used for both the message log and the main
1326 log. */
1327
1328 g = reset_point = string_get(256);
1329
1330 /* Create the address string for logging. Must not do this earlier, because
1331 an OK result may be changed to FAIL when a pipe returns text. */
1332
1333 g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
1334
1335 if (*queue_name)
1336 g = string_append(g, 2, US" Q=", queue_name);
1337
1338 /* Either driver_name contains something and driver_kind contains
1339 " router" or " transport" (note the leading space), or driver_name is
1340 a null string and driver_kind contains "routing" without the leading
1341 space, if all routing has been deferred. When a domain has been held,
1342 so nothing has been done at all, both variables contain null strings. */
1343
1344 if (driver_name)
1345 {
1346 if (driver_kind[1] == 't' && addr->router)
1347 g = string_append(g, 2, US" R=", addr->router->name);
1348 g = string_cat(g, string_sprintf(" %c=%s", toupper(driver_kind[1]), driver_name));
1349 }
1350 else if (driver_kind)
1351 g = string_append(g, 2, US" ", driver_kind);
1352
1353 /*XXX need an s+s+p sprintf */
1354 g = string_cat(g, string_sprintf(" defer (%d)", addr->basic_errno));
1355
1356 if (addr->basic_errno > 0)
1357 g = string_append(g, 2, US": ",
1358 US strerror(addr->basic_errno));
1359
1360 if (addr->host_used)
1361 {
1362 g = string_append(g, 5,
1363 US" H=", addr->host_used->name,
1364 US" [", addr->host_used->address, US"]");
1365 if (LOGGING(outgoing_port))
1366 {
1367 int port = addr->host_used->port;
1368 g = string_append(g, 2,
1369 US":", port == PORT_NONE ? US"25" : string_sprintf("%d", port));
1370 }
1371 }
1372
1373 if (addr->message)
1374 g = string_append(g, 2, US": ", addr->message);
1375
1376 (void) string_from_gstring(g);
1377
1378 /* Log the deferment in the message log, but don't clutter it
1379 up with retry-time defers after the first delivery attempt. */
1380
1381 if (f.deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1382 deliver_msglog("%s %s\n", now, g->s);
1383
1384 /* Write the main log and reset the store.
1385 For errors of the type "retry time not reached" (also remotes skipped
1386 on queue run), logging is controlled by L_retry_defer. Note that this kind
1387 of error number is negative, and all the retry ones are less than any
1388 others. */
1389
1390
1391 log_write(addr->basic_errno <= ERRNO_RETRY_BASE ? L_retry_defer : 0, logflags,
1392 "== %s", g->s);
1393
1394 store_reset(reset_point);
1395 return;
1396 }
1397
1398
1399
1400 static void
1401 failure_log(address_item * addr, uschar * driver_kind, uschar * now)
1402 {
1403 void * reset_point;
1404 gstring * g = reset_point = string_get(256);
1405
1406 #ifndef DISABLE_EVENT
1407 /* Message failures for which we will send a DSN get their event raised
1408 later so avoid doing it here. */
1409
1410 if ( !addr->prop.ignore_error
1411 && !(addr->dsn_flags & (rf_dsnflags & ~rf_notify_failure))
1412 )
1413 msg_event_raise(US"msg:fail:delivery", addr);
1414 #endif
1415
1416 /* Build up the log line for the message and main logs */
1417
1418 /* Create the address string for logging. Must not do this earlier, because
1419 an OK result may be changed to FAIL when a pipe returns text. */
1420
1421 g = string_log_address(g, addr, LOGGING(all_parents), FALSE);
1422
1423 if (LOGGING(sender_on_delivery))
1424 g = string_append(g, 3, US" F=<", sender_address, US">");
1425
1426 if (*queue_name)
1427 g = string_append(g, 2, US" Q=", queue_name);
1428
1429 /* Return path may not be set if no delivery actually happened */
1430
1431 if (used_return_path && LOGGING(return_path_on_delivery))
1432 g = string_append(g, 3, US" P=<", used_return_path, US">");
1433
1434 if (addr->router)
1435 g = string_append(g, 2, US" R=", addr->router->name);
1436 if (addr->transport)
1437 g = string_append(g, 2, US" T=", addr->transport->name);
1438
1439 if (addr->host_used)
1440 g = d_hostlog(g, addr);
1441
1442 #ifdef SUPPORT_TLS
1443 g = d_tlslog(g, addr);
1444 #endif
1445
1446 if (addr->basic_errno > 0)
1447 g = string_append(g, 2, US": ", US strerror(addr->basic_errno));
1448
1449 if (addr->message)
1450 g = string_append(g, 2, US": ", addr->message);
1451
1452 (void) string_from_gstring(g);
1453
1454 /* Do the logging. For the message log, "routing failed" for those cases,
1455 just to make it clearer. */
1456
1457 if (driver_kind)
1458 deliver_msglog("%s %s failed for %s\n", now, driver_kind, g->s);
1459 else
1460 deliver_msglog("%s %s\n", now, g->s);
1461
1462 log_write(0, LOG_MAIN, "** %s", g->s);
1463
1464 store_reset(reset_point);
1465 return;
1466 }
1467
1468
1469
1470 /*************************************************
1471 * Actions at the end of handling an address *
1472 *************************************************/
1473
1474 /* This is a function for processing a single address when all that can be done
1475 with it has been done.
1476
1477 Arguments:
1478 addr points to the address block
1479 result the result of the delivery attempt
1480 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
1481 driver_type indicates which type of driver (transport, or router) was last
1482 to process the address
1483 logchar '=' or '-' for use when logging deliveries with => or ->
1484
1485 Returns: nothing
1486 */
1487
1488 static void
1489 post_process_one(address_item *addr, int result, int logflags, int driver_type,
1490 int logchar)
1491 {
1492 uschar *now = tod_stamp(tod_log);
1493 uschar *driver_kind = NULL;
1494 uschar *driver_name = NULL;
1495
1496 DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
1497
1498 /* Set up driver kind and name for logging. Disable logging if the router or
1499 transport has disabled it. */
1500
1501 if (driver_type == EXIM_DTYPE_TRANSPORT)
1502 {
1503 if (addr->transport)
1504 {
1505 driver_name = addr->transport->name;
1506 driver_kind = US" transport";
1507 f.disable_logging = addr->transport->disable_logging;
1508 }
1509 else driver_kind = US"transporting";
1510 }
1511 else if (driver_type == EXIM_DTYPE_ROUTER)
1512 {
1513 if (addr->router)
1514 {
1515 driver_name = addr->router->name;
1516 driver_kind = US" router";
1517 f.disable_logging = addr->router->disable_logging;
1518 }
1519 else driver_kind = US"routing";
1520 }
1521
1522 /* If there's an error message set, ensure that it contains only printing
1523 characters - it should, but occasionally things slip in and this at least
1524 stops the log format from getting wrecked. We also scan the message for an LDAP
1525 expansion item that has a password setting, and flatten the password. This is a
1526 fudge, but I don't know a cleaner way of doing this. (If the item is badly
1527 malformed, it won't ever have gone near LDAP.) */
1528
1529 if (addr->message)
1530 {
1531 const uschar * s = string_printing(addr->message);
1532
1533 /* deconst cast ok as string_printing known to have alloc'n'copied */
1534 addr->message = expand_hide_passwords(US s);
1535 }
1536
1537 /* If we used a transport that has one of the "return_output" options set, and
1538 if it did in fact generate some output, then for return_output we treat the
1539 message as failed if it was not already set that way, so that the output gets
1540 returned to the sender, provided there is a sender to send it to. For
1541 return_fail_output, do this only if the delivery failed. Otherwise we just
1542 unlink the file, and remove the name so that if the delivery failed, we don't
1543 try to send back an empty or unwanted file. The log_output options operate only
1544 on a non-empty file.
1545
1546 In any case, we close the message file, because we cannot afford to leave a
1547 file-descriptor for one address while processing (maybe very many) others. */
1548
1549 if (addr->return_file >= 0 && addr->return_filename)
1550 {
1551 BOOL return_output = FALSE;
1552 struct stat statbuf;
1553 (void)EXIMfsync(addr->return_file);
1554
1555 /* If there is no output, do nothing. */
1556
1557 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
1558 {
1559 transport_instance *tb = addr->transport;
1560
1561 /* Handle logging options */
1562
1563 if ( tb->log_output
1564 || result == FAIL && tb->log_fail_output
1565 || result == DEFER && tb->log_defer_output
1566 )
1567 {
1568 uschar *s;
1569 FILE *f = Ufopen(addr->return_filename, "rb");
1570 if (!f)
1571 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
1572 "from %s transport: %s", addr->return_filename, tb->name,
1573 strerror(errno));
1574 else
1575 if ((s = US Ufgets(big_buffer, big_buffer_size, f)))
1576 {
1577 uschar *p = big_buffer + Ustrlen(big_buffer);
1578 const uschar * sp;
1579 while (p > big_buffer && isspace(p[-1])) p--;
1580 *p = 0;
1581 sp = string_printing(big_buffer);
1582 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
1583 addr->address, tb->name, sp);
1584 }
1585 (void)fclose(f);
1586 }
1587
1588 /* Handle returning options, but only if there is an address to return
1589 the text to. */
1590
1591 if (sender_address[0] != 0 || addr->prop.errors_address)
1592 if (tb->return_output)
1593 {
1594 addr->transport_return = result = FAIL;
1595 if (addr->basic_errno == 0 && !addr->message)
1596 addr->message = US"return message generated";
1597 return_output = TRUE;
1598 }
1599 else
1600 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
1601 }
1602
1603 /* Get rid of the file unless it might be returned, but close it in
1604 all cases. */
1605
1606 if (!return_output)
1607 {
1608 Uunlink(addr->return_filename);
1609 addr->return_filename = NULL;
1610 addr->return_file = -1;
1611 }
1612
1613 (void)close(addr->return_file);
1614 }
1615
1616 /* The success case happens only after delivery by a transport. */
1617
1618 if (result == OK)
1619 {
1620 addr->next = addr_succeed;
1621 addr_succeed = addr;
1622
1623 /* Call address_done() to ensure that we don't deliver to this address again,
1624 and write appropriate things to the message log. If it is a child address, we
1625 call child_done() to scan the ancestors and mark them complete if this is the
1626 last child to complete. */
1627
1628 address_done(addr, now);
1629 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
1630
1631 if (!addr->parent)
1632 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
1633 driver_name, driver_kind);
1634 else
1635 {
1636 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
1637 addr->parent->address, driver_name, driver_kind);
1638 child_done(addr, now);
1639 }
1640
1641 /* Certificates for logging (via events) */
1642 #ifdef SUPPORT_TLS
1643 tls_out.ourcert = addr->ourcert;
1644 addr->ourcert = NULL;
1645 tls_out.peercert = addr->peercert;
1646 addr->peercert = NULL;
1647
1648 tls_out.cipher = addr->cipher;
1649 tls_out.peerdn = addr->peerdn;
1650 tls_out.ocsp = addr->ocsp;
1651 # ifdef SUPPORT_DANE
1652 tls_out.dane_verified = testflag(addr, af_dane_verified);
1653 # endif
1654 #endif
1655
1656 delivery_log(LOG_MAIN, addr, logchar, NULL);
1657
1658 #ifdef SUPPORT_TLS
1659 tls_free_cert(&tls_out.ourcert);
1660 tls_free_cert(&tls_out.peercert);
1661 tls_out.cipher = NULL;
1662 tls_out.peerdn = NULL;
1663 tls_out.ocsp = OCSP_NOT_REQ;
1664 # ifdef SUPPORT_DANE
1665 tls_out.dane_verified = FALSE;
1666 # endif
1667 #endif
1668 }
1669
1670
1671 /* Soft failure, or local delivery process failed; freezing may be
1672 requested. */
1673
1674 else if (result == DEFER || result == PANIC)
1675 {
1676 if (result == PANIC) logflags |= LOG_PANIC;
1677
1678 /* This puts them on the chain in reverse order. Do not change this, because
1679 the code for handling retries assumes that the one with the retry
1680 information is last. */
1681
1682 addr->next = addr_defer;
1683 addr_defer = addr;
1684
1685 /* The only currently implemented special action is to freeze the
1686 message. Logging of this is done later, just before the -H file is
1687 updated. */
1688
1689 if (addr->special_action == SPECIAL_FREEZE)
1690 {
1691 f.deliver_freeze = TRUE;
1692 deliver_frozen_at = time(NULL);
1693 update_spool = TRUE;
1694 }
1695
1696 /* If doing a 2-stage queue run, we skip writing to either the message
1697 log or the main log for SMTP defers. */
1698
1699 if (!f.queue_2stage || addr->basic_errno != 0)
1700 deferral_log(addr, now, logflags, driver_name, driver_kind);
1701 }
1702
1703
1704 /* Hard failure. If there is an address to which an error message can be sent,
1705 put this address on the failed list. If not, put it on the deferred list and
1706 freeze the mail message for human attention. The latter action can also be
1707 explicitly requested by a router or transport. */
1708
1709 else
1710 {
1711 /* If this is a delivery error, or a message for which no replies are
1712 wanted, and the message's age is greater than ignore_bounce_errors_after,
1713 force the af_ignore_error flag. This will cause the address to be discarded
1714 later (with a log entry). */
1715
1716 if (!*sender_address && message_age >= ignore_bounce_errors_after)
1717 addr->prop.ignore_error = TRUE;
1718
1719 /* Freeze the message if requested, or if this is a bounce message (or other
1720 message with null sender) and this address does not have its own errors
1721 address. However, don't freeze if errors are being ignored. The actual code
1722 to ignore occurs later, instead of sending a message. Logging of freezing
1723 occurs later, just before writing the -H file. */
1724
1725 if ( !addr->prop.ignore_error
1726 && ( addr->special_action == SPECIAL_FREEZE
1727 || (sender_address[0] == 0 && !addr->prop.errors_address)
1728 ) )
1729 {
1730 frozen_info = addr->special_action == SPECIAL_FREEZE
1731 ? US""
1732 : f.sender_local && !f.local_error_message
1733 ? US" (message created with -f <>)"
1734 : US" (delivery error message)";
1735 f.deliver_freeze = TRUE;
1736 deliver_frozen_at = time(NULL);
1737 update_spool = TRUE;
1738
1739 /* The address is put on the defer rather than the failed queue, because
1740 the message is being retained. */
1741
1742 addr->next = addr_defer;
1743 addr_defer = addr;
1744 }
1745
1746 /* Don't put the address on the nonrecipients tree yet; wait until an
1747 error message has been successfully sent. */
1748
1749 else
1750 {
1751 addr->next = addr_failed;
1752 addr_failed = addr;
1753 }
1754
1755 failure_log(addr, driver_name ? NULL : driver_kind, now);
1756 }
1757
1758 /* Ensure logging is turned on again in all cases */
1759
1760 f.disable_logging = FALSE;
1761 }
1762
1763
1764
1765
1766 /*************************************************
1767 * Address-independent error *
1768 *************************************************/
1769
1770 /* This function is called when there's an error that is not dependent on a
1771 particular address, such as an expansion string failure. It puts the error into
1772 all the addresses in a batch, logs the incident on the main and panic logs, and
1773 clears the expansions. It is mostly called from local_deliver(), but can be
1774 called for a remote delivery via findugid().
1775
1776 Arguments:
1777 logit TRUE if (MAIN+PANIC) logging required
1778 addr the first of the chain of addresses
1779 code the error code
1780 format format string for error message, or NULL if already set in addr
1781 ... arguments for the format
1782
1783 Returns: nothing
1784 */
1785
1786 static void
1787 common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1788 {
1789 address_item *addr2;
1790 addr->basic_errno = code;
1791
1792 if (format)
1793 {
1794 va_list ap;
1795 uschar buffer[512];
1796 va_start(ap, format);
1797 if (!string_vformat(buffer, sizeof(buffer), CS format, ap))
1798 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
1799 "common_error expansion was longer than " SIZE_T_FMT, sizeof(buffer));
1800 va_end(ap);
1801 addr->message = string_copy(buffer);
1802 }
1803
1804 for (addr2 = addr->next; addr2; addr2 = addr2->next)
1805 {
1806 addr2->basic_errno = code;
1807 addr2->message = addr->message;
1808 }
1809
1810 if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1811 deliver_set_expansions(NULL);
1812 }
1813
1814
1815
1816
1817 /*************************************************
1818 * Check a "never users" list *
1819 *************************************************/
1820
1821 /* This function is called to check whether a uid is on one of the two "never
1822 users" lists.
1823
1824 Arguments:
1825 uid the uid to be checked
1826 nusers the list to be scanned; the first item in the list is the count
1827
1828 Returns: TRUE if the uid is on the list
1829 */
1830
1831 static BOOL
1832 check_never_users(uid_t uid, uid_t *nusers)
1833 {
1834 int i;
1835 if (!nusers) return FALSE;
1836 for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1837 return FALSE;
1838 }
1839
1840
1841
1842 /*************************************************
1843 * Find uid and gid for a transport *
1844 *************************************************/
1845
1846 /* This function is called for both local and remote deliveries, to find the
1847 uid/gid under which to run the delivery. The values are taken preferentially
1848 from the transport (either explicit or deliver_as_creator), then from the
1849 address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1850 the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1851 panic error is logged, and the function fails (which normally leads to delivery
1852 deferral).
1853
1854 Arguments:
1855 addr the address (possibly a chain)
1856 tp the transport
1857 uidp pointer to uid field
1858 gidp pointer to gid field
1859 igfp pointer to the use_initgroups field
1860
1861 Returns: FALSE if failed - error has been set in address(es)
1862 */
1863
1864 static BOOL
1865 findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1866 BOOL *igfp)
1867 {
1868 uschar *nuname;
1869 BOOL gid_set = FALSE;
1870
1871 /* Default initgroups flag comes from the transport */
1872
1873 *igfp = tp->initgroups;
1874
1875 /* First see if there's a gid on the transport, either fixed or expandable.
1876 The expanding function always logs failure itself. */
1877
1878 if (tp->gid_set)
1879 {
1880 *gidp = tp->gid;
1881 gid_set = TRUE;
1882 }
1883 else if (tp->expand_gid)
1884 {
1885 if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1886 &(addr->message)))
1887 {
1888 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1889 return FALSE;
1890 }
1891 gid_set = TRUE;
1892 }
1893
1894 /* If the transport did not set a group, see if the router did. */
1895
1896 if (!gid_set && testflag(addr, af_gid_set))
1897 {
1898 *gidp = addr->gid;
1899 gid_set = TRUE;
1900 }
1901
1902 /* Pick up a uid from the transport if one is set. */
1903
1904 if (tp->uid_set) *uidp = tp->uid;
1905
1906 /* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1907 it does not provide a passwd value from which a gid can be taken. */
1908
1909 else if (tp->expand_uid)
1910 {
1911 struct passwd *pw;
1912 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1913 uidp, &(addr->message)))
1914 {
1915 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1916 return FALSE;
1917 }
1918 if (!gid_set && pw)
1919 {
1920 *gidp = pw->pw_gid;
1921 gid_set = TRUE;
1922 }
1923 }
1924
1925 /* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1926
1927 else if (tp->deliver_as_creator)
1928 {
1929 *uidp = originator_uid;
1930 if (!gid_set)
1931 {
1932 *gidp = originator_gid;
1933 gid_set = TRUE;
1934 }
1935 }
1936
1937 /* Otherwise see if the address specifies the uid and if so, take it and its
1938 initgroups flag. */
1939
1940 else if (testflag(addr, af_uid_set))
1941 {
1942 *uidp = addr->uid;
1943 *igfp = testflag(addr, af_initgroups);
1944 }
1945
1946 /* Nothing has specified the uid - default to the Exim user, and group if the
1947 gid is not set. */
1948
1949 else
1950 {
1951 *uidp = exim_uid;
1952 if (!gid_set)
1953 {
1954 *gidp = exim_gid;
1955 gid_set = TRUE;
1956 }
1957 }
1958
1959 /* If no gid is set, it is a disaster. We default to the Exim gid only if
1960 defaulting to the Exim uid. In other words, if the configuration has specified
1961 a uid, it must also provide a gid. */
1962
1963 if (!gid_set)
1964 {
1965 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1966 "%s transport", tp->name);
1967 return FALSE;
1968 }
1969
1970 /* Check that the uid is not on the lists of banned uids that may not be used
1971 for delivery processes. */
1972
1973 nuname = check_never_users(*uidp, never_users)
1974 ? US"never_users"
1975 : check_never_users(*uidp, fixed_never_users)
1976 ? US"fixed_never_users"
1977 : NULL;
1978 if (nuname)
1979 {
1980 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1981 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1982 return FALSE;
1983 }
1984
1985 /* All is well */
1986
1987 return TRUE;
1988 }
1989
1990
1991
1992
1993 /*************************************************
1994 * Check the size of a message for a transport *
1995 *************************************************/
1996
1997 /* Checks that the message isn't too big for the selected transport.
1998 This is called only when it is known that the limit is set.
1999
2000 Arguments:
2001 tp the transport
2002 addr the (first) address being delivered
2003
2004 Returns: OK
2005 DEFER expansion failed or did not yield an integer
2006 FAIL message too big
2007 */
2008
2009 int
2010 check_message_size(transport_instance *tp, address_item *addr)
2011 {
2012 int rc = OK;
2013 int size_limit;
2014
2015 deliver_set_expansions(addr);
2016 size_limit = expand_string_integer(tp->message_size_limit, TRUE);
2017 deliver_set_expansions(NULL);
2018
2019 if (expand_string_message)
2020 {
2021 rc = DEFER;
2022 addr->message = size_limit == -1
2023 ? string_sprintf("failed to expand message_size_limit "
2024 "in %s transport: %s", tp->name, expand_string_message)
2025 : string_sprintf("invalid message_size_limit "
2026 "in %s transport: %s", tp->name, expand_string_message);
2027 }
2028 else if (size_limit > 0 && message_size > size_limit)
2029 {
2030 rc = FAIL;
2031 addr->message =
2032 string_sprintf("message is too big (transport limit = %d)",
2033 size_limit);
2034 }
2035
2036 return rc;
2037 }
2038
2039
2040
2041 /*************************************************
2042 * Transport-time check for a previous delivery *
2043 *************************************************/
2044
2045 /* Check that this base address hasn't previously been delivered to its routed
2046 transport. If it has been delivered, mark it done. The check is necessary at
2047 delivery time in order to handle homonymic addresses correctly in cases where
2048 the pattern of redirection changes between delivery attempts (so the unique
2049 fields change). Non-homonymic previous delivery is detected earlier, at routing
2050 time (which saves unnecessary routing).
2051
2052 Arguments:
2053 addr the address item
2054 testing TRUE if testing wanted only, without side effects
2055
2056 Returns: TRUE if previously delivered by the transport
2057 */
2058
2059 static BOOL
2060 previously_transported(address_item *addr, BOOL testing)
2061 {
2062 (void)string_format(big_buffer, big_buffer_size, "%s/%s",
2063 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
2064
2065 if (tree_search(tree_nonrecipients, big_buffer) != 0)
2066 {
2067 DEBUG(D_deliver|D_route|D_transport)
2068 debug_printf("%s was previously delivered (%s transport): discarded\n",
2069 addr->address, addr->transport->name);
2070 if (!testing) child_done(addr, tod_stamp(tod_log));
2071 return TRUE;
2072 }
2073
2074 return FALSE;
2075 }
2076
2077
2078
2079 /******************************************************
2080 * Check for a given header in a header string *
2081 ******************************************************/
2082
2083 /* This function is used when generating quota warnings. The configuration may
2084 specify any header lines it likes in quota_warn_message. If certain of them are
2085 missing, defaults are inserted, so we need to be able to test for the presence
2086 of a given header.
2087
2088 Arguments:
2089 hdr the required header name
2090 hstring the header string
2091
2092 Returns: TRUE the header is in the string
2093 FALSE the header is not in the string
2094 */
2095
2096 static BOOL
2097 contains_header(uschar *hdr, uschar *hstring)
2098 {
2099 int len = Ustrlen(hdr);
2100 uschar *p = hstring;
2101 while (*p != 0)
2102 {
2103 if (strncmpic(p, hdr, len) == 0)
2104 {
2105 p += len;
2106 while (*p == ' ' || *p == '\t') p++;
2107 if (*p == ':') return TRUE;
2108 }
2109 while (*p != 0 && *p != '\n') p++;
2110 if (*p == '\n') p++;
2111 }
2112 return FALSE;
2113 }
2114
2115
2116
2117
2118 /*************************************************
2119 * Perform a local delivery *
2120 *************************************************/
2121
2122 /* Each local delivery is performed in a separate process which sets its
2123 uid and gid as specified. This is a safer way than simply changing and
2124 restoring using seteuid(); there is a body of opinion that seteuid() cannot be
2125 used safely. From release 4, Exim no longer makes any use of it. Besides, not
2126 all systems have seteuid().
2127
2128 If the uid/gid are specified in the transport_instance, they are used; the
2129 transport initialization must ensure that either both or neither are set.
2130 Otherwise, the values associated with the address are used. If neither are set,
2131 it is a configuration error.
2132
2133 The transport or the address may specify a home directory (transport over-
2134 rides), and if they do, this is set as $home. If neither have set a working
2135 directory, this value is used for that as well. Otherwise $home is left unset
2136 and the cwd is set to "/" - a directory that should be accessible to all users.
2137
2138 Using a separate process makes it more complicated to get error information
2139 back. We use a pipe to pass the return code and also an error code and error
2140 text string back to the parent process.
2141
2142 Arguments:
2143 addr points to an address block for this delivery; for "normal" local
2144 deliveries this is the only address to be delivered, but for
2145 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
2146 a number of addresses can be handled simultaneously, and in this
2147 case addr will point to a chain of addresses with the same
2148 characteristics.
2149
2150 shadowing TRUE if running a shadow transport; this causes output from pipes
2151 to be ignored.
2152
2153 Returns: nothing
2154 */
2155
2156 static void
2157 deliver_local(address_item *addr, BOOL shadowing)
2158 {
2159 BOOL use_initgroups;
2160 uid_t uid;
2161 gid_t gid;
2162 int status, len, rc;
2163 int pfd[2];
2164 pid_t pid;
2165 uschar *working_directory;
2166 address_item *addr2;
2167 transport_instance *tp = addr->transport;
2168
2169 /* Set up the return path from the errors or sender address. If the transport
2170 has its own return path setting, expand it and replace the existing value. */
2171
2172 if(addr->prop.errors_address)
2173 return_path = addr->prop.errors_address;
2174 #ifdef EXPERIMENTAL_SRS
2175 else if (addr->prop.srs_sender)
2176 return_path = addr->prop.srs_sender;
2177 #endif
2178 else
2179 return_path = sender_address;
2180
2181 if (tp->return_path)
2182 {
2183 uschar *new_return_path = expand_string(tp->return_path);
2184 if (!new_return_path)
2185 {
2186 if (!f.expand_string_forcedfail)
2187 {
2188 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
2189 US"Failed to expand return path \"%s\" in %s transport: %s",
2190 tp->return_path, tp->name, expand_string_message);
2191 return;
2192 }
2193 }
2194 else return_path = new_return_path;
2195 }
2196
2197 /* For local deliveries, one at a time, the value used for logging can just be
2198 set directly, once and for all. */
2199
2200 used_return_path = return_path;
2201
2202 /* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
2203 gets put into the address(es), and the expansions are unset, so we can just
2204 return. */
2205
2206 if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
2207
2208 /* See if either the transport or the address specifies a home directory. A
2209 home directory set in the address may already be expanded; a flag is set to
2210 indicate that. In other cases we must expand it. */
2211
2212 if ( (deliver_home = tp->home_dir) /* Set in transport, or */
2213 || ( (deliver_home = addr->home_dir) /* Set in address and */
2214 && !testflag(addr, af_home_expanded) /* not expanded */
2215 ) )
2216 {
2217 uschar *rawhome = deliver_home;
2218 deliver_home = NULL; /* in case it contains $home */
2219 if (!(deliver_home = expand_string(rawhome)))
2220 {
2221 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
2222 "to expand for %s transport: %s", rawhome, tp->name,
2223 expand_string_message);
2224 return;
2225 }
2226 if (*deliver_home != '/')
2227 {
2228 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
2229 "is not absolute for %s transport", deliver_home, tp->name);
2230 return;
2231 }
2232 }
2233
2234 /* See if either the transport or the address specifies a current directory,
2235 and if so, expand it. If nothing is set, use the home directory, unless it is
2236 also unset in which case use "/", which is assumed to be a directory to which
2237 all users have access. It is necessary to be in a visible directory for some
2238 operating systems when running pipes, as some commands (e.g. "rm" under Solaris
2239 2.5) require this. */
2240
2241 working_directory = tp->current_dir ? tp->current_dir : addr->current_dir;
2242 if (working_directory)
2243 {
2244 uschar *raw = working_directory;
2245 if (!(working_directory = expand_string(raw)))
2246 {
2247 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
2248 "failed to expand for %s transport: %s", raw, tp->name,
2249 expand_string_message);
2250 return;
2251 }
2252 if (*working_directory != '/')
2253 {
2254 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
2255 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
2256 return;
2257 }
2258 }
2259 else working_directory = deliver_home ? deliver_home : US"/";
2260
2261 /* If one of the return_output flags is set on the transport, create and open a
2262 file in the message log directory for the transport to write its output onto.
2263 This is mainly used by pipe transports. The file needs to be unique to the
2264 address. This feature is not available for shadow transports. */
2265
2266 if ( !shadowing
2267 && ( tp->return_output || tp->return_fail_output
2268 || tp->log_output || tp->log_fail_output || tp->log_defer_output
2269 ) )
2270 {
2271 uschar * error;
2272
2273 addr->return_filename =
2274 spool_fname(US"msglog", message_subdir, message_id,
2275 string_sprintf("-%d-%d", getpid(), return_count++));
2276
2277 if ((addr->return_file = open_msglog_file(addr->return_filename, 0400, &error)) < 0)
2278 {
2279 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
2280 "to return message: %s", error, tp->name, strerror(errno));
2281 return;
2282 }
2283 }
2284
2285 /* Create the pipe for inter-process communication. */
2286
2287 if (pipe(pfd) != 0)
2288 {
2289 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
2290 strerror(errno));
2291 return;
2292 }
2293
2294 /* Now fork the process to do the real work in the subprocess, but first
2295 ensure that all cached resources are freed so that the subprocess starts with
2296 a clean slate and doesn't interfere with the parent process. */
2297
2298 search_tidyup();
2299
2300 if ((pid = fork()) == 0)
2301 {
2302 BOOL replicate = TRUE;
2303
2304 /* Prevent core dumps, as we don't want them in users' home directories.
2305 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
2306 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
2307 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
2308 complain if the error is "not supported".
2309
2310 There are two scenarios where changing the max limit has an effect. In one,
2311 the user is using a .forward and invoking a command of their choice via pipe;
2312 for these, we do need the max limit to be 0 unless the admin chooses to
2313 permit an increased limit. In the other, the command is invoked directly by
2314 the transport and is under administrator control, thus being able to raise
2315 the limit aids in debugging. So there's no general always-right answer.
2316
2317 Thus we inhibit core-dumps completely but let individual transports, while
2318 still root, re-raise the limits back up to aid debugging. We make the
2319 default be no core-dumps -- few enough people can use core dumps in
2320 diagnosis that it's reasonable to make them something that has to be explicitly requested.
2321 */
2322
2323 #ifdef RLIMIT_CORE
2324 struct rlimit rl;
2325 rl.rlim_cur = 0;
2326 rl.rlim_max = 0;
2327 if (setrlimit(RLIMIT_CORE, &rl) < 0)
2328 {
2329 # ifdef SETRLIMIT_NOT_SUPPORTED
2330 if (errno != ENOSYS && errno != ENOTSUP)
2331 # endif
2332 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
2333 strerror(errno));
2334 }
2335 #endif
2336
2337 /* Reset the random number generator, so different processes don't all
2338 have the same sequence. */
2339
2340 random_seed = 0;
2341
2342 /* If the transport has a setup entry, call this first, while still
2343 privileged. (Appendfile uses this to expand quota, for example, while
2344 able to read private files.) */
2345
2346 if (addr->transport->setup)
2347 switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,
2348 &(addr->message)))
2349 {
2350 case DEFER:
2351 addr->transport_return = DEFER;
2352 goto PASS_BACK;
2353
2354 case FAIL:
2355 addr->transport_return = PANIC;
2356 goto PASS_BACK;
2357 }
2358
2359 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
2360 when the process becomes unprivileged, it won't be able to write to the
2361 process log. SIGHUP is ignored throughout exim, except when it is being
2362 run as a daemon. */
2363
2364 signal(SIGINT, SIG_IGN);
2365 signal(SIGTERM, SIG_IGN);
2366 signal(SIGUSR1, SIG_IGN);
2367
2368 /* Close the unwanted half of the pipe, and set close-on-exec for the other
2369 half - for transports that exec things (e.g. pipe). Then set the required
2370 gid/uid. */
2371
2372 (void)close(pfd[pipe_read]);
2373 (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
2374 FD_CLOEXEC);
2375 exim_setugid(uid, gid, use_initgroups,
2376 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
2377 addr->address, addr->transport->name));
2378
2379 DEBUG(D_deliver)
2380 {
2381 address_item *batched;
2382 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
2383 for (batched = addr->next; batched; batched = batched->next)
2384 debug_printf("additional batched address: %s\n", batched->address);
2385 }
2386
2387 /* Set an appropriate working directory. */
2388
2389 if (Uchdir(working_directory) < 0)
2390 {
2391 addr->transport_return = DEFER;
2392 addr->basic_errno = errno;
2393 addr->message = string_sprintf("failed to chdir to %s", working_directory);
2394 }
2395
2396 /* If successful, call the transport */
2397
2398 else
2399 {
2400 BOOL ok = TRUE;
2401 set_process_info("delivering %s to %s using %s", message_id,
2402 addr->local_part, addr->transport->name);
2403
2404 /* Setting this global in the subprocess means we need never clear it */
2405 transport_name = addr->transport->name;
2406
2407 /* If a transport filter has been specified, set up its argument list.
2408 Any errors will get put into the address, and FALSE yielded. */
2409
2410 if (addr->transport->filter_command)
2411 {
2412 ok = transport_set_up_command(&transport_filter_argv,
2413 addr->transport->filter_command,
2414 TRUE, PANIC, addr, US"transport filter", NULL);
2415 transport_filter_timeout = addr->transport->filter_timeout;
2416 }
2417 else transport_filter_argv = NULL;
2418
2419 if (ok)
2420 {
2421 debug_print_string(addr->transport->debug_string);
2422 replicate = !(addr->transport->info->code)(addr->transport, addr);
2423 }
2424 }
2425
2426 /* Pass the results back down the pipe. If necessary, first replicate the
2427 status in the top address to the others in the batch. The label is the
2428 subject of a goto when a call to the transport's setup function fails. We
2429 pass the pointer to the transport back in case it got changed as a result of
2430 file_format in appendfile. */
2431
2432 PASS_BACK:
2433
2434 if (replicate) replicate_status(addr);
2435 for (addr2 = addr; addr2; addr2 = addr2->next)
2436 {
2437 int i;
2438 int local_part_length = Ustrlen(addr2->local_part);
2439 uschar *s;
2440 int ret;
2441
2442 if( (ret = write(pfd[pipe_write], &addr2->transport_return, sizeof(int))) != sizeof(int)
2443 || (ret = write(pfd[pipe_write], &transport_count, sizeof(transport_count))) != sizeof(transport_count)
2444 || (ret = write(pfd[pipe_write], &addr2->flags, sizeof(addr2->flags))) != sizeof(addr2->flags)
2445 || (ret = write(pfd[pipe_write], &addr2->basic_errno, sizeof(int))) != sizeof(int)
2446 || (ret = write(pfd[pipe_write], &addr2->more_errno, sizeof(int))) != sizeof(int)
2447 || (ret = write(pfd[pipe_write], &addr2->delivery_usec, sizeof(int))) != sizeof(int)
2448 || (ret = write(pfd[pipe_write], &addr2->special_action, sizeof(int))) != sizeof(int)
2449 || (ret = write(pfd[pipe_write], &addr2->transport,
2450 sizeof(transport_instance *))) != sizeof(transport_instance *)
2451
2452 /* For a file delivery, pass back the local part, in case the original
2453 was only part of the final delivery path. This gives more complete
2454 logging. */
2455
2456 || (testflag(addr2, af_file)
2457 && ( (ret = write(pfd[pipe_write], &local_part_length, sizeof(int))) != sizeof(int)
2458 || (ret = write(pfd[pipe_write], addr2->local_part, local_part_length)) != local_part_length
2459 )
2460 )
2461 )
2462 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
2463 ret == -1 ? strerror(errno) : "short write");
2464
2465 /* Now any messages */
2466
2467 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
2468 {
2469 int message_length = s ? Ustrlen(s) + 1 : 0;
2470 if( (ret = write(pfd[pipe_write], &message_length, sizeof(int))) != sizeof(int)
2471 || message_length > 0 && (ret = write(pfd[pipe_write], s, message_length)) != message_length
2472 )
2473 log_write(0, LOG_MAIN|LOG_PANIC, "Failed writing transport results to pipe: %s",
2474 ret == -1 ? strerror(errno) : "short write");
2475 }
2476 }
2477
2478 /* OK, this process is now done. Free any cached resources that it opened,
2479 and close the pipe we were writing down before exiting. */
2480
2481 (void)close(pfd[pipe_write]);
2482 search_tidyup();
2483 exit(EXIT_SUCCESS);
2484 }
2485
2486 /* Back in the main process: panic if the fork did not succeed. This seems
2487 better than returning an error - if forking is failing it is probably best
2488 not to try other deliveries for this message. */
2489
2490 if (pid < 0)
2491 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
2492 addr->address);
2493
2494 /* Read the pipe to get the delivery status codes and error messages. Our copy
2495 of the writing end must be closed first, as otherwise read() won't return zero
2496 on an empty pipe. We check that a status exists for each address before
2497 overwriting the address structure. If data is missing, the default DEFER status
2498 will remain. Afterwards, close the reading end. */
2499
2500 (void)close(pfd[pipe_write]);
2501
2502 for (addr2 = addr; addr2; addr2 = addr2->next)
2503 {
2504 if ((len = read(pfd[pipe_read], &status, sizeof(int))) > 0)
2505 {
2506 int i;
2507 uschar **sptr;
2508
2509 addr2->transport_return = status;
2510 len = read(pfd[pipe_read], &transport_count,
2511 sizeof(transport_count));
2512 len = read(pfd[pipe_read], &addr2->flags, sizeof(addr2->flags));
2513 len = read(pfd[pipe_read], &addr2->basic_errno, sizeof(int));
2514 len = read(pfd[pipe_read], &addr2->more_errno, sizeof(int));
2515 len = read(pfd[pipe_read], &addr2->delivery_usec, sizeof(int));
2516 len = read(pfd[pipe_read], &addr2->special_action, sizeof(int));
2517 len = read(pfd[pipe_read], &addr2->transport,
2518 sizeof(transport_instance *));
2519
2520 if (testflag(addr2, af_file))
2521 {
2522 int llen;
2523 if ( read(pfd[pipe_read], &llen, sizeof(int)) != sizeof(int)
2524 || llen > 64*4 /* limit from rfc 5821, times I18N factor */
2525 )
2526 {
2527 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part length read"
2528 " from delivery subprocess");
2529 break;
2530 }
2531 /* sanity-checked llen so disable the Coverity error */
2532 /* coverity[tainted_data] */
2533 if (read(pfd[pipe_read], big_buffer, llen) != llen)
2534 {
2535 log_write(0, LOG_MAIN|LOG_PANIC, "bad local_part read"
2536 " from delivery subprocess");
2537 break;
2538 }
2539 big_buffer[llen] = 0;
2540 addr2->local_part = string_copy(big_buffer);
2541 }
2542
2543 for (i = 0, sptr = &addr2->message; i < 2; i++, sptr = &addr2->user_message)
2544 {
2545 int message_length;
2546 len = read(pfd[pipe_read], &message_length, sizeof(int));
2547 if (message_length > 0)
2548 {
2549 len = read(pfd[pipe_read], big_buffer, message_length);
2550 big_buffer[big_buffer_size-1] = '\0'; /* guard byte */
2551 if (len > 0) *sptr = string_copy(big_buffer);
2552 }
2553 }
2554 }
2555
2556 else
2557 {
2558 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
2559 "from delivery subprocess", addr2->unique);
2560 break;
2561 }
2562 }
2563
2564 (void)close(pfd[pipe_read]);
2565
2566 /* Unless shadowing, write all successful addresses immediately to the journal
2567 file, to ensure they are recorded asap. For homonymic addresses, use the base
2568 address plus the transport name. Failure to write the journal is panic-worthy,
2569 but don't stop, as it may prove possible subsequently to update the spool file
2570 in order to record the delivery. */
2571
2572 if (!shadowing)
2573 {
2574 for (addr2 = addr; addr2; addr2 = addr2->next)
2575 if (addr2->transport_return == OK)
2576 {
2577 if (testflag(addr2, af_homonym))
2578 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
2579 else
2580 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
2581
2582 /* In the test harness, wait just a bit to let the subprocess finish off
2583 any debug output etc first. */
2584
2585 if (f.running_in_test_harness) millisleep(300);
2586
2587 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
2588 len = Ustrlen(big_buffer);
2589 if (write(journal_fd, big_buffer, len) != len)
2590 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
2591 big_buffer, strerror(errno));
2592 }
2593
2594 /* Ensure the journal file is pushed out to disk. */
2595
2596 if (EXIMfsync(journal_fd) < 0)
2597 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
2598 strerror(errno));
2599 }
2600
2601 /* Wait for the process to finish. If it terminates with a non-zero code,
2602 freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
2603 status values of all the addresses as they are. Take care to handle the case
2604 when the subprocess doesn't seem to exist. This has been seen on one system
2605 when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
2606 happens, wait() doesn't recognize the termination of child processes. Exim now
2607 resets SIGCHLD to SIG_DFL, but this code should still be robust. */
2608
2609 while ((rc = wait(&status)) != pid)
2610 if (rc < 0 && errno == ECHILD) /* Process has vanished */
2611 {
2612 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
2613 addr->transport->driver_name);
2614 status = 0;
2615 break;
2616 }
2617
2618 if ((status & 0xffff) != 0)
2619 {
2620 int msb = (status >> 8) & 255;
2621 int lsb = status & 255;
2622 int code = (msb == 0)? (lsb & 0x7f) : msb;
2623 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
2624 addr->special_action = SPECIAL_FREEZE;
2625 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
2626 "status 0x%04x: %s %d",
2627 addr->transport->driver_name,
2628 status,
2629 msb == 0 ? "terminated by signal" : "exit code",
2630 code);
2631 }
2632
2633 /* If SPECIAL_WARN is set in the top address, send a warning message. */
2634
2635 if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message)
2636 {
2637 int fd;
2638 uschar *warn_message;
2639 pid_t pid;
2640
2641 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
2642
2643 if (!(warn_message = expand_string(addr->transport->warn_message)))
2644 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
2645 "message for %s transport): %s", addr->transport->warn_message,
2646 addr->transport->name, expand_string_message);
2647
2648 else if ((pid = child_open_exim(&fd)) > 0)
2649 {
2650 FILE *f = fdopen(fd, "wb");
2651 if (errors_reply_to && !contains_header(US"Reply-To", warn_message))
2652 fprintf(f, "Reply-To: %s\n", errors_reply_to);
2653 fprintf(f, "Auto-Submitted: auto-replied\n");
2654 if (!contains_header(US"From", warn_message))
2655 moan_write_from(f);
2656 fprintf(f, "%s", CS warn_message);
2657
2658 /* Close and wait for child process to complete, without a timeout. */
2659
2660 (void)fclose(f);
2661 (void)child_close(pid, 0);
2662 }
2663
2664 addr->special_action = SPECIAL_NONE;
2665 }
2666 }
2667
2668
2669
2670
2671 /* Check transport for the given concurrency limit. Return TRUE if over
2672 the limit (or an expansion failure), else FALSE and if there was a limit,
2673 the key for the hints database used for the concurrency count. */
2674
2675 static BOOL
2676 tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key)
2677 {
2678 unsigned max_parallel;
2679
2680 if (!tp->max_parallel) return FALSE;
2681
2682 max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE);
2683 if (expand_string_message)
2684 {
2685 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option "
2686 "in %s transport (%s): %s", tp->name, addr->address,
2687 expand_string_message);
2688 return TRUE;
2689 }
2690
2691 if (max_parallel > 0)
2692 {
2693 uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name);
2694 if (!enq_start(serialize_key, max_parallel))
2695 {
2696 address_item * next;
2697 DEBUG(D_transport)
2698 debug_printf("skipping tpt %s because concurrency limit %u reached\n",
2699 tp->name, max_parallel);
2700 do
2701 {
2702 next = addr->next;
2703 addr->message = US"concurrency limit reached for transport";
2704 addr->basic_errno = ERRNO_TRETRY;
2705 post_process_one(addr, DEFER, LOG_MAIN, EXIM_DTYPE_TRANSPORT, 0);
2706 } while ((addr = next));
2707 return TRUE;
2708 }
2709 *key = serialize_key;
2710 }
2711 return FALSE;
2712 }
2713
2714
2715
2716 /*************************************************
2717 * Do local deliveries *
2718 *************************************************/
2719
2720 /* This function processes the list of addresses in addr_local. True local
2721 deliveries are always done one address at a time. However, local deliveries can
2722 be batched up in some cases. Typically this is when writing batched SMTP output
2723 files for use by some external transport mechanism, or when running local
2724 deliveries over LMTP.
2725
2726 Arguments: None
2727 Returns: Nothing
2728 */
2729
2730 static void
2731 do_local_deliveries(void)
2732 {
2733 open_db dbblock;
2734 open_db *dbm_file = NULL;
2735 time_t now = time(NULL);
2736
2737 /* Loop until we have exhausted the supply of local deliveries */
2738
2739 while (addr_local)
2740 {
2741 struct timeval delivery_start;
2742 struct timeval deliver_time;
2743 address_item *addr2, *addr3, *nextaddr;
2744 int logflags = LOG_MAIN;
2745 int logchar = f.dont_deliver? '*' : '=';
2746 transport_instance *tp;
2747 uschar * serialize_key = NULL;
2748
2749 /* Pick the first undelivered address off the chain */
2750
2751 address_item *addr = addr_local;
2752 addr_local = addr->next;
2753 addr->next = NULL;
2754
2755 DEBUG(D_deliver|D_transport)
2756 debug_printf("--------> %s <--------\n", addr->address);
2757
2758 /* An internal disaster if there is no transport. Should not occur! */
2759
2760 if (!(tp = addr->transport))
2761 {
2762 logflags |= LOG_PANIC;
2763 f.disable_logging = FALSE; /* Jic */
2764 addr->message = addr->router
2765 ? string_sprintf("No transport set by %s router", addr->router->name)
2766 : string_sprintf("No transport set by system filter");
2767 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
2768 continue;
2769 }
2770
2771 /* Check that this base address hasn't previously been delivered to this
2772 transport. The check is necessary at this point to handle homonymic addresses
2773 correctly in cases where the pattern of redirection changes between delivery
2774 attempts. Non-homonymic previous delivery is detected earlier, at routing
2775 time. */
2776
2777 if (previously_transported(addr, FALSE)) continue;
2778
2779 /* There are weird cases where logging is disabled */
2780
2781 f.disable_logging = tp->disable_logging;
2782
2783 /* Check for batched addresses and possible amalgamation. Skip all the work
2784 if either batch_max <= 1 or there aren't any other addresses for local
2785 delivery. */
2786
2787 if (tp->batch_max > 1 && addr_local)
2788 {
2789 int batch_count = 1;
2790 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
2791 BOOL uses_lp = ( testflag(addr, af_pfr)
2792 && (testflag(addr, af_file) || addr->local_part[0] == '|')
2793 )
2794 || readconf_depends((driver_instance *)tp, US"local_part");
2795 uschar *batch_id = NULL;
2796 address_item **anchor = &addr_local;
2797 address_item *last = addr;
2798 address_item *next;
2799
2800 /* Expand the batch_id string for comparison with other addresses.
2801 Expansion failure suppresses batching. */
2802
2803 if (tp->batch_id)
2804 {
2805 deliver_set_expansions(addr);
2806 batch_id = expand_string(tp->batch_id);
2807 deliver_set_expansions(NULL);
2808 if (!batch_id)
2809 {
2810 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2811 "in %s transport (%s): %s", tp->name, addr->address,
2812 expand_string_message);
2813 batch_count = tp->batch_max;
2814 }
2815 }
2816
2817 /* Until we reach the batch_max limit, pick off addresses which have the
2818 same characteristics. These are:
2819
2820 same transport
2821 not previously delivered (see comment about 50 lines above)
2822 same local part if the transport's configuration contains $local_part
2823 or if this is a file or pipe delivery from a redirection
2824 same domain if the transport's configuration contains $domain
2825 same errors address
2826 same additional headers
2827 same headers to be removed
2828 same uid/gid for running the transport
2829 same first host if a host list is set
2830 */
2831
2832 while ((next = *anchor) && batch_count < tp->batch_max)
2833 {
2834 BOOL ok =
2835 tp == next->transport
2836 && !previously_transported(next, TRUE)
2837 && testflag(addr, af_pfr) == testflag(next, af_pfr)
2838 && testflag(addr, af_file) == testflag(next, af_file)
2839 && (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0)
2840 && (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0)
2841 && same_strings(next->prop.errors_address, addr->prop.errors_address)
2842 && same_headers(next->prop.extra_headers, addr->prop.extra_headers)
2843 && same_strings(next->prop.remove_headers, addr->prop.remove_headers)
2844 && same_ugid(tp, addr, next)
2845 && ( !addr->host_list && !next->host_list
2846 || addr->host_list
2847 && next->host_list
2848 && Ustrcmp(addr->host_list->name, next->host_list->name) == 0
2849 );
2850
2851 /* If the transport has a batch_id setting, batch_id will be non-NULL
2852 from the expansion outside the loop. Expand for this address and compare.
2853 Expansion failure makes this address ineligible for batching. */
2854
2855 if (ok && batch_id)
2856 {
2857 uschar *bid;
2858 address_item *save_nextnext = next->next;
2859 next->next = NULL; /* Expansion for a single address */
2860 deliver_set_expansions(next);
2861 next->next = save_nextnext;
2862 bid = expand_string(tp->batch_id);
2863 deliver_set_expansions(NULL);
2864 if (!bid)
2865 {
2866 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2867 "in %s transport (%s): %s", tp->name, next->address,
2868 expand_string_message);
2869 ok = FALSE;
2870 }
2871 else ok = (Ustrcmp(batch_id, bid) == 0);
2872 }
2873
2874 /* Take address into batch if OK. */
2875
2876 if (ok)
2877 {
2878 *anchor = next->next; /* Include the address */
2879 next->next = NULL;
2880 last->next = next;
2881 last = next;
2882 batch_count++;
2883 }
2884 else anchor = &next->next; /* Skip the address */
2885 }
2886 }
2887
2888 /* We now have one or more addresses that can be delivered in a batch. Check
2889 whether the transport is prepared to accept a message of this size. If not,
2890 fail them all forthwith. If the expansion fails, or does not yield an
2891 integer, defer delivery. */
2892
2893 if (tp->message_size_limit)
2894 {
2895 int rc = check_message_size(tp, addr);
2896 if (rc != OK)
2897 {
2898 replicate_status(addr);
2899 while (addr)
2900 {
2901 addr2 = addr->next;
2902 post_process_one(addr, rc, logflags, EXIM_DTYPE_TRANSPORT, 0);
2903 addr = addr2;
2904 }
2905 continue; /* With next batch of addresses */
2906 }
2907 }
2908
2909 /* If we are not running the queue, or if forcing, all deliveries will be
2910 attempted. Otherwise, we must respect the retry times for each address. Even
2911 when not doing this, we need to set up the retry key string, and determine
2912 whether a retry record exists, because after a successful delivery, a delete
2913 retry item must be set up. Keep the retry database open only for the duration
2914 of these checks, rather than for all local deliveries, because some local
2915 deliveries (e.g. to pipes) can take a substantial time. */
2916
2917 if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE)))
2918 {
2919 DEBUG(D_deliver|D_retry|D_hints_lookup)
2920 debug_printf("no retry data available\n");
2921 }
2922
2923 addr2 = addr;
2924 addr3 = NULL;
2925 while (addr2)
2926 {
2927 BOOL ok = TRUE; /* to deliver this address */
2928 uschar *retry_key;
2929
2930 /* Set up the retry key to include the domain or not, and change its
2931 leading character from "R" to "T". Must make a copy before doing this,
2932 because the old key may be pointed to from a "delete" retry item after
2933 a routing delay. */
2934
2935 retry_key = string_copy(
2936 tp->retry_use_local_part ? addr2->address_retry_key :
2937 addr2->domain_retry_key);
2938 *retry_key = 'T';
2939
2940 /* Inspect the retry data. If there is no hints file, delivery happens. */
2941
2942 if (dbm_file)
2943 {
2944 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2945
2946 /* If there is no retry record, delivery happens. If there is,
2947 remember it exists so it can be deleted after a successful delivery. */
2948
2949 if (retry_record)
2950 {
2951 setflag(addr2, af_lt_retry_exists);
2952
2953 /* A retry record exists for this address. If queue running and not
2954 forcing, inspect its contents. If the record is too old, or if its
2955 retry time has come, or if it has passed its cutoff time, delivery
2956 will go ahead. */
2957
2958 DEBUG(D_retry)
2959 {
2960 debug_printf("retry record exists: age=%s ",
2961 readconf_printtime(now - retry_record->time_stamp));
2962 debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
2963 debug_printf(" time to retry = %s expired = %d\n",
2964 readconf_printtime(retry_record->next_try - now),
2965 retry_record->expired);
2966 }
2967
2968 if (f.queue_running && !f.deliver_force)
2969 {
2970 ok = (now - retry_record->time_stamp > retry_data_expire)
2971 || (now >= retry_record->next_try)
2972 || retry_record->expired;
2973
2974 /* If we haven't reached the retry time, there is one more check
2975 to do, which is for the ultimate address timeout. */
2976
2977 if (!ok)
2978 ok = retry_ultimate_address_timeout(retry_key, addr2->domain,
2979 retry_record, now);
2980 }
2981 }
2982 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2983 }
2984
2985 /* This address is to be delivered. Leave it on the chain. */
2986
2987 if (ok)
2988 {
2989 addr3 = addr2;
2990 addr2 = addr2->next;
2991 }
2992
2993 /* This address is to be deferred. Take it out of the chain, and
2994 post-process it as complete. Must take it out of the chain first,
2995 because post processing puts it on another chain. */
2996
2997 else
2998 {
2999 address_item *this = addr2;
3000 this->message = US"Retry time not yet reached";
3001 this->basic_errno = ERRNO_LRETRY;
3002 addr2 = addr3 ? (addr3->next = addr2->next)
3003 : (addr = addr2->next);
3004 post_process_one(this, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
3005 }
3006 }
3007
3008 if (dbm_file) dbfn_close(dbm_file);
3009
3010 /* If there are no addresses left on the chain, they all deferred. Loop
3011 for the next set of addresses. */
3012
3013 if (!addr) continue;
3014
3015 /* If the transport is limited for parallellism, enforce that here.
3016 We use a hints DB entry, incremented here and decremented after
3017 the transport (and any shadow transport) completes. */
3018
3019 if (tpt_parallel_check(tp, addr, &serialize_key))
3020 {
3021 if (expand_string_message)
3022 {
3023 logflags |= LOG_PANIC;
3024 do
3025 {
3026 addr = addr->next;
3027 post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0);
3028 } while ((addr = addr2));
3029 }
3030 continue; /* Loop for the next set of addresses. */
3031 }
3032
3033
3034 /* So, finally, we do have some addresses that can be passed to the
3035 transport. Before doing so, set up variables that are relevant to a
3036 single delivery. */
3037
3038 deliver_set_expansions(addr);
3039
3040 gettimeofday(&delivery_start, NULL);
3041 deliver_local(addr, FALSE);
3042 timesince(&deliver_time, &delivery_start);
3043
3044 /* If a shadow transport (which must perforce be another local transport), is
3045 defined, and its condition is met, we must pass the message to the shadow
3046 too, but only those addresses that succeeded. We do this by making a new
3047 chain of addresses - also to keep the original chain uncontaminated. We must
3048 use a chain rather than doing it one by one, because the shadow transport may
3049 batch.
3050
3051 NOTE: if the condition fails because of a lookup defer, there is nothing we
3052 can do! */
3053
3054 if ( tp->shadow
3055 && ( !tp->shadow_condition
3056 || expand_check_condition(tp->shadow_condition, tp->name, US"transport")
3057 ) )
3058 {
3059 transport_instance *stp;
3060 address_item *shadow_addr = NULL;
3061 address_item **last = &shadow_addr;
3062
3063 for (stp = transports; stp; stp = stp->next)
3064 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
3065
3066 if (!stp)
3067 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
3068 tp->shadow);
3069
3070 /* Pick off the addresses that have succeeded, and make clones. Put into
3071 the shadow_message field a pointer to the shadow_message field of the real
3072 address. */
3073
3074 else for (addr2 = addr; addr2; addr2 = addr2->next)
3075 if (addr2->transport_return == OK)
3076 {
3077 addr3 = store_get(sizeof(address_item));
3078 *addr3 = *addr2;
3079 addr3->next = NULL;
3080 addr3->shadow_message = US &addr2->shadow_message;
3081 addr3->transport = stp;
3082 addr3->transport_return = DEFER;
3083 addr3->return_filename = NULL;
3084 addr3->return_file = -1;
3085 *last = addr3;
3086 last = &addr3->next;
3087 }
3088
3089 /* If we found any addresses to shadow, run the delivery, and stick any
3090 message back into the shadow_message field in the original. */
3091
3092 if (shadow_addr)
3093 {
3094 int save_count = transport_count;
3095
3096 DEBUG(D_deliver|D_transport)
3097 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
3098 deliver_local(shadow_addr, TRUE);
3099
3100 for(; shadow_addr; shadow_addr = shadow_addr->next)
3101 {
3102 int sresult = shadow_addr->transport_return;
3103 *(uschar **)shadow_addr->shadow_message =
3104 sresult == OK
3105 ? string_sprintf(" ST=%s", stp->name)
3106 : string_sprintf(" ST=%s (%s%s%s)", stp->name,
3107 shadow_addr->basic_errno <= 0
3108 ? US""
3109 : US strerror(shadow_addr->basic_errno),
3110 shadow_addr->basic_errno <= 0 || !shadow_addr->message
3111 ? US""
3112 : US": ",
3113 shadow_addr->message
3114 ? shadow_addr->message
3115 : shadow_addr->basic_errno <= 0
3116 ? US"unknown error"
3117 : US"");
3118
3119 DEBUG(D_deliver|D_transport)
3120 debug_printf("%s shadow transport returned %s for %s\n",
3121 stp->name,
3122 sresult == OK ? "OK" :
3123 sresult == DEFER ? "DEFER" :
3124 sresult == FAIL ? "FAIL" :
3125 sresult == PANIC ? "PANIC" : "?",
3126 shadow_addr->address);
3127 }
3128
3129 DEBUG(D_deliver|D_transport)
3130 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
3131
3132 transport_count = save_count; /* Restore original transport count */
3133 }
3134 }
3135
3136 /* Cancel the expansions that were set up for the delivery. */
3137
3138 deliver_set_expansions(NULL);
3139
3140 /* If the transport was parallelism-limited, decrement the hints DB record. */
3141
3142 if (serialize_key) enq_end(serialize_key);
3143
3144 /* Now we can process the results of the real transport. We must take each
3145 address off the chain first, because post_process_one() puts it on another
3146 chain. */
3147
3148 for (addr2 = addr; addr2; addr2 = nextaddr)
3149 {
3150 int result = addr2->transport_return;
3151 nextaddr = addr2->next;
3152
3153 DEBUG(D_deliver|D_transport)
3154 debug_printf("%s transport returned %s for %s\n",
3155 tp->name,
3156 result == OK ? "OK" :
3157 result == DEFER ? "DEFER" :
3158 result == FAIL ? "FAIL" :
3159 result == PANIC ? "PANIC" : "?",
3160 addr2->address);
3161
3162 /* If there is a retry_record, or if delivery is deferred, build a retry
3163 item for setting a new retry time or deleting the old retry record from
3164 the database. These items are handled all together after all addresses
3165 have been handled (so the database is open just for a short time for
3166 updating). */
3167
3168 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
3169 {
3170 int flags = result == DEFER ? 0 : rf_delete;
3171 uschar *retry_key = string_copy(tp->retry_use_local_part
3172 ? addr2->address_retry_key : addr2->domain_retry_key);
3173 *retry_key = 'T';
3174 retry_add_item(addr2, retry_key, flags);
3175 }
3176
3177 /* Done with this address */
3178
3179 if (result == OK)
3180 {
3181 addr2->more_errno = deliver_time.tv_sec;
3182 addr2->delivery_usec = deliver_time.tv_usec;
3183 }
3184 post_process_one(addr2, result, logflags, EXIM_DTYPE_TRANSPORT, logchar);
3185
3186 /* If a pipe delivery generated text to be sent back, the result may be
3187 changed to FAIL, and we must copy this for subsequent addresses in the
3188 batch. */
3189
3190 if (addr2->transport_return != result)
3191 {
3192 for (addr3 = nextaddr; addr3; addr3 = addr3->next)
3193 {
3194 addr3->transport_return = addr2->transport_return;
3195 addr3->basic_errno = addr2->basic_errno;
3196 addr3->message = addr2->message;
3197 }
3198 result = addr2->transport_return;
3199 }
3200
3201 /* Whether or not the result was changed to FAIL, we need to copy the
3202 return_file value from the first address into all the addresses of the
3203 batch, so they are all listed in the error message. */
3204
3205 addr2->return_file = addr->return_file;
3206
3207 /* Change log character for recording successful deliveries. */
3208
3209 if (result == OK) logchar = '-';
3210 }
3211 } /* Loop back for next batch of addresses */
3212 }
3213
3214
3215
3216
3217 /*************************************************
3218 * Sort remote deliveries *
3219 *************************************************/
3220
3221 /* This function is called if remote_sort_domains is set. It arranges that the
3222 chain of addresses for remote deliveries is ordered according to the strings
3223 specified. Try to make this shuffling reasonably efficient by handling
3224 sequences of addresses rather than just single ones.
3225
3226 Arguments: None
3227 Returns: Nothing
3228 */
3229
3230 static void
3231 sort_remote_deliveries(void)
3232 {
3233 int sep = 0;
3234 address_item **aptr = &addr_remote;
3235 const uschar *listptr = remote_sort_domains;
3236 uschar *pattern;
3237 uschar patbuf[256];
3238
3239 while ( *aptr
3240 && (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
3241 )
3242 {
3243 address_item *moved = NULL;
3244 address_item **bptr = &moved;
3245
3246 while (*aptr)
3247 {
3248 address_item **next;
3249 deliver_domain = (*aptr)->domain; /* set $domain */
3250 if (match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
3251 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
3252 {
3253 aptr = &(*aptr)->next;
3254 continue;
3255 }
3256
3257 next = &(*aptr)->next;
3258 while ( *next
3259 && (deliver_domain = (*next)->domain, /* Set $domain */
3260 match_isinlist(deliver_domain, (const uschar **)&pattern, UCHAR_MAX+1,
3261 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK
3262 )
3263 next = &(*next)->next;
3264
3265 /* If the batch of non-matchers is at the end, add on any that were
3266 extracted further up the chain, and end this iteration. Otherwise,
3267 extract them from the chain and hang on the moved chain. */
3268
3269 if (!*next)
3270 {
3271 *next = moved;
3272 break;
3273 }
3274
3275 *bptr = *aptr;
3276 *aptr = *next;
3277 *next = NULL;
3278 bptr = next;
3279 aptr = &(*aptr)->next;
3280 }
3281
3282 /* If the loop ended because the final address matched, *aptr will
3283 be NULL. Add on to the end any extracted non-matching addresses. If
3284 *aptr is not NULL, the loop ended via "break" when *next is null, that
3285 is, there was a string of non-matching addresses at the end. In this
3286 case the extracted addresses have already been added on the end. */
3287
3288 if (!*aptr) *aptr = moved;
3289 }
3290
3291 DEBUG(D_deliver)
3292 {
3293 address_item *addr;
3294 debug_printf("remote addresses after sorting:\n");
3295 for (addr = addr_remote; addr; addr = addr->next)
3296 debug_printf(" %s\n", addr->address);
3297 }
3298 }
3299
3300
3301
3302 /*************************************************
3303 * Read from pipe for remote delivery subprocess *
3304 *************************************************/
3305
3306 /* This function is called when the subprocess is complete, but can also be
3307 called before it is complete, in order to empty a pipe that is full (to prevent
3308 deadlock). It must therefore keep track of its progress in the parlist data
3309 block.
3310
3311 We read the pipe to get the delivery status codes and a possible error message
3312 for each address, optionally preceded by unusability data for the hosts and
3313 also by optional retry data.
3314
3315 Read in large chunks into the big buffer and then scan through, interpreting
3316 the data therein. In most cases, only a single read will be necessary. No
3317 individual item will ever be anywhere near 2500 bytes in length, so by ensuring
3318 that we read the next chunk when there is less than 2500 bytes left in the
3319 non-final chunk, we can assume each item is complete in the buffer before
3320 handling it. Each item is written using a single write(), which is atomic for
3321 small items (less than PIPE_BUF, which seems to be at least 512 in any Unix and
3322 often bigger) so even if we are reading while the subprocess is still going, we
3323 should never have only a partial item in the buffer.
3324
3325 hs12: This assumption is not true anymore, since we get quite large items (certificate
3326 information and such).
3327
3328 Argument:
3329 poffset the offset of the parlist item
3330 eop TRUE if the process has completed
3331
3332 Returns: TRUE if the terminating 'Z' item has been read,
3333 or there has been a disaster (i.e. no more data needed);
3334 FALSE otherwise
3335 */
3336
3337 static BOOL
3338 par_read_pipe(int poffset, BOOL eop)
3339 {
3340 host_item *h;
3341 pardata *p = parlist + poffset;
3342 address_item *addrlist = p->addrlist;
3343 address_item *addr = p->addr;
3344 pid_t pid = p->pid;
3345 int fd = p->fd;
3346
3347 uschar *msg = p->msg;
3348 BOOL done = p->done;
3349
3350 /* Loop through all items, reading from the pipe when necessary. The pipe
3351 used to be non-blocking. But I do not see a reason for using non-blocking I/O
3352 here, as the preceding select() tells us, if data is available for reading.
3353
3354 A read() on a "selected" handle should never block, but(!) it may return
3355 less data then we expected. (The buffer size we pass to read() shouldn't be
3356 understood as a "request", but as a "limit".)
3357
3358 Each separate item is written to the pipe in a timely manner. But, especially for
3359 larger items, the read(2) may already return partial data from the write(2).
3360
3361 The write is atomic mostly (depending on the amount written), but atomic does
3362 not imply "all or noting", it just is "not intermixed" with other writes on the
3363 same channel (pipe).
3364
3365 */
3366
3367 DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
3368 (int)p->pid, eop? "ended" : "not ended yet");
3369
3370 while (!done)
3371 {
3372 retry_item *r, **rp;
3373 uschar pipeheader[PIPE_HEADER_SIZE+1];
3374 uschar *id = &pipeheader[0];
3375 uschar *subid = &pipeheader[1];
3376 uschar *ptr = big_buffer;
3377 size_t required = PIPE_HEADER_SIZE; /* first the pipehaeder, later the data */
3378 ssize_t got;
3379
3380 DEBUG(D_deliver) debug_printf(
3381 "expect %lu bytes (pipeheader) from tpt process %d\n", (u_long)required, pid);
3382
3383 /* We require(!) all the PIPE_HEADER_SIZE bytes here, as we know,
3384 they're written in a timely manner, so waiting for the write shouldn't hurt a lot.
3385 If we get less, we can assume the subprocess do be done and do not expect any further
3386 information from it. */
3387
3388 if ((got = readn(fd, pipeheader, required)) != required)
3389 {
3390 msg = string_sprintf("got " SSIZE_T_FMT " of %d bytes (pipeheader) "
3391 "from transport process %d for transport %s",
3392 got, PIPE_HEADER_SIZE, pid, addr->transport->driver_name);
3393 done = TRUE;
3394 break;
3395 }
3396
3397 pipeheader[PIPE_HEADER_SIZE] = '\0';
3398 DEBUG(D_deliver)
3399 debug_printf("got %ld bytes (pipeheader) from transport process %d\n",
3400 (long) got, pid);
3401
3402 {
3403 /* If we can't decode the pipeheader, the subprocess seems to have a
3404 problem, we do not expect any furher information from it. */
3405 char *endc;
3406 required = Ustrtol(pipeheader+2, &endc, 10);
3407 if (*endc)
3408 {
3409 msg = string_sprintf("failed to read pipe "
3410 "from transport process %d for transport %s: error decoding size from header",
3411 pid, addr->transport->driver_name);
3412 done = TRUE;
3413 break;
3414 }
3415 }
3416
3417 DEBUG(D_deliver)
3418 debug_printf("expect %lu bytes (pipedata) from transport process %d\n",
3419 (u_long)required, pid);
3420
3421 /* Same as above, the transport process will write the bytes announced
3422 in a timely manner, so we can just wait for the bytes, getting less than expected
3423 is considered a problem of the subprocess, we do not expect anything else from it. */
3424 if ((got = readn(fd, big_buffer, required)) != required)
3425 {
3426 msg = string_sprintf("got only " SSIZE_T_FMT " of " SIZE_T_FMT
3427 " bytes (pipedata) from transport process %d for transport %s",
3428 got, required, pid, addr->transport->driver_name);
3429 done = TRUE;
3430 break;
3431 }
3432
3433 /* Handle each possible type of item, assuming the complete item is
3434 available in store. */
3435
3436 switch (*id)
3437 {
3438 /* Host items exist only if any hosts were marked unusable. Match
3439 up by checking the IP address. */
3440
3441 case 'H':
3442 for (h = addrlist->host_list; h; h = h->next)
3443 {
3444 if (!h->address || Ustrcmp(h->address, ptr+2) != 0) continue;
3445 h->status = ptr[0];
3446 h->why = ptr[1];
3447 }
3448 ptr += 2;
3449 while (*ptr++);
3450 break;
3451
3452 /* Retry items are sent in a preceding R item for each address. This is
3453 kept separate to keep each message short enough to guarantee it won't
3454 be split in the pipe. Hopefully, in the majority of cases, there won't in
3455 fact be any retry items at all.
3456
3457 The complete set of retry items might include an item to delete a
3458 routing retry if there was a previous routing delay. However, routing
3459 retries are also used when a remote transport identifies an address error.
3460 In that case, there may also be an "add" item for the same key. Arrange
3461 that a "delete" item is dropped in favour of an "add" item. */
3462
3463 case 'R':
3464 if (!addr) goto ADDR_MISMATCH;
3465
3466 DEBUG(D_deliver|D_retry)
3467 debug_printf("reading retry information for %s from subprocess\n",
3468 ptr+1);
3469
3470 /* Cut out any "delete" items on the list. */
3471
3472 for (rp = &addr->retries; (r = *rp); rp = &r->next)
3473 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
3474 {
3475 if (!(r->flags & rf_delete)) break; /* It was not "delete" */
3476 *rp = r->next; /* Excise a delete item */
3477 DEBUG(D_deliver|D_retry)
3478 debug_printf(" existing delete item dropped\n");
3479 }
3480
3481 /* We want to add a delete item only if there is no non-delete item;
3482 however we still have to step ptr through the data. */
3483
3484 if (!r || !(*ptr & rf_delete))
3485 {
3486 r = store_get(sizeof(retry_item));
3487 r->next = addr->retries;
3488 addr->retries = r;
3489 r->flags = *ptr++;
3490 r->key = string_copy(ptr);
3491 while (*ptr++);
3492 memcpy(&r->basic_errno, ptr, sizeof(r->basic_errno));
3493 ptr += sizeof(r->basic_errno);
3494 memcpy(&r->more_errno, ptr, sizeof(r->more_errno));
3495 ptr += sizeof(r->more_errno);
3496 r->message = *ptr ? string_copy(ptr) : NULL;
3497 DEBUG(D_deliver|D_retry) debug_printf(" added %s item\n",
3498 r->flags & rf_delete ? "delete" : "retry");
3499 }
3500
3501 else
3502 {
3503 DEBUG(D_deliver|D_retry)
3504 debug_printf(" delete item not added: non-delete item exists\n");
3505 ptr++;
3506 while(*ptr++);
3507 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
3508 }
3509
3510 while(*ptr++);
3511 break;
3512
3513 /* Put the amount of data written into the parlist block */
3514
3515 case 'S':
3516 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
3517 ptr += sizeof(transport_count);
3518 break;
3519
3520 /* Address items are in the order of items on the address chain. We
3521 remember the current address value in case this function is called
3522 several times to empty the pipe in stages. Information about delivery
3523 over TLS is sent in a preceding X item for each address. We don't put
3524 it in with the other info, in order to keep each message short enough to
3525 guarantee it won't be split in the pipe. */
3526
3527 #ifdef SUPPORT_TLS
3528 case 'X':
3529 if (!addr) goto ADDR_MISMATCH; /* Below, in 'A' handler */
3530 switch (*subid)
3531 {
3532 case '1':
3533 addr->cipher = NULL;
3534 addr->peerdn = NULL;
3535
3536 if (*ptr)
3537 addr->cipher = string_copy(ptr);
3538 while (*ptr++);
3539 if (*ptr)
3540 addr->peerdn = string_copy(ptr);
3541 break;
3542
3543 case '2':
3544 if (*ptr)
3545 (void) tls_import_cert(ptr, &addr->peercert);
3546 else
3547 addr->peercert = NULL;
3548 break;
3549
3550 case '3':
3551 if (*ptr)
3552 (void) tls_import_cert(ptr, &addr->ourcert);
3553 else
3554 addr->ourcert = NULL;
3555 break;
3556
3557 # ifndef DISABLE_OCSP
3558 case '4':
3559 addr->ocsp = *ptr ? *ptr - '0' : OCSP_NOT_REQ;
3560 break;
3561 # endif
3562 }
3563 while (*ptr++);
3564 break;
3565 #endif /*SUPPORT_TLS*/
3566
3567 case 'C': /* client authenticator information */
3568 switch (*subid)
3569 {
3570 case '1': addr->authenticator = *ptr ? string_copy(ptr) : NULL; break;
3571 case '2': addr->auth_id = *ptr ? string_copy(ptr) : NULL; break;
3572 case '3': addr->auth_sndr = *ptr ? string_copy(ptr) : NULL; break;
3573 }
3574 while (*ptr++);
3575 break;
3576
3577 #ifndef DISABLE_PRDR
3578 case 'P':
3579 setflag(addr, af_prdr_used);
3580 break;
3581 #endif
3582
3583 case 'L':
3584 switch (*subid)
3585 {
3586 #ifdef EXPERIMENTAL_PIPE_CONNECT
3587 case 2: setflag(addr, af_early_pipe); /*FALLTHROUGH*/
3588 #endif
3589 case 1: setflag(addr, af_pipelining); break;
3590 }
3591 break;
3592
3593 case 'K':
3594 setflag(addr, af_chunking_used);
3595 break;
3596
3597 case 'T':
3598 setflag(addr, af_tcp_fastopen_conn);
3599 if (*subid > '0') setflag(addr, af_tcp_fastopen);
3600 if (*subid > '1') setflag(addr, af_tcp_fastopen_data);
3601 break;
3602
3603 case 'D':
3604 if (!addr) goto ADDR_MISMATCH;
3605 memcpy(&(addr->dsn_aware), ptr, sizeof(addr->dsn_aware));
3606 ptr += sizeof(addr->dsn_aware);
3607 DEBUG(D_deliver) debug_printf("DSN read: addr->dsn_aware = %d\n", addr->dsn_aware);
3608 break;
3609
3610 case 'A':
3611 if (!addr)
3612 {
3613 ADDR_MISMATCH:
3614 msg = string_sprintf("address count mismatch for data read from pipe "
3615 "for transport process %d for transport %s", pid,
3616 addrlist->transport->driver_name);
3617 done = TRUE;
3618 break;
3619 }
3620
3621 switch (*subid)
3622 {
3623 #ifdef SUPPORT_SOCKS
3624 case '2': /* proxy information; must arrive before A0 and applies to that addr XXX oops*/
3625 proxy_session = TRUE; /*XXX should this be cleared somewhere? */
3626 if (*ptr == 0)
3627 ptr++;
3628 else
3629 {
3630 proxy_local_address = string_copy(ptr);
3631 while(*ptr++);
3632 memcpy(&proxy_local_port, ptr, sizeof(proxy_local_port));
3633 ptr += sizeof(proxy_local_port);
3634 }
3635 break;
3636 #endif
3637
3638 #ifdef EXPERIMENTAL_DSN_INFO
3639 case '1': /* must arrive before A0, and applies to that addr */
3640 /* Two strings: smtp_greeting and helo_response */
3641 addr->smtp_greeting = string_copy(ptr);
3642 while(*ptr++);
3643 addr->helo_response = string_copy(ptr);
3644 while(*ptr++);
3645 break;
3646 #endif
3647
3648 case '0':
3649 DEBUG(D_deliver) debug_printf("A0 %s tret %d\n", addr->address, *ptr);
3650 addr->transport_return = *ptr++;
3651 addr->special_action = *ptr++;
3652 memcpy(&addr->basic_errno, ptr, sizeof(addr->basic_errno));
3653 ptr += sizeof(addr->basic_errno);
3654 memcpy(&addr->more_errno, ptr, sizeof(addr->more_errno));
3655 ptr += sizeof(addr->more_errno);
3656 memcpy(&addr->delivery_usec, ptr, sizeof(addr->delivery_usec));
3657 ptr += sizeof(addr->delivery_usec);
3658 memcpy(&addr->flags, ptr, sizeof(addr->flags));
3659 ptr += sizeof(addr->flags);
3660 addr->message = *ptr ? string_copy(ptr) : NULL;
3661 while(*ptr++);
3662 addr->user_message = *ptr ? string_copy(ptr) : NULL;
3663 while(*ptr++);
3664
3665 /* Always two strings for host information, followed by the port number and DNSSEC mark */
3666
3667 if (*ptr)
3668 {
3669 h = store_get(sizeof(host_item));
3670 h->name = string_copy(ptr);
3671 while (*ptr++);
3672 h->address = string_copy(ptr);
3673 while(*ptr++);
3674 memcpy(&h->port, ptr, sizeof(h->port));
3675 ptr += sizeof(h->port);
3676 h->dnssec = *ptr == '2' ? DS_YES
3677 : *ptr == '1' ? DS_NO
3678 : DS_UNK;
3679 ptr++;
3680 addr->host_used = h;
3681 }
3682 else ptr++;
3683
3684 /* Finished with this address */
3685
3686 addr = addr->next;
3687 break;
3688 }
3689 break;
3690
3691 /* Local interface address/port */
3692 case 'I':
3693 if (*ptr) sending_ip_address = string_copy(ptr);
3694 while (*ptr++) ;
3695 if (*ptr) sending_port = atoi(CS ptr);
3696 while (*ptr++) ;
3697 break;
3698
3699 /* Z marks the logical end of the data. It is followed by '0' if
3700 continue_transport was NULL at the end of transporting, otherwise '1'.
3701 We need to know when it becomes NULL during a delivery down a passed SMTP
3702 channel so that we don't try to pass anything more down it. Of course, for
3703 most normal messages it will remain NULL all the time. */
3704
3705 case 'Z':
3706 if (*ptr == '0')
3707 {
3708 continue_transport = NULL;
3709 continue_hostname = NULL;
3710 }
3711 done = TRUE;
3712 DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr);
3713 break;
3714
3715 /* Anything else is a disaster. */
3716
3717 default:
3718 msg = string_sprintf("malformed data (%d) read from pipe for transport "
3719 "process %d for transport %s", ptr[-1], pid,
3720 addr->transport->driver_name);
3721 done = TRUE;
3722 break;
3723 }
3724 }
3725
3726 /* The done flag is inspected externally, to determine whether or not to
3727 call the function again when the process finishes. */
3728
3729 p->done = done;
3730
3731 /* If the process hadn't finished, and we haven't seen the end of the data
3732 or if we suffered a disaster, update the rest of the state, and return FALSE to
3733 indicate "not finished". */
3734
3735 if (!eop && !done)
3736 {
3737 p->addr = addr;
3738 p->msg = msg;
3739 return FALSE;
3740 }
3741
3742 /* Close our end of the pipe, to prevent deadlock if the far end is still
3743 pushing stuff into it. */
3744
3745 (void)close(fd);
3746 p->fd = -1;
3747
3748 /* If we have finished without error, but haven't had data for every address,
3749 something is wrong. */
3750
3751 if (!msg && addr)
3752 msg = string_sprintf("insufficient address data read from pipe "
3753 "for transport process %d for transport %s", pid,
3754 addr->transport->driver_name);
3755
3756 /* If an error message is set, something has gone wrong in getting back
3757 the delivery data. Put the message into each address and freeze it. */
3758
3759 if (msg)
3760 for (addr = addrlist; addr; addr = addr->next)
3761 {
3762 addr->transport_return = DEFER;
3763 addr->special_action = SPECIAL_FREEZE;
3764 addr->message = msg;
3765 log_write(0, LOG_MAIN|LOG_PANIC, "Delivery status for %s: %s\n", addr->address, addr->message);
3766 }
3767
3768 /* Return TRUE to indicate we have got all we need from this process, even
3769 if it hasn't actually finished yet. */
3770
3771 return TRUE;
3772 }
3773
3774
3775
3776 /*************************************************
3777 * Post-process a set of remote addresses *
3778 *************************************************/
3779
3780 /* Do what has to be done immediately after a remote delivery for each set of
3781 addresses, then re-write the spool if necessary. Note that post_process_one
3782 puts the address on an appropriate queue; hence we must fish off the next
3783 one first. This function is also called if there is a problem with setting
3784 up a subprocess to do a remote delivery in parallel. In this case, the final
3785 argument contains a message, and the action must be forced to DEFER.
3786
3787 Argument:
3788 addr pointer to chain of address items
3789 logflags flags for logging
3790 msg NULL for normal cases; -> error message for unexpected problems
3791 fallback TRUE if processing fallback hosts
3792
3793 Returns: nothing
3794 */
3795
3796 static void
3797 remote_post_process(address_item *addr, int logflags, uschar *msg,
3798 BOOL fallback)
3799 {
3800 host_item *h;
3801
3802 /* If any host addresses were found to be unusable, add them to the unusable
3803 tree so that subsequent deliveries don't try them. */
3804
3805 for (h = addr->host_list; h; h = h->next)
3806 if (h->address)
3807 if (h->status >= hstatus_unusable) tree_add_unusable(h);
3808
3809 /* Now handle each address on the chain. The transport has placed '=' or '-'
3810 into the special_action field for each successful delivery. */
3811
3812 while (addr)
3813 {
3814 address_item *next = addr->next;
3815
3816 /* If msg == NULL (normal processing) and the result is DEFER and we are
3817 processing the main hosts and there are fallback hosts available, put the
3818 address on the list for fallback delivery. */
3819
3820 if ( addr->transport_return == DEFER
3821 && addr->fallback_hosts
3822 && !fallback
3823 && !msg
3824 )
3825 {
3826 addr->host_list = addr->fallback_hosts;
3827 addr->next = addr_fallback;
3828 addr_fallback = addr;
3829 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
3830 }
3831
3832 /* If msg is set (=> unexpected problem), set it in the address before
3833 doing the ordinary post processing. */
3834
3835 else
3836 {
3837 if (msg)
3838 {
3839 addr->message = msg;
3840 addr->transport_return = DEFER;
3841 }
3842 (void)post_process_one(addr, addr->transport_return, logflags,
3843 EXIM_DTYPE_TRANSPORT, addr->special_action);
3844 }
3845
3846 /* Next address */
3847
3848 addr = next;
3849 }
3850
3851 /* If we have just delivered down a passed SMTP channel, and that was
3852 the last address, the channel will have been closed down. Now that
3853 we have logged that delivery, set continue_sequence to 1 so that
3854 any subsequent deliveries don't get "*" incorrectly logged. */
3855
3856 if (!continue_transport) continue_sequence = 1;
3857 }
3858
3859
3860
3861 /*************************************************
3862 * Wait for one remote delivery subprocess *
3863 *************************************************/
3864
3865 /* This function is called while doing remote deliveries when either the
3866 maximum number of processes exist and we need one to complete so that another
3867 can be created, or when waiting for the last ones to complete. It must wait for
3868 the completion of one subprocess, empty the control block slot, and return a
3869 pointer to the address chain.
3870
3871 Arguments: none
3872 Returns: pointer to the chain of addresses handled by the process;
3873 NULL if no subprocess found - this is an unexpected error
3874 */
3875
3876 static address_item *
3877 par_wait(void)
3878 {
3879 int poffset, status;
3880 address_item *addr, *addrlist;
3881 pid_t pid;
3882
3883 set_process_info("delivering %s: waiting for a remote delivery subprocess "
3884 "to finish", message_id);
3885
3886 /* Loop until either a subprocess completes, or there are no subprocesses in
3887 existence - in which case give an error return. We cannot proceed just by
3888 waiting for a completion, because a subprocess may have filled up its pipe, and
3889 be waiting for it to be emptied. Therefore, if no processes have finished, we
3890 wait for one of the pipes to acquire some data by calling select(), with a
3891 timeout just in case.
3892
3893 The simple approach is just to iterate after reading data from a ready pipe.
3894 This leads to non-ideal behaviour when the subprocess has written its final Z
3895 item, closed the pipe, and is in the process of exiting (the common case). A
3896 call to waitpid() yields nothing completed, but select() shows the pipe ready -
3897 reading it yields EOF, so you end up with busy-waiting until the subprocess has
3898 actually finished.
3899
3900 To avoid this, if all the data that is needed has been read from a subprocess
3901 after select(), an explicit wait() for it is done. We know that all it is doing
3902 is writing to the pipe and then exiting, so the wait should not be long.
3903
3904 The non-blocking waitpid() is to some extent just insurance; if we could
3905 reliably detect end-of-file on the pipe, we could always know when to do a
3906 blocking wait() for a completed process. However, because some systems use
3907 NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
3908 use code that functions without the need to recognize EOF.
3909
3910 There's a double loop here just in case we end up with a process that is not in
3911 the list of remote delivery processes. Something has obviously gone wrong if
3912 this is the case. (For example, a process that is incorrectly left over from
3913 routing or local deliveries might be found.) The damage can be minimized by
3914 looping back and looking for another process. If there aren't any, the error
3915 return will happen. */
3916
3917 for (;;) /* Normally we do not repeat this loop */
3918 {
3919 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3920 {
3921 struct timeval tv;
3922 fd_set select_pipes;
3923 int maxpipe, readycount;
3924
3925 /* A return value of -1 can mean several things. If errno != ECHILD, it
3926 either means invalid options (which we discount), or that this process was
3927 interrupted by a signal. Just loop to try the waitpid() again.
3928
3929 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3930 in existence. This should never happen, and is an unexpected error.
3931 However, there is a nasty complication when running under Linux. If "strace
3932 -f" is being used under Linux to trace this process and its children,
3933 subprocesses are "stolen" from their parents and become the children of the
3934 tracing process. A general wait such as the one we've just obeyed returns
3935 as if there are no children while subprocesses are running. Once a
3936 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3937 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3938 palliative.
3939
3940 This does not happen using "truss" on Solaris, nor (I think) with other
3941 tracing facilities on other OS. It seems to be specific to Linux.
3942
3943 What we do to get round this is to use kill() to see if any of our
3944 subprocesses are still in existence. If kill() gives an OK return, we know
3945 it must be for one of our processes - it can't be for a re-use of the pid,
3946 because if our process had finished, waitpid() would have found it. If any
3947 of our subprocesses are in existence, we proceed to use select() as if
3948 waitpid() had returned zero. I think this is safe. */
3949
3950 if (pid < 0)
3951 {
3952 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3953
3954 DEBUG(D_deliver)
3955 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3956 "for process existence\n");
3957
3958 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3959 {
3960 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3961 {
3962 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3963 "stolen by strace\n", (int)pid);
3964 break; /* With poffset set */
3965 }
3966 }
3967
3968 if (poffset >= remote_max_parallel)
3969 {
3970 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3971 return NULL; /* This is the error return */
3972 }
3973 }
3974
3975 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3976 been handled above. A return value of zero means that there is at least one
3977 subprocess, but there are no completed subprocesses. See if any pipes are
3978 ready with any data for reading. */
3979
3980 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3981
3982 maxpipe = 0;
3983 FD_ZERO(&select_pipes);
3984 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3985 if (parlist[poffset].pid != 0)
3986 {
3987 int fd = parlist[poffset].fd;
3988 FD_SET(fd, &select_pipes);
3989 if (fd > maxpipe) maxpipe = fd;
3990 }
3991
3992 /* Stick in a 60-second timeout, just in case. */
3993
3994 tv.tv_sec = 60;
3995 tv.tv_usec = 0;
3996
3997 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3998 NULL, NULL, &tv);
3999
4000 /* Scan through the pipes and read any that are ready; use the count
4001 returned by select() to stop when there are no more. Select() can return
4002 with no processes (e.g. if interrupted). This shouldn't matter.
4003
4004 If par_read_pipe() returns TRUE, it means that either the terminating Z was
4005 read, or there was a disaster. In either case, we are finished with this
4006 process. Do an explicit wait() for the process and break the main loop if
4007 it succeeds.
4008
4009 It turns out that we have to deal with the case of an interrupted system
4010 call, which can happen on some operating systems if the signal handling is
4011 set up to do that by default. */
4012
4013 for (poffset = 0;
4014 readycount > 0 && poffset < remote_max_parallel;
4015 poffset++)
4016 {
4017 if ( (pid = parlist[poffset].pid) != 0
4018 && FD_ISSET(parlist[poffset].fd, &select_pipes)
4019 )
4020 {
4021 readycount--;
4022 if (par_read_pipe(poffset, FALSE)) /* Finished with this pipe */
4023 for (;;) /* Loop for signals */
4024 {
4025 pid_t endedpid = waitpid(pid, &status, 0);
4026 if (endedpid == pid) goto PROCESS_DONE;
4027 if (endedpid != (pid_t)(-1) || errno != EINTR)
4028 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return "
4029 "%d (errno = %d) from waitpid() for process %d",
4030 (int)endedpid, errno, (int)pid);
4031 }
4032 }
4033 }
4034
4035 /* Now go back and look for a completed subprocess again. */
4036 }
4037
4038 /* A completed process was detected by the non-blocking waitpid(). Find the
4039 data block that corresponds to this subprocess. */
4040
4041 for (poffset = 0; poffset < remote_max_parallel; poffset++)
4042 if (pid == parlist[poffset].pid) break;
4043
4044 /* Found the data block; this is a known remote delivery process. We don't
4045 need to repeat the outer loop. This should be what normally happens. */
4046
4047 if (poffset < remote_max_parallel) break;
4048
4049 /* This situation is an error, but it's probably better to carry on looking
4050 for another process than to give up (as we used to do). */