Added log_selector=+queue_time_overall.
[exim.git] / src / src / deliver.c
1 /* $Cambridge: exim/src/src/deliver.c,v 1.3 2004/11/24 14:38:13 ph10 Exp $ */
2
3 /*************************************************
4 * Exim - an Internet mail transport agent *
5 *************************************************/
6
7 /* Copyright (c) University of Cambridge 1995 - 2004 */
8 /* See the file NOTICE for conditions of use and distribution. */
9
10 /* The main code for delivering a message. */
11
12
13 #include "exim.h"
14
15
16 /* Data block for keeping track of subprocesses for parallel remote
17 delivery. */
18
19 typedef struct pardata {
20 address_item *addrlist; /* chain of addresses */
21 address_item *addr; /* next address data expected for */
22 pid_t pid; /* subprocess pid */
23 int fd; /* pipe fd for getting result from subprocess */
24 int transport_count; /* returned transport count value */
25 BOOL done; /* no more data needed */
26 uschar *msg; /* error message */
27 uschar *return_path; /* return_path for these addresses */
28 } pardata;
29
30 /* Values for the process_recipients variable */
31
32 enum { RECIP_ACCEPT, RECIP_IGNORE, RECIP_DEFER,
33 RECIP_FAIL, RECIP_FAIL_FILTER, RECIP_FAIL_TIMEOUT,
34 RECIP_FAIL_LOOP};
35
36 /* Mutually recursive functions for marking addresses done. */
37
38 static void child_done(address_item *, uschar *);
39 static void address_done(address_item *, uschar *);
40
41 /* Table for turning base-62 numbers into binary */
42
43 static uschar tab62[] =
44 {0,1,2,3,4,5,6,7,8,9,0,0,0,0,0,0, /* 0-9 */
45 0,10,11,12,13,14,15,16,17,18,19,20, /* A-K */
46 21,22,23,24,25,26,27,28,29,30,31,32, /* L-W */
47 33,34,35, 0, 0, 0, 0, 0, /* X-Z */
48 0,36,37,38,39,40,41,42,43,44,45,46, /* a-k */
49 47,48,49,50,51,52,53,54,55,56,57,58, /* l-w */
50 59,60,61}; /* x-z */
51
52
53 /*************************************************
54 * Local static variables *
55 *************************************************/
56
57 /* addr_duplicate is global because it needs to be seen from the Envelope-To
58 writing code. */
59
60 static address_item *addr_defer = NULL;
61 static address_item *addr_failed = NULL;
62 static address_item *addr_fallback = NULL;
63 static address_item *addr_local = NULL;
64 static address_item *addr_new = NULL;
65 static address_item *addr_remote = NULL;
66 static address_item *addr_route = NULL;
67 static address_item *addr_succeed = NULL;
68
69 static FILE *message_log = NULL;
70 static BOOL update_spool;
71 static BOOL remove_journal;
72 static int parcount = 0;
73 static pardata *parlist = NULL;
74 static int return_count;
75 static uschar *frozen_info = US"";
76 static uschar *used_return_path = NULL;
77
78 static uschar spoolname[PATH_MAX];
79
80
81
82 /*************************************************
83 * Make a new address item *
84 *************************************************/
85
86 /* This function gets the store and initializes with default values. The
87 transport_return value defaults to DEFER, so that any unexpected failure to
88 deliver does not wipe out the message. The default unique string is set to a
89 copy of the address, so that its domain can be lowercased.
90
91 Argument:
92 address the RFC822 address string
93 copy force a copy of the address
94
95 Returns: a pointer to an initialized address_item
96 */
97
98 address_item *
99 deliver_make_addr(uschar *address, BOOL copy)
100 {
101 address_item *addr = store_get(sizeof(address_item));
102 *addr = address_defaults;
103 if (copy) address = string_copy(address);
104 addr->address = address;
105 addr->unique = string_copy(address);
106 return addr;
107 }
108
109
110
111
112 /*************************************************
113 * Set expansion values for an address *
114 *************************************************/
115
116 /* Certain expansion variables are valid only when handling an address or
117 address list. This function sets them up or clears the values, according to its
118 argument.
119
120 Arguments:
121 addr the address in question, or NULL to clear values
122 Returns: nothing
123 */
124
125 void
126 deliver_set_expansions(address_item *addr)
127 {
128 if (addr == NULL)
129 {
130 uschar ***p = address_expansions;
131 while (*p != NULL) **p++ = NULL;
132 return;
133 }
134
135 /* Exactly what gets set depends on whether there is one or more addresses, and
136 what they contain. These first ones are always set, taking their values from
137 the first address. */
138
139 if (addr->host_list == NULL)
140 {
141 deliver_host = deliver_host_address = US"";
142 }
143 else
144 {
145 deliver_host = addr->host_list->name;
146 deliver_host_address = addr->host_list->address;
147 }
148
149 deliver_recipients = addr;
150 deliver_address_data = addr->p.address_data;
151 deliver_domain_data = addr->p.domain_data;
152 deliver_localpart_data = addr->p.localpart_data;
153
154 /* These may be unset for multiple addresses */
155
156 deliver_domain = addr->domain;
157 self_hostname = addr->self_hostname;
158
159 /* If there's only one address we can set everything. */
160
161 if (addr->next == NULL)
162 {
163 address_item *addr_orig;
164
165 deliver_localpart = addr->local_part;
166 deliver_localpart_prefix = addr->prefix;
167 deliver_localpart_suffix = addr->suffix;
168
169 for (addr_orig = addr; addr_orig->parent != NULL;
170 addr_orig = addr_orig->parent);
171 deliver_domain_orig = addr_orig->domain;
172
173 /* Re-instate any prefix and suffix in the original local part. In all
174 normal cases, the address will have a router associated with it, and we can
175 choose the caseful or caseless version accordingly. However, when a system
176 filter sets up a pipe, file, or autoreply delivery, no router is involved.
177 In this case, though, there won't be any prefix or suffix to worry about. */
178
179 deliver_localpart_orig = (addr_orig->router == NULL)? addr_orig->local_part :
180 addr_orig->router->caseful_local_part?
181 addr_orig->cc_local_part : addr_orig->lc_local_part;
182
183 /* If there's a parent, make its domain and local part available, and if
184 delivering to a pipe or file, or sending an autoreply, get the local
185 part from the parent. For pipes and files, put the pipe or file string
186 into address_pipe and address_file. */
187
188 if (addr->parent != NULL)
189 {
190 deliver_domain_parent = addr->parent->domain;
191 deliver_localpart_parent = (addr->parent->router == NULL)?
192 addr->parent->local_part :
193 addr->parent->router->caseful_local_part?
194 addr->parent->cc_local_part : addr->parent->lc_local_part;
195
196 /* File deliveries have their own flag because they need to be picked out
197 as special more often. */
198
199 if (testflag(addr, af_pfr))
200 {
201 if (testflag(addr, af_file)) address_file = addr->local_part;
202 else if (deliver_localpart[0] == '|') address_pipe = addr->local_part;
203 deliver_localpart = addr->parent->local_part;
204 deliver_localpart_prefix = addr->parent->prefix;
205 deliver_localpart_suffix = addr->parent->suffix;
206 }
207 }
208 }
209
210 /* For multiple addresses, don't set local part, and leave the domain and
211 self_hostname set only if it is the same for all of them. */
212
213 else
214 {
215 address_item *addr2;
216 for (addr2 = addr->next; addr2 != NULL; addr2 = addr2->next)
217 {
218 if (deliver_domain != NULL &&
219 Ustrcmp(deliver_domain, addr2->domain) != 0)
220 deliver_domain = NULL;
221 if (self_hostname != NULL && (addr2->self_hostname == NULL ||
222 Ustrcmp(self_hostname, addr2->self_hostname) != 0))
223 self_hostname = NULL;
224 if (deliver_domain == NULL && self_hostname == NULL) break;
225 }
226 }
227 }
228
229
230
231
232 /*************************************************
233 * Open a msglog file *
234 *************************************************/
235
236 /* This function is used both for normal message logs, and for files in the
237 msglog directory that are used to catch output from pipes. Try to create the
238 directory if it does not exist. From release 4.21, normal message logs should
239 be created when the message is received.
240
241 Argument:
242 filename the file name
243 mode the mode required
244 error used for saying what failed
245
246 Returns: a file descriptor, or -1 (with errno set)
247 */
248
249 static int
250 open_msglog_file(uschar *filename, int mode, uschar **error)
251 {
252 int fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
253
254 if (fd < 0 && errno == ENOENT)
255 {
256 uschar temp[16];
257 sprintf(CS temp, "msglog/%s", message_subdir);
258 if (message_subdir[0] == 0) temp[6] = 0;
259 (void)directory_make(spool_directory, temp, MSGLOG_DIRECTORY_MODE, TRUE);
260 fd = Uopen(filename, O_WRONLY|O_APPEND|O_CREAT, mode);
261 }
262
263 /* Set the close-on-exec flag and change the owner to the exim uid/gid (this
264 function is called as root). Double check the mode, because the group setting
265 doesn't always get set automatically. */
266
267 if (fd >= 0)
268 {
269 fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
270 if (fchown(fd, exim_uid, exim_gid) < 0)
271 {
272 *error = US"chown";
273 return -1;
274 }
275 if (fchmod(fd, mode) < 0)
276 {
277 *error = US"chmod";
278 return -1;
279 }
280 }
281 else *error = US"create";
282
283 return fd;
284 }
285
286
287
288
289 /*************************************************
290 * Write to msglog if required *
291 *************************************************/
292
293 /* Write to the message log, if configured. This function may also be called
294 from transports.
295
296 Arguments:
297 format a string format
298
299 Returns: nothing
300 */
301
302 void
303 deliver_msglog(const char *format, ...)
304 {
305 va_list ap;
306 if (!message_logs) return;
307 va_start(ap, format);
308 vfprintf(message_log, format, ap);
309 fflush(message_log);
310 va_end(ap);
311 }
312
313
314
315
316 /*************************************************
317 * Replicate status for batch *
318 *************************************************/
319
320 /* When a transport handles a batch of addresses, it may treat them
321 individually, or it may just put the status in the first one, and return FALSE,
322 requesting that the status be copied to all the others externally. This is the
323 replication function. As well as the status, it copies the transport pointer,
324 which may have changed if appendfile passed the addresses on to a different
325 transport.
326
327 Argument: pointer to the first address in a chain
328 Returns: nothing
329 */
330
331 static void
332 replicate_status(address_item *addr)
333 {
334 address_item *addr2;
335 for (addr2 = addr->next; addr2 != NULL; addr2 = addr2->next)
336 {
337 addr2->transport = addr->transport;
338 addr2->transport_return = addr->transport_return;
339 addr2->basic_errno = addr->basic_errno;
340 addr2->more_errno = addr->more_errno;
341 addr2->special_action = addr->special_action;
342 addr2->message = addr->message;
343 addr2->user_message = addr->user_message;
344 }
345 }
346
347
348
349 /*************************************************
350 * Compare lists of hosts *
351 *************************************************/
352
353 /* This function is given two pointers to chains of host items, and it yields
354 TRUE if the lists refer to the same hosts in the same order, except that
355
356 (1) Multiple hosts with the same non-negative MX values are permitted to appear
357 in different orders. Round-robinning nameservers can cause this to happen.
358
359 (2) Multiple hosts with the same negative MX values less than MX_NONE are also
360 permitted to appear in different orders. This is caused by randomizing
361 hosts lists.
362
363 This enables Exim to use a single SMTP transaction for sending to two entirely
364 different domains that happen to end up pointing at the same hosts.
365
366 Arguments:
367 one points to the first host list
368 two points to the second host list
369
370 Returns: TRUE if the lists refer to the same host set
371 */
372
373 static BOOL
374 same_hosts(host_item *one, host_item *two)
375 {
376 while (one != NULL && two != NULL)
377 {
378 if (Ustrcmp(one->name, two->name) != 0)
379 {
380 int mx = one->mx;
381 host_item *end_one = one;
382 host_item *end_two = two;
383
384 /* Batch up only if there was no MX and the list was not randomized */
385
386 if (mx == MX_NONE) return FALSE;
387
388 /* Find the ends of the shortest sequence of identical MX values */
389
390 while (end_one->next != NULL && end_one->next->mx == mx &&
391 end_two->next != NULL && end_two->next->mx == mx)
392 {
393 end_one = end_one->next;
394 end_two = end_two->next;
395 }
396
397 /* If there aren't any duplicates, there's no match. */
398
399 if (end_one == one) return FALSE;
400
401 /* For each host in the 'one' sequence, check that it appears in the 'two'
402 sequence, returning FALSE if not. */
403
404 for (;;)
405 {
406 host_item *hi;
407 for (hi = two; hi != end_two->next; hi = hi->next)
408 if (Ustrcmp(one->name, hi->name) == 0) break;
409 if (hi == end_two->next) return FALSE;
410 if (one == end_one) break;
411 one = one->next;
412 }
413
414 /* All the hosts in the 'one' sequence were found in the 'two' sequence.
415 Ensure both are pointing at the last host, and carry on as for equality. */
416
417 two = end_two;
418 }
419
420 /* Hosts matched */
421
422 one = one->next;
423 two = two->next;
424 }
425
426 /* True if both are NULL */
427
428 return (one == two);
429 }
430
431
432
433 /*************************************************
434 * Compare header lines *
435 *************************************************/
436
437 /* This function is given two pointers to chains of header items, and it yields
438 TRUE if they are the same header texts in the same order.
439
440 Arguments:
441 one points to the first header list
442 two points to the second header list
443
444 Returns: TRUE if the lists refer to the same header set
445 */
446
447 static BOOL
448 same_headers(header_line *one, header_line *two)
449 {
450 for (;;)
451 {
452 if (one == two) return TRUE; /* Includes the case where both NULL */
453 if (one == NULL || two == NULL) return FALSE;
454 if (Ustrcmp(one->text, two->text) != 0) return FALSE;
455 one = one->next;
456 two = two->next;
457 }
458 }
459
460
461
462 /*************************************************
463 * Compare string settings *
464 *************************************************/
465
466 /* This function is given two pointers to strings, and it returns
467 TRUE if they are the same pointer, or if the two strings are the same.
468
469 Arguments:
470 one points to the first string
471 two points to the second string
472
473 Returns: TRUE or FALSE
474 */
475
476 static BOOL
477 same_strings(uschar *one, uschar *two)
478 {
479 if (one == two) return TRUE; /* Includes the case where both NULL */
480 if (one == NULL || two == NULL) return FALSE;
481 return (Ustrcmp(one, two) == 0);
482 }
483
484
485
486 /*************************************************
487 * Compare uid/gid for addresses *
488 *************************************************/
489
490 /* This function is given a transport and two addresses. It yields TRUE if the
491 uid/gid/initgroups settings for the two addresses are going to be the same when
492 they are delivered.
493
494 Arguments:
495 tp the transort
496 addr1 the first address
497 addr2 the second address
498
499 Returns: TRUE or FALSE
500 */
501
502 static BOOL
503 same_ugid(transport_instance *tp, address_item *addr1, address_item *addr2)
504 {
505 if (!tp->uid_set && tp->expand_uid == NULL && !tp->deliver_as_creator)
506 {
507 if (testflag(addr1, af_uid_set) != testflag(addr2, af_gid_set) ||
508 (testflag(addr1, af_uid_set) &&
509 (addr1->uid != addr2->uid ||
510 testflag(addr1, af_initgroups) != testflag(addr2, af_initgroups))))
511 return FALSE;
512 }
513
514 if (!tp->gid_set && tp->expand_gid == NULL)
515 {
516 if (testflag(addr1, af_gid_set) != testflag(addr2, af_gid_set) ||
517 (testflag(addr1, af_gid_set) && addr1->gid != addr2->gid))
518 return FALSE;
519 }
520
521 return TRUE;
522 }
523
524
525
526
527 /*************************************************
528 * Record that an address is complete *
529 *************************************************/
530
531 /* This function records that an address is complete. This is straightforward
532 for most addresses, where the unique address is just the full address with the
533 domain lower cased. For homonyms (addresses that are the same as one of their
534 ancestors) their are complications. Their unique addresses have \x\ prepended
535 (where x = 0, 1, 2...), so that de-duplication works correctly for siblings and
536 cousins.
537
538 Exim used to record the unique addresses of homonyms as "complete". This,
539 however, fails when the pattern of redirection varies over time (e.g. if taking
540 unseen copies at only some times of day) because the prepended numbers may vary
541 from one delivery run to the next. This problem is solved by never recording
542 prepended unique addresses as complete. Instead, when a homonymic address has
543 actually been delivered via a transport, we record its basic unique address
544 followed by the name of the transport. This is checked in subsequent delivery
545 runs whenever an address is routed to a transport.
546
547 If the completed address is a top-level one (has no parent, which means it
548 cannot be homonymic) we also add the original address to the non-recipients
549 tree, so that it gets recorded in the spool file and therefore appears as
550 "done" in any spool listings. The original address may differ from the unique
551 address in the case of the domain.
552
553 Finally, this function scans the list of duplicates, marks as done any that
554 match this address, and calls child_done() for their ancestors.
555
556 Arguments:
557 addr address item that has been completed
558 now current time as a string
559
560 Returns: nothing
561 */
562
563 static void
564 address_done(address_item *addr, uschar *now)
565 {
566 address_item *dup;
567
568 update_spool = TRUE; /* Ensure spool gets updated */
569
570 /* Top-level address */
571
572 if (addr->parent == NULL)
573 {
574 tree_add_nonrecipient(addr->unique);
575 tree_add_nonrecipient(addr->address);
576 }
577
578 /* Homonymous child address */
579
580 else if (testflag(addr, af_homonym))
581 {
582 if (addr->transport != NULL)
583 {
584 tree_add_nonrecipient(
585 string_sprintf("%s/%s", addr->unique + 3, addr->transport->name));
586 }
587 }
588
589 /* Non-homonymous child address */
590
591 else tree_add_nonrecipient(addr->unique);
592
593 /* Check the list of duplicate addresses and ensure they are now marked
594 done as well. */
595
596 for (dup = addr_duplicate; dup != NULL; dup = dup->next)
597 {
598 if (Ustrcmp(addr->unique, dup->unique) == 0)
599 {
600 tree_add_nonrecipient(dup->address);
601 child_done(dup, now);
602 }
603 }
604 }
605
606
607
608
609 /*************************************************
610 * Decrease counts in parents and mark done *
611 *************************************************/
612
613 /* This function is called when an address is complete. If there is a parent
614 address, its count of children is decremented. If there are still other
615 children outstanding, the function exits. Otherwise, if the count has become
616 zero, address_done() is called to mark the parent and its duplicates complete.
617 Then loop for any earlier ancestors.
618
619 Arguments:
620 addr points to the completed address item
621 now the current time as a string, for writing to the message log
622
623 Returns: nothing
624 */
625
626 static void
627 child_done(address_item *addr, uschar *now)
628 {
629 address_item *aa;
630 while (addr->parent != NULL)
631 {
632 addr = addr->parent;
633 if ((addr->child_count -= 1) > 0) return; /* Incomplete parent */
634 address_done(addr, now);
635
636 /* Log the completion of all descendents only when there is no ancestor with
637 the same original address. */
638
639 for (aa = addr->parent; aa != NULL; aa = aa->parent)
640 if (Ustrcmp(aa->address, addr->address) == 0) break;
641 if (aa != NULL) continue;
642
643 deliver_msglog("%s %s: children all complete\n", now, addr->address);
644 DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->address);
645 }
646 }
647
648
649
650
651 /*************************************************
652 * Actions at the end of handling an address *
653 *************************************************/
654
655 /* This is a function for processing a single address when all that can be done
656 with it has been done.
657
658 Arguments:
659 addr points to the address block
660 result the result of the delivery attempt
661 logflags flags for log_write() (LOG_MAIN and/or LOG_PANIC)
662 driver_type indicates which type of driver (transport, or router) was last
663 to process the address
664 logchar '=' or '-' for use when logging deliveries with => or ->
665
666 Returns: nothing
667 */
668
669 static void
670 post_process_one(address_item *addr, int result, int logflags, int driver_type,
671 int logchar)
672 {
673 uschar *now = tod_stamp(tod_log);
674 uschar *driver_kind = NULL;
675 uschar *driver_name = NULL;
676 uschar *log_address;
677
678 int size = 256; /* Used for a temporary, */
679 int ptr = 0; /* expanding buffer, for */
680 uschar *s; /* building log lines; */
681 void *reset_point; /* released afterwards. */
682
683
684 DEBUG(D_deliver) debug_printf("post-process %s (%d)\n", addr->address, result);
685
686 /* Set up driver kind and name for logging. Disable logging if the router or
687 transport has disabled it. */
688
689 if (driver_type == DTYPE_TRANSPORT)
690 {
691 if (addr->transport != NULL)
692 {
693 driver_name = addr->transport->name;
694 driver_kind = US" transport";
695 disable_logging = addr->transport->disable_logging;
696 }
697 else driver_kind = US"transporting";
698 }
699 else if (driver_type == DTYPE_ROUTER)
700 {
701 if (addr->router != NULL)
702 {
703 driver_name = addr->router->name;
704 driver_kind = US" router";
705 disable_logging = addr->router->disable_logging;
706 }
707 else driver_kind = US"routing";
708 }
709
710 /* If there's an error message set, ensure that it contains only printing
711 characters - it should, but occasionally things slip in and this at least
712 stops the log format from getting wrecked. */
713
714 if (addr->message != NULL) addr->message = string_printing(addr->message);
715
716 /* If we used a transport that has one of the "return_output" options set, and
717 if it did in fact generate some output, then for return_output we treat the
718 message as failed if it was not already set that way, so that the output gets
719 returned to the sender, provided there is a sender to send it to. For
720 return_fail_output, do this only if the delivery failed. Otherwise we just
721 unlink the file, and remove the name so that if the delivery failed, we don't
722 try to send back an empty or unwanted file. The log_output options operate only
723 on a non-empty file.
724
725 In any case, we close the message file, because we cannot afford to leave a
726 file-descriptor for one address while processing (maybe very many) others. */
727
728 if (addr->return_file >= 0 && addr->return_filename != NULL)
729 {
730 BOOL return_output = FALSE;
731 struct stat statbuf;
732 fsync(addr->return_file);
733
734 /* If there is no output, do nothing. */
735
736 if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0)
737 {
738 transport_instance *tb = addr->transport;
739
740 /* Handle logging options */
741
742 if (tb->log_output || (result == FAIL && tb->log_fail_output) ||
743 (result == DEFER && tb->log_defer_output))
744 {
745 uschar *s;
746 FILE *f = Ufopen(addr->return_filename, "rb");
747 if (f == NULL)
748 log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output "
749 "from %s transport: %s", addr->return_filename, tb->name,
750 strerror(errno));
751 else
752 {
753 s = US Ufgets(big_buffer, big_buffer_size, f);
754 if (s != NULL)
755 {
756 uschar *p = big_buffer + Ustrlen(big_buffer);
757 while (p > big_buffer && isspace(p[-1])) p--;
758 *p = 0;
759 s = string_printing(big_buffer);
760 log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
761 addr->address, tb->name, s);
762 }
763 fclose(f);
764 }
765 }
766
767 /* Handle returning options, but only if there is an address to return
768 the text to. */
769
770 if (sender_address[0] != 0 || addr->p.errors_address != NULL)
771 {
772 if (tb->return_output)
773 {
774 addr->transport_return = result = FAIL;
775 if (addr->basic_errno == 0 && addr->message == NULL)
776 addr->message = US"return message generated";
777 return_output = TRUE;
778 }
779 else
780 if (tb->return_fail_output && result == FAIL) return_output = TRUE;
781 }
782 }
783
784 /* Get rid of the file unless it might be returned, but close it in
785 all cases. */
786
787 if (!return_output)
788 {
789 Uunlink(addr->return_filename);
790 addr->return_filename = NULL;
791 addr->return_file = -1;
792 }
793
794 close(addr->return_file);
795 }
796
797 /* Create the address string for logging. Must not do this earlier, because
798 an OK result may be changed to FAIL when a pipe returns text. */
799
800 log_address = string_log_address(addr,
801 (log_write_selector & L_all_parents) != 0, result == OK);
802
803 /* The sucess case happens only after delivery by a transport. */
804
805 if (result == OK)
806 {
807 addr->next = addr_succeed;
808 addr_succeed = addr;
809
810 /* Call address_done() to ensure that we don't deliver to this address again,
811 and write appropriate things to the message log. If it is a child address, we
812 call child_done() to scan the ancestors and mark them complete if this is the
813 last child to complete. */
814
815 address_done(addr, now);
816 DEBUG(D_deliver) debug_printf("%s delivered\n", addr->address);
817
818 if (addr->parent == NULL)
819 {
820 deliver_msglog("%s %s: %s%s succeeded\n", now, addr->address,
821 driver_name, driver_kind);
822 }
823 else
824 {
825 deliver_msglog("%s %s <%s>: %s%s succeeded\n", now, addr->address,
826 addr->parent->address, driver_name, driver_kind);
827 child_done(addr, now);
828 }
829
830 /* Log the delivery on the main log. We use an extensible string to build up
831 the log line, and reset the store afterwards. Remote deliveries should always
832 have a pointer to the host item that succeeded; local deliveries can have a
833 pointer to a single host item in their host list, for use by the transport. */
834
835 s = reset_point = store_get(size);
836 s[ptr++] = logchar;
837
838 s = string_append(s, &size, &ptr, 2, US"> ", log_address);
839
840 if ((log_extra_selector & LX_sender_on_delivery) != 0)
841 s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
842
843 /* You might think that the return path must always be set for a successful
844 delivery; indeed, I did for some time, until this statement crashed. The case
845 when it is not set is for a delivery to /dev/null which is optimised by not
846 being run at all. */
847
848 if (used_return_path != NULL &&
849 (log_extra_selector & LX_return_path_on_delivery) != 0)
850 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
851
852 /* For a delivery from a system filter, there may not be a router */
853
854 if (addr->router != NULL)
855 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
856
857 s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
858
859 if ((log_extra_selector & LX_delivery_size) != 0)
860 s = string_append(s, &size, &ptr, 2, US" S=",
861 string_sprintf("%d", transport_count));
862
863 /* Local delivery */
864
865 if (addr->transport->info->local)
866 {
867 if (addr->host_list != NULL)
868 s = string_append(s, &size, &ptr, 2, US" H=", addr->host_list->name);
869 if (addr->shadow_message != NULL)
870 s = string_cat(s, &size, &ptr, addr->shadow_message,
871 Ustrlen(addr->shadow_message));
872 }
873
874 /* Remote delivery */
875
876 else
877 {
878 if (addr->host_used != NULL)
879 {
880 s = string_append(s, &size, &ptr, 5, US" H=", addr->host_used->name,
881 US" [", addr->host_used->address, US"]");
882 if ((log_extra_selector & LX_outgoing_port) != 0)
883 s = string_append(s, &size, &ptr, 2, US":", string_sprintf("%d",
884 addr->host_used->port));
885 if (continue_sequence > 1)
886 s = string_cat(s, &size, &ptr, US"*", 1);
887 }
888
889 #ifdef SUPPORT_TLS
890 if ((log_extra_selector & LX_tls_cipher) != 0 && addr->cipher != NULL)
891 s = string_append(s, &size, &ptr, 2, US" X=", addr->cipher);
892 if ((log_extra_selector & LX_tls_certificate_verified) != 0 &&
893 addr->cipher != NULL)
894 s = string_append(s, &size, &ptr, 2, US" CV=",
895 testflag(addr, af_cert_verified)? "yes":"no");
896 if ((log_extra_selector & LX_tls_peerdn) != 0 && addr->peerdn != NULL)
897 s = string_append(s, &size, &ptr, 3, US" DN=\"", addr->peerdn, US"\"");
898 #endif
899
900 if ((log_extra_selector & LX_smtp_confirmation) != 0 &&
901 addr->message != NULL)
902 {
903 int i;
904 uschar *p = big_buffer;
905 uschar *ss = addr->message;
906 *p++ = '\"';
907 for (i = 0; i < 100 && ss[i] != 0; i++)
908 {
909 if (ss[i] == '\"' || ss[i] == '\\') *p++ = '\\';
910 *p++ = ss[i];
911 }
912 *p++ = '\"';
913 *p = 0;
914 s = string_append(s, &size, &ptr, 2, US" C=", big_buffer);
915 }
916 }
917
918 /* Time on queue and actual time taken to deliver */
919
920 if ((log_extra_selector & LX_queue_time) != 0)
921 {
922 s = string_append(s, &size, &ptr, 2, US" QT=",
923 readconf_printtime(time(NULL) - received_time));
924 }
925
926 if ((log_extra_selector & LX_deliver_time) != 0)
927 {
928 s = string_append(s, &size, &ptr, 2, US" DT=",
929 readconf_printtime(addr->more_errno));
930 }
931
932 /* string_cat() always leaves room for the terminator. Release the
933 store we used to build the line after writing it. */
934
935 s[ptr] = 0;
936 log_write(0, LOG_MAIN, "%s", s);
937 store_reset(reset_point);
938 }
939
940
941 /* Soft failure, or local delivery process failed; freezing may be
942 requested. */
943
944 else if (result == DEFER || result == PANIC)
945 {
946 if (result == PANIC) logflags |= LOG_PANIC;
947
948 /* This puts them on the chain in reverse order. Do not change this, because
949 the code for handling retries assumes that the one with the retry
950 information is last. */
951
952 addr->next = addr_defer;
953 addr_defer = addr;
954
955 /* The only currently implemented special action is to freeze the
956 message. Logging of this is done later, just before the -H file is
957 updated. */
958
959 if (addr->special_action == SPECIAL_FREEZE)
960 {
961 deliver_freeze = TRUE;
962 deliver_frozen_at = time(NULL);
963 update_spool = TRUE;
964 }
965
966 /* If doing a 2-stage queue run, we skip writing to either the message
967 log or the main log for SMTP defers. */
968
969 if (!queue_2stage || addr->basic_errno != 0)
970 {
971 uschar ss[32];
972
973 /* For errors of the type "retry time not reached" (also remotes skipped
974 on queue run), logging is controlled by L_retry_defer. Note that this kind
975 of error number is negative, and all the retry ones are less than any
976 others. */
977
978 unsigned int use_log_selector = (addr->basic_errno <= ERRNO_RETRY_BASE)?
979 L_retry_defer : 0;
980
981 /* Build up the line that is used for both the message log and the main
982 log. */
983
984 s = reset_point = store_get(size);
985 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
986
987 /* Either driver_name contains something and driver_kind contains
988 " router" or " transport" (note the leading space), or driver_name is
989 a null string and driver_kind contains "routing" without the leading
990 space, if all routing has been deferred. When a domain has been held,
991 so nothing has been done at all, both variables contain null strings. */
992
993 if (driver_name == NULL)
994 {
995 if (driver_kind != NULL)
996 s = string_append(s, &size, &ptr, 2, US" ", driver_kind);
997 }
998 else
999 {
1000 if (driver_kind[1] == 't' && addr->router != NULL)
1001 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
1002 Ustrcpy(ss, " ?=");
1003 ss[1] = toupper(driver_kind[1]);
1004 s = string_append(s, &size, &ptr, 2, ss, driver_name);
1005 }
1006
1007 sprintf(CS ss, " defer (%d)", addr->basic_errno);
1008 s = string_cat(s, &size, &ptr, ss, Ustrlen(ss));
1009
1010 if (addr->basic_errno > 0)
1011 s = string_append(s, &size, &ptr, 2, US": ",
1012 US strerror(addr->basic_errno));
1013
1014 if (addr->message != NULL)
1015 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1016
1017 s[ptr] = 0;
1018
1019 /* Log the deferment in the message log, but don't clutter it
1020 up with retry-time defers after the first delivery attempt. */
1021
1022 if (deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE)
1023 deliver_msglog("%s %s\n", now, s);
1024
1025 /* Write the main log and reset the store */
1026
1027 log_write(use_log_selector, logflags, "== %s", s);
1028 store_reset(reset_point);
1029 }
1030 }
1031
1032
1033 /* Hard failure. If there is an address to which an error message can be sent,
1034 put this address on the failed list. If not, put it on the deferred list and
1035 freeze the mail message for human attention. The latter action can also be
1036 explicitly requested by a router or transport. */
1037
1038 else
1039 {
1040 /* If this is a delivery error, or a message for which no replies are
1041 wanted, and the message's age is greater than ignore_bounce_errors_after,
1042 force the af_ignore_error flag. This will cause the address to be discarded
1043 later (with a log entry). */
1044
1045 if (sender_address[0] == 0 && message_age >= ignore_bounce_errors_after)
1046 setflag(addr, af_ignore_error);
1047
1048 /* Freeze the message if requested, or if this is a bounce message (or other
1049 message with null sender) and this address does not have its own errors
1050 address. However, don't freeze if errors are being ignored. The actual code
1051 to ignore occurs later, instead of sending a message. Logging of freezing
1052 occurs later, just before writing the -H file. */
1053
1054 if (!testflag(addr, af_ignore_error) &&
1055 (addr->special_action == SPECIAL_FREEZE ||
1056 (sender_address[0] == 0 && addr->p.errors_address == NULL)
1057 ))
1058 {
1059 frozen_info = (addr->special_action == SPECIAL_FREEZE)? US"" :
1060 (sender_local && !local_error_message)?
1061 US" (message created with -f <>)" : US" (delivery error message)";
1062 deliver_freeze = TRUE;
1063 deliver_frozen_at = time(NULL);
1064 update_spool = TRUE;
1065
1066 /* The address is put on the defer rather than the failed queue, because
1067 the message is being retained. */
1068
1069 addr->next = addr_defer;
1070 addr_defer = addr;
1071 }
1072
1073 /* Don't put the address on the nonrecipients tree yet; wait until an
1074 error message has been successfully sent. */
1075
1076 else
1077 {
1078 addr->next = addr_failed;
1079 addr_failed = addr;
1080 }
1081
1082 /* Build up the log line for the message and main logs */
1083
1084 s = reset_point = store_get(size);
1085 s = string_cat(s, &size, &ptr, log_address, Ustrlen(log_address));
1086
1087 if ((log_extra_selector & LX_sender_on_delivery) != 0)
1088 s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
1089
1090 /* Return path may not be set if no delivery actually happened */
1091
1092 if (used_return_path != NULL &&
1093 (log_extra_selector & LX_return_path_on_delivery) != 0)
1094 {
1095 s = string_append(s, &size, &ptr, 3, US" P=<", used_return_path, US">");
1096 }
1097
1098 if (addr->router != NULL)
1099 s = string_append(s, &size, &ptr, 2, US" R=", addr->router->name);
1100 if (addr->transport != NULL)
1101 s = string_append(s, &size, &ptr, 2, US" T=", addr->transport->name);
1102
1103 if (addr->host_used != NULL)
1104 s = string_append(s, &size, &ptr, 5, US" H=", addr->host_used->name,
1105 US" [", addr->host_used->address, US"]");
1106
1107 if (addr->basic_errno > 0)
1108 s = string_append(s, &size, &ptr, 2, US": ",
1109 US strerror(addr->basic_errno));
1110
1111 if (addr->message != NULL)
1112 s = string_append(s, &size, &ptr, 2, US": ", addr->message);
1113
1114 s[ptr] = 0;
1115
1116 /* Do the logging. For the message log, "routing failed" for those cases,
1117 just to make it clearer. */
1118
1119 if (driver_name == NULL)
1120 deliver_msglog("%s %s failed for %s\n", now, driver_kind, s);
1121 else
1122 deliver_msglog("%s %s\n", now, s);
1123
1124 log_write(0, LOG_MAIN, "** %s", s);
1125 store_reset(reset_point);
1126 }
1127
1128 /* Ensure logging is turned on again in all cases */
1129
1130 disable_logging = FALSE;
1131 }
1132
1133
1134
1135
1136 /*************************************************
1137 * Address-independent error *
1138 *************************************************/
1139
1140 /* This function is called when there's an error that is not dependent on a
1141 particular address, such as an expansion string failure. It puts the error into
1142 all the addresses in a batch, logs the incident on the main and panic logs, and
1143 clears the expansions. It is mostly called from local_deliver(), but can be
1144 called for a remote delivery via findugid().
1145
1146 Arguments:
1147 logit TRUE if (MAIN+PANIC) logging required
1148 addr the first of the chain of addresses
1149 code the error code
1150 format format string for error message, or NULL if already set in addr
1151 ... arguments for the format
1152
1153 Returns: nothing
1154 */
1155
1156 static void
1157 common_error(BOOL logit, address_item *addr, int code, uschar *format, ...)
1158 {
1159 address_item *addr2;
1160 addr->basic_errno = code;
1161
1162 if (format != NULL)
1163 {
1164 va_list ap;
1165 uschar buffer[512];
1166 va_start(ap, format);
1167 if (!string_vformat(buffer, sizeof(buffer), CS format, ap))
1168 log_write(0, LOG_MAIN|LOG_PANIC_DIE,
1169 "common_error expansion was longer than %d", sizeof(buffer));
1170 va_end(ap);
1171 addr->message = string_copy(buffer);
1172 }
1173
1174 for (addr2 = addr->next; addr2 != NULL; addr2 = addr2->next)
1175 {
1176 addr2->basic_errno = code;
1177 addr2->message = addr->message;
1178 }
1179
1180 if (logit) log_write(0, LOG_MAIN|LOG_PANIC, "%s", addr->message);
1181 deliver_set_expansions(NULL);
1182 }
1183
1184
1185
1186
1187 /*************************************************
1188 * Check a "never users" list *
1189 *************************************************/
1190
1191 /* This function is called to check whether a uid is on one of the two "never
1192 users" lists.
1193
1194 Arguments:
1195 uid the uid to be checked
1196 nusers the list to be scanned; the first item in the list is the count
1197
1198 Returns: TRUE if the uid is on the list
1199 */
1200
1201 static BOOL
1202 check_never_users(uid_t uid, uid_t *nusers)
1203 {
1204 int i;
1205 if (nusers == NULL) return FALSE;
1206 for (i = 1; i <= (int)(nusers[0]); i++) if (nusers[i] == uid) return TRUE;
1207 return FALSE;
1208 }
1209
1210
1211
1212 /*************************************************
1213 * Find uid and gid for a transport *
1214 *************************************************/
1215
1216 /* This function is called for both local and remote deliveries, to find the
1217 uid/gid under which to run the delivery. The values are taken preferentially
1218 from the transport (either explicit or deliver_as_creator), then from the
1219 address (i.e. the router), and if nothing is set, the exim uid/gid are used. If
1220 the resulting uid is on the "never_users" or the "fixed_never_users" list, a
1221 panic error is logged, and the function fails (which normally leads to delivery
1222 deferral).
1223
1224 Arguments:
1225 addr the address (possibly a chain)
1226 tp the transport
1227 uidp pointer to uid field
1228 gidp pointer to gid field
1229 igfp pointer to the use_initgroups field
1230
1231 Returns: FALSE if failed - error has been set in address(es)
1232 */
1233
1234 static BOOL
1235 findugid(address_item *addr, transport_instance *tp, uid_t *uidp, gid_t *gidp,
1236 BOOL *igfp)
1237 {
1238 uschar *nuname = NULL;
1239 BOOL gid_set = FALSE;
1240
1241 /* Default initgroups flag comes from the transport */
1242
1243 *igfp = tp->initgroups;
1244
1245 /* First see if there's a gid on the transport, either fixed or expandable.
1246 The expanding function always logs failure itself. */
1247
1248 if (tp->gid_set)
1249 {
1250 *gidp = tp->gid;
1251 gid_set = TRUE;
1252 }
1253 else if (tp->expand_gid != NULL)
1254 {
1255 if (route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp,
1256 &(addr->message))) gid_set = TRUE;
1257 else
1258 {
1259 common_error(FALSE, addr, ERRNO_GIDFAIL, NULL);
1260 return FALSE;
1261 }
1262 }
1263
1264 /* Pick up a uid from the transport if one is set. */
1265
1266 if (tp->uid_set) *uidp = tp->uid;
1267
1268 /* Otherwise, try for an expandable uid field. If it ends up as a numeric id,
1269 it does not provide a passwd value from which a gid can be taken. */
1270
1271 else if (tp->expand_uid != NULL)
1272 {
1273 struct passwd *pw;
1274 if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw,
1275 uidp, &(addr->message)))
1276 {
1277 common_error(FALSE, addr, ERRNO_UIDFAIL, NULL);
1278 return FALSE;
1279 }
1280 if (!gid_set && pw != NULL)
1281 {
1282 *gidp = pw->pw_gid;
1283 gid_set = TRUE;
1284 }
1285 }
1286
1287 /* If the transport doesn't set the uid, test the deliver_as_creator flag. */
1288
1289 else if (tp->deliver_as_creator)
1290 {
1291 *uidp = originator_uid;
1292 if (!gid_set)
1293 {
1294 *gidp = originator_gid;
1295 gid_set = TRUE;
1296 }
1297 }
1298
1299 /* Otherwise see if the address specifies the uid and if so, take its
1300 initgroups flag. The gid from the address is taken only if the transport hasn't
1301 set it. In other words, a gid on the transport overrides the gid on the
1302 address. */
1303
1304 else if (testflag(addr, af_uid_set))
1305 {
1306 *uidp = addr->uid;
1307 *igfp = testflag(addr, af_initgroups);
1308 if (!gid_set)
1309 {
1310 *gidp = addr->gid;
1311 gid_set = TRUE;
1312 }
1313 }
1314
1315 /* Nothing has specified the uid - default to the Exim user, and group if the
1316 gid is not set. */
1317
1318 else
1319 {
1320 *uidp = exim_uid;
1321 if (!gid_set)
1322 {
1323 *gidp = exim_gid;
1324 gid_set = TRUE;
1325 }
1326 }
1327
1328 /* If no gid is set, it is a disaster. */
1329
1330 if (!gid_set)
1331 {
1332 common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for "
1333 "%s transport", tp->name);
1334 return FALSE;
1335 }
1336
1337 /* Check that the uid is not on the lists of banned uids that may not be used
1338 for delivery processes. */
1339
1340 if (check_never_users(*uidp, never_users))
1341 nuname = US"never_users";
1342 else if (check_never_users(*uidp, fixed_never_users))
1343 nuname = US"fixed_never_users";
1344
1345 if (nuname != NULL)
1346 {
1347 common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport "
1348 "is on the %s list", (long int)(*uidp), tp->name, nuname);
1349 return FALSE;
1350 }
1351
1352 /* All is well */
1353
1354 return TRUE;
1355 }
1356
1357
1358
1359
1360 /*************************************************
1361 * Check the size of a message for a transport *
1362 *************************************************/
1363
1364 /* Checks that the message isn't too big for the selected transport.
1365 This is called only when it is known that the limit is set.
1366
1367 Arguments:
1368 tp the transport
1369 addr the (first) address being delivered
1370
1371 Returns: OK
1372 DEFER expansion failed or did not yield an integer
1373 FAIL message too big
1374 */
1375
1376 int
1377 check_message_size(transport_instance *tp, address_item *addr)
1378 {
1379 int rc = OK;
1380 int size_limit;
1381
1382 deliver_set_expansions(addr);
1383 size_limit = expand_string_integer(tp->message_size_limit);
1384 deliver_set_expansions(NULL);
1385
1386 if (size_limit < 0)
1387 {
1388 rc = DEFER;
1389 if (size_limit == -1)
1390 addr->message = string_sprintf("failed to expand message_size_limit "
1391 "in %s transport: %s", tp->name, expand_string_message);
1392 else
1393 addr->message = string_sprintf("invalid message_size_limit "
1394 "in %s transport: %s", tp->name, expand_string_message);
1395 }
1396 else if (size_limit > 0 && message_size > size_limit)
1397 {
1398 rc = FAIL;
1399 addr->message =
1400 string_sprintf("message is too big (transport limit = %d)",
1401 size_limit);
1402 }
1403
1404 return rc;
1405 }
1406
1407
1408
1409 /*************************************************
1410 * Transport-time check for a previous delivery *
1411 *************************************************/
1412
1413 /* Check that this base address hasn't previously been delivered to its routed
1414 transport. The check is necessary at delivery time in order to handle homonymic
1415 addresses correctly in cases where the pattern of redirection changes between
1416 delivery attempts (so the unique fields change). Non-homonymic previous
1417 delivery is detected earlier, at routing time (which saves unnecessary
1418 routing).
1419
1420 Argument: the address item
1421 Returns: TRUE if previously delivered by the transport
1422 */
1423
1424 static BOOL
1425 previously_transported(address_item *addr)
1426 {
1427 (void)string_format(big_buffer, big_buffer_size, "%s/%s",
1428 addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name);
1429
1430 if (tree_search(tree_nonrecipients, big_buffer) != 0)
1431 {
1432 DEBUG(D_deliver|D_route|D_transport)
1433 debug_printf("%s was previously delivered (%s transport): discarded\n",
1434 addr->address, addr->transport->name);
1435 child_done(addr, tod_stamp(tod_log));
1436 return TRUE;
1437 }
1438
1439 return FALSE;
1440 }
1441
1442
1443
1444
1445 /*************************************************
1446 * Perform a local delivery *
1447 *************************************************/
1448
1449 /* Each local delivery is performed in a separate process which sets its
1450 uid and gid as specified. This is a safer way than simply changing and
1451 restoring using seteuid(); there is a body of opinion that seteuid() cannot be
1452 used safely. From release 4, Exim no longer makes any use of it. Besides, not
1453 all systems have seteuid().
1454
1455 If the uid/gid are specified in the transport_instance, they are used; the
1456 transport initialization must ensure that either both or neither are set.
1457 Otherwise, the values associated with the address are used. If neither are set,
1458 it is a configuration error.
1459
1460 The transport or the address may specify a home directory (transport over-
1461 rides), and if they do, this is set as $home. If neither have set a working
1462 directory, this value is used for that as well. Otherwise $home is left unset
1463 and the cwd is set to "/" - a directory that should be accessible to all users.
1464
1465 Using a separate process makes it more complicated to get error information
1466 back. We use a pipe to pass the return code and also an error code and error
1467 text string back to the parent process.
1468
1469 Arguments:
1470 addr points to an address block for this delivery; for "normal" local
1471 deliveries this is the only address to be delivered, but for
1472 pseudo-remote deliveries (e.g. by batch SMTP to a file or pipe)
1473 a number of addresses can be handled simultaneously, and in this
1474 case addr will point to a chain of addresses with the same
1475 characteristics.
1476
1477 shadowing TRUE if running a shadow transport; this causes output from pipes
1478 to be ignored.
1479
1480 Returns: nothing
1481 */
1482
1483 static void
1484 deliver_local(address_item *addr, BOOL shadowing)
1485 {
1486 BOOL use_initgroups;
1487 uid_t uid;
1488 gid_t gid;
1489 int status, len, rc;
1490 int pfd[2];
1491 pid_t pid;
1492 uschar *working_directory;
1493 address_item *addr2;
1494 transport_instance *tp = addr->transport;
1495
1496 /* Set up the return path from the errors or sender address. If the transport
1497 has its own return path setting, expand it and replace the existing value. */
1498
1499 return_path = (addr->p.errors_address != NULL)?
1500 addr->p.errors_address : sender_address;
1501
1502 if (tp->return_path != NULL)
1503 {
1504 uschar *new_return_path = expand_string(tp->return_path);
1505 if (new_return_path == NULL)
1506 {
1507 if (!expand_string_forcedfail)
1508 {
1509 common_error(TRUE, addr, ERRNO_EXPANDFAIL,
1510 US"Failed to expand return path \"%s\" in %s transport: %s",
1511 tp->return_path, tp->name, expand_string_message);
1512 return;
1513 }
1514 }
1515 else return_path = new_return_path;
1516 }
1517
1518 /* For local deliveries, one at a time, the value used for logging can just be
1519 set directly, once and for all. */
1520
1521 used_return_path = return_path;
1522
1523 /* Sort out the uid, gid, and initgroups flag. If an error occurs, the message
1524 gets put into the address(es), and the expansions are unset, so we can just
1525 return. */
1526
1527 if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
1528
1529 /* See if either the transport or the address specifies a home and/or a current
1530 working directory. Expand it if necessary. If nothing is set, use "/", for the
1531 working directory, which is assumed to be a directory to which all users have
1532 access. It is necessary to be in a visible directory for some operating systems
1533 when running pipes, as some commands (e.g. "rm" under Solaris 2.5) require
1534 this. */
1535
1536 deliver_home = (tp->home_dir != NULL)? tp->home_dir :
1537 (addr->home_dir != NULL)? addr->home_dir : NULL;
1538
1539 if (deliver_home != NULL && !testflag(addr, af_home_expanded))
1540 {
1541 uschar *rawhome = deliver_home;
1542 deliver_home = NULL; /* in case it contains $home */
1543 deliver_home = expand_string(rawhome);
1544 if (deliver_home == NULL)
1545 {
1546 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed "
1547 "to expand for %s transport: %s", rawhome, tp->name,
1548 expand_string_message);
1549 return;
1550 }
1551 if (*deliver_home != '/')
1552 {
1553 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" "
1554 "is not absolute for %s transport", deliver_home, tp->name);
1555 return;
1556 }
1557 }
1558
1559 working_directory = (tp->current_dir != NULL)? tp->current_dir :
1560 (addr->current_dir != NULL)? addr->current_dir : NULL;
1561
1562 if (working_directory != NULL)
1563 {
1564 uschar *raw = working_directory;
1565 working_directory = expand_string(raw);
1566 if (working_directory == NULL)
1567 {
1568 common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" "
1569 "failed to expand for %s transport: %s", raw, tp->name,
1570 expand_string_message);
1571 return;
1572 }
1573 if (*working_directory != '/')
1574 {
1575 common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path "
1576 "\"%s\" is not absolute for %s transport", working_directory, tp->name);
1577 return;
1578 }
1579 }
1580 else working_directory = (deliver_home == NULL)? US"/" : deliver_home;
1581
1582 /* If one of the return_output flags is set on the transport, create and open a
1583 file in the message log directory for the transport to write its output onto.
1584 This is mainly used by pipe transports. The file needs to be unique to the
1585 address. This feature is not available for shadow transports. */
1586
1587 if (!shadowing && (tp->return_output || tp->return_fail_output ||
1588 tp->log_output || tp->log_fail_output))
1589 {
1590 uschar *error;
1591 addr->return_filename =
1592 string_sprintf("%s/msglog/%s/%s-%d-%d", spool_directory, message_subdir,
1593 message_id, getpid(), return_count++);
1594 addr->return_file = open_msglog_file(addr->return_filename, 0400, &error);
1595 if (addr->return_file < 0)
1596 {
1597 common_error(TRUE, addr, errno, US"Unable to %s file for %s transport "
1598 "to return message: %s", error, tp->name, strerror(errno));
1599 return;
1600 }
1601 }
1602
1603 /* Create the pipe for inter-process communication. */
1604
1605 if (pipe(pfd) != 0)
1606 {
1607 common_error(TRUE, addr, ERRNO_PIPEFAIL, US"Creation of pipe failed: %s",
1608 strerror(errno));
1609 return;
1610 }
1611
1612 /* Now fork the process to do the real work in the subprocess, but first
1613 ensure that all cached resources are freed so that the subprocess starts with
1614 a clean slate and doesn't interfere with the parent process. */
1615
1616 search_tidyup();
1617
1618 if ((pid = fork()) == 0)
1619 {
1620 BOOL replicate = TRUE;
1621
1622 /* Prevent core dumps, as we don't want them in users' home directories.
1623 HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
1624 system. Some experimental/developing systems (e.g. GNU/Hurd) may define
1625 RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
1626 complain if the error is "not supported". */
1627
1628 #ifdef RLIMIT_CORE
1629 struct rlimit rl;
1630 rl.rlim_cur = 0;
1631 rl.rlim_max = 0;
1632 if (setrlimit(RLIMIT_CORE, &rl) < 0)
1633 {
1634 #ifdef SETRLIMIT_NOT_SUPPORTED
1635 if (errno != ENOSYS && errno != ENOTSUP)
1636 #endif
1637 log_write(0, LOG_MAIN|LOG_PANIC, "setrlimit(RLIMIT_CORE) failed: %s",
1638 strerror(errno));
1639 }
1640 #endif
1641
1642 /* Reset the random number generator, so different processes don't all
1643 have the same sequence. */
1644
1645 random_seed = 0;
1646
1647 /* If the transport has a setup entry, call this first, while still
1648 privileged. (Appendfile uses this to expand quota, for example, while
1649 able to read private files.) */
1650
1651 if (addr->transport->setup != NULL)
1652 {
1653 switch((addr->transport->setup)(addr->transport, addr, NULL,
1654 &(addr->message)))
1655 {
1656 case DEFER:
1657 addr->transport_return = DEFER;
1658 goto PASS_BACK;
1659
1660 case FAIL:
1661 addr->transport_return = PANIC;
1662 goto PASS_BACK;
1663 }
1664 }
1665
1666 /* Ignore SIGINT and SIGTERM during delivery. Also ignore SIGUSR1, as
1667 when the process becomes unprivileged, it won't be able to write to the
1668 process log. SIGHUP is ignored throughout exim, except when it is being
1669 run as a daemon. */
1670
1671 signal(SIGINT, SIG_IGN);
1672 signal(SIGTERM, SIG_IGN);
1673 signal(SIGUSR1, SIG_IGN);
1674
1675 /* Close the unwanted half of the pipe, and set close-on-exec for the other
1676 half - for transports that exec things (e.g. pipe). Then set the required
1677 gid/uid. */
1678
1679 close(pfd[pipe_read]);
1680 fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
1681 FD_CLOEXEC);
1682 exim_setugid(uid, gid, use_initgroups,
1683 string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
1684 addr->address, addr->transport->name));
1685
1686 DEBUG(D_deliver)
1687 {
1688 address_item *batched;
1689 debug_printf(" home=%s current=%s\n", deliver_home, working_directory);
1690 for (batched = addr->next; batched != NULL; batched = batched->next)
1691 debug_printf("additional batched address: %s\n", batched->address);
1692 }
1693
1694 /* Set an appropriate working directory. */
1695
1696 if (Uchdir(working_directory) < 0)
1697 {
1698 addr->transport_return = DEFER;
1699 addr->basic_errno = errno;
1700 addr->message = string_sprintf("failed to chdir to %s", working_directory);
1701 }
1702
1703 /* If successful, call the transport */
1704
1705 else
1706 {
1707 BOOL ok = TRUE;
1708 set_process_info("delivering %s to %s using %s", message_id,
1709 addr->local_part, addr->transport->name);
1710
1711 /* If a transport filter has been specified, set up its argument list.
1712 Any errors will get put into the address, and FALSE yielded. */
1713
1714 if (addr->transport->filter_command != NULL)
1715 {
1716 ok = transport_set_up_command(&transport_filter_argv,
1717 addr->transport->filter_command,
1718 TRUE, PANIC, addr, US"transport filter", NULL);
1719 transport_filter_timeout = addr->transport->filter_timeout;
1720 }
1721 else transport_filter_argv = NULL;
1722
1723 if (ok)
1724 {
1725 debug_print_string(addr->transport->debug_string);
1726 replicate = !(addr->transport->info->code)(addr->transport, addr);
1727 }
1728 }
1729
1730 /* Pass the results back down the pipe. If necessary, first replicate the
1731 status in the top address to the others in the batch. The label is the
1732 subject of a goto when a call to the transport's setup function fails. We
1733 pass the pointer to the transport back in case it got changed as a result of
1734 file_format in appendfile. */
1735
1736 PASS_BACK:
1737
1738 if (replicate) replicate_status(addr);
1739 for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
1740 {
1741 int i;
1742 int local_part_length = Ustrlen(addr2->local_part);
1743 uschar *s;
1744
1745 write(pfd[pipe_write], (void *)&(addr2->transport_return), sizeof(int));
1746 write(pfd[pipe_write], (void *)&transport_count, sizeof(transport_count));
1747 write(pfd[pipe_write], (void *)&(addr2->flags), sizeof(addr2->flags));
1748 write(pfd[pipe_write], (void *)&(addr2->basic_errno), sizeof(int));
1749 write(pfd[pipe_write], (void *)&(addr2->more_errno), sizeof(int));
1750 write(pfd[pipe_write], (void *)&(addr2->special_action), sizeof(int));
1751 write(pfd[pipe_write], (void *)&(addr2->transport),
1752 sizeof(transport_instance *));
1753
1754 /* For a file delivery, pass back the local part, in case the original
1755 was only part of the final delivery path. This gives more complete
1756 logging. */
1757
1758 if (testflag(addr2, af_file))
1759 {
1760 write(pfd[pipe_write], (void *)&local_part_length, sizeof(int));
1761 write(pfd[pipe_write], addr2->local_part, local_part_length);
1762 }
1763
1764 /* Now any messages */
1765
1766 for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
1767 {
1768 int message_length = (s == NULL)? 0 : Ustrlen(s) + 1;
1769 write(pfd[pipe_write], (void *)&message_length, sizeof(int));
1770 if (message_length > 0) write(pfd[pipe_write], s, message_length);
1771 }
1772 }
1773
1774 /* OK, this process is now done. Free any cached resources that it opened,
1775 and close the pipe we were writing down before exiting. */
1776
1777 close(pfd[pipe_write]);
1778 search_tidyup();
1779 exit(EXIT_SUCCESS);
1780 }
1781
1782 /* Back in the main process: panic if the fork did not succeed. This seems
1783 better than returning an error - if forking is failing it is probably best
1784 not to try other deliveries for this message. */
1785
1786 if (pid < 0)
1787 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Fork failed for local delivery to %s",
1788 addr->address);
1789
1790 /* Read the pipe to get the delivery status codes and error messages. Our copy
1791 of the writing end must be closed first, as otherwise read() won't return zero
1792 on an empty pipe. We check that a status exists for each address before
1793 overwriting the address structure. If data is missing, the default DEFER status
1794 will remain. Afterwards, close the reading end. */
1795
1796 close(pfd[pipe_write]);
1797
1798 for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
1799 {
1800 len = read(pfd[pipe_read], (void *)&status, sizeof(int));
1801 if (len > 0)
1802 {
1803 int i;
1804 uschar **sptr;
1805
1806 addr2->transport_return = status;
1807 len = read(pfd[pipe_read], (void *)&transport_count,
1808 sizeof(transport_count));
1809 len = read(pfd[pipe_read], (void *)&(addr2->flags), sizeof(addr2->flags));
1810 len = read(pfd[pipe_read], (void *)&(addr2->basic_errno), sizeof(int));
1811 len = read(pfd[pipe_read], (void *)&(addr2->more_errno), sizeof(int));
1812 len = read(pfd[pipe_read], (void *)&(addr2->special_action), sizeof(int));
1813 len = read(pfd[pipe_read], (void *)&(addr2->transport),
1814 sizeof(transport_instance *));
1815
1816 if (testflag(addr2, af_file))
1817 {
1818 int local_part_length;
1819 len = read(pfd[pipe_read], (void *)&local_part_length, sizeof(int));
1820 len = read(pfd[pipe_read], (void *)big_buffer, local_part_length);
1821 big_buffer[local_part_length] = 0;
1822 addr2->local_part = string_copy(big_buffer);
1823 }
1824
1825 for (i = 0, sptr = &(addr2->message); i < 2;
1826 i++, sptr = &(addr2->user_message))
1827 {
1828 int message_length;
1829 len = read(pfd[pipe_read], (void *)&message_length, sizeof(int));
1830 if (message_length > 0)
1831 {
1832 len = read(pfd[pipe_read], (void *)big_buffer, message_length);
1833 if (len > 0) *sptr = string_copy(big_buffer);
1834 }
1835 }
1836 }
1837
1838 else
1839 {
1840 log_write(0, LOG_MAIN|LOG_PANIC, "failed to read delivery status for %s "
1841 "from delivery subprocess", addr2->unique);
1842 break;
1843 }
1844 }
1845
1846 close(pfd[pipe_read]);
1847
1848 /* Unless shadowing, write all successful addresses immediately to the journal
1849 file, to ensure they are recorded asap. For homonymic addresses, use the base
1850 address plus the transport name. Failure to write the journal is panic-worthy,
1851 but don't stop, as it may prove possible subsequently to update the spool file
1852 in order to record the delivery. */
1853
1854 if (!shadowing)
1855 {
1856 for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
1857 {
1858 if (addr2->transport_return != OK) continue;
1859
1860 if (testflag(addr2, af_homonym))
1861 sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name);
1862 else
1863 sprintf(CS big_buffer, "%.500s\n", addr2->unique);
1864
1865 /* In the test harness, wait just a bit to let the subprocess finish off
1866 any debug output etc first. */
1867
1868 if (running_in_test_harness) millisleep(300);
1869
1870 DEBUG(D_deliver) debug_printf("journalling %s", big_buffer);
1871 len = Ustrlen(big_buffer);
1872 if (write(journal_fd, big_buffer, len) != len)
1873 log_write(0, LOG_MAIN|LOG_PANIC, "failed to update journal for %s: %s",
1874 big_buffer, strerror(errno));
1875 }
1876
1877 /* Ensure the journal file is pushed out to disk. */
1878
1879 if (fsync(journal_fd) < 0)
1880 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
1881 strerror(errno));
1882 }
1883
1884 /* Wait for the process to finish. If it terminates with a non-zero code,
1885 freeze the message (except for SIGTERM, SIGKILL and SIGQUIT), but leave the
1886 status values of all the addresses as they are. Take care to handle the case
1887 when the subprocess doesn't seem to exist. This has been seen on one system
1888 when Exim was called from an MUA that set SIGCHLD to SIG_IGN. When that
1889 happens, wait() doesn't recognize the termination of child processes. Exim now
1890 resets SIGCHLD to SIG_DFL, but this code should still be robust. */
1891
1892 while ((rc = wait(&status)) != pid)
1893 {
1894 if (rc < 0 && errno == ECHILD) /* Process has vanished */
1895 {
1896 log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly",
1897 addr->transport->driver_name);
1898 status = 0;
1899 break;
1900 }
1901 }
1902
1903 if ((status & 0xffff) != 0)
1904 {
1905 int msb = (status >> 8) & 255;
1906 int lsb = status & 255;
1907 int code = (msb == 0)? (lsb & 0x7f) : msb;
1908 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
1909 addr->special_action = SPECIAL_FREEZE;
1910 log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero "
1911 "status 0x%04x: %s %d",
1912 addr->transport->driver_name,
1913 status,
1914 (msb == 0)? "terminated by signal" : "exit code",
1915 code);
1916 }
1917
1918 /* If SPECIAL_WARN is set in the top address, send a warning message. */
1919
1920 if (addr->special_action == SPECIAL_WARN &&
1921 addr->transport->warn_message != NULL)
1922 {
1923 int fd;
1924 uschar *warn_message;
1925
1926 DEBUG(D_deliver) debug_printf("Warning message requested by transport\n");
1927
1928 warn_message = expand_string(addr->transport->warn_message);
1929 if (warn_message == NULL)
1930 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning "
1931 "message for %s transport): %s", addr->transport->warn_message,
1932 addr->transport->name, expand_string_message);
1933 else
1934 {
1935 pid_t pid = child_open_exim(&fd);
1936 if (pid > 0)
1937 {
1938 FILE *f = fdopen(fd, "wb");
1939
1940 if (errors_reply_to != NULL)
1941 fprintf(f, "Reply-To: %s\n", errors_reply_to);
1942 fprintf(f, "Auto-Submitted: auto-generated\n");
1943 fprintf(f, "From: Mail Delivery System <Mailer-Daemon@%s>\n",
1944 qualify_domain_sender);
1945 fprintf(f, "%s", CS warn_message);
1946
1947 /* Close and wait for child process to complete, without a timeout. */
1948
1949 fclose(f);
1950 (void)child_close(pid, 0);
1951 }
1952 }
1953
1954 addr->special_action = SPECIAL_NONE;
1955 }
1956 }
1957
1958
1959
1960 /*************************************************
1961 * Do local deliveries *
1962 *************************************************/
1963
1964 /* This function processes the list of addresses in addr_local. True local
1965 deliveries are always done one address at a time. However, local deliveries can
1966 be batched up in some cases. Typically this is when writing batched SMTP output
1967 files for use by some external transport mechanism, or when running local
1968 deliveries over LMTP.
1969
1970 Arguments: None
1971 Returns: Nothing
1972 */
1973
1974 static void
1975 do_local_deliveries(void)
1976 {
1977 open_db dbblock;
1978 open_db *dbm_file = NULL;
1979 time_t now = time(NULL);
1980
1981 /* Loop until we have exhausted the supply of local deliveries */
1982
1983 while (addr_local != NULL)
1984 {
1985 time_t delivery_start;
1986 int deliver_time;
1987 address_item *addr2, *addr3, *nextaddr;
1988 int logflags = LOG_MAIN;
1989 int logchar = dont_deliver? '*' : '=';
1990 transport_instance *tp;
1991
1992 /* Pick the first undelivered address off the chain */
1993
1994 address_item *addr = addr_local;
1995 addr_local = addr->next;
1996 addr->next = NULL;
1997
1998 DEBUG(D_deliver|D_transport)
1999 debug_printf("--------> %s <--------\n", addr->address);
2000
2001 /* An internal disaster if there is no transport. Should not occur! */
2002
2003 if ((tp = addr->transport) == NULL)
2004 {
2005 logflags |= LOG_PANIC;
2006 disable_logging = FALSE; /* Jic */
2007 addr->message =
2008 (addr->router != NULL)?
2009 string_sprintf("No transport set by %s router", addr->router->name)
2010 :
2011 string_sprintf("No transport set by system filter");
2012 post_process_one(addr, DEFER, logflags, DTYPE_TRANSPORT, 0);
2013 continue;
2014 }
2015
2016 /* Check that this base address hasn't previously been delivered to this
2017 transport. The check is necessary at this point to handle homonymic addresses
2018 correctly in cases where the pattern of redirection changes between delivery
2019 attempts. Non-homonymic previous delivery is detected earlier, at routing
2020 time. */
2021
2022 if (previously_transported(addr)) continue;
2023
2024 /* There are weird cases where logging is disabled */
2025
2026 disable_logging = tp->disable_logging;
2027
2028 /* Check for batched addresses and possible amalgamation. File deliveries can
2029 never be batched. Skip all the work if either batch_max <= 1 or there aren't
2030 any other addresses for local delivery. */
2031
2032 if (!testflag(addr, af_file) && tp->batch_max > 1 && addr_local != NULL)
2033 {
2034 int batch_count = 1;
2035 BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
2036 BOOL uses_lp = readconf_depends((driver_instance *)tp, US"local_part");
2037 uschar *batch_id = NULL;
2038 address_item **anchor = &addr_local;
2039 address_item *last = addr;
2040 address_item *next;
2041
2042 /* Expand the batch_id string for comparison with other addresses.
2043 Expansion failure suppresses batching. */
2044
2045 if (tp->batch_id != NULL)
2046 {
2047 deliver_set_expansions(addr);
2048 batch_id = expand_string(tp->batch_id);
2049 deliver_set_expansions(NULL);
2050 if (batch_id == NULL)
2051 {
2052 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2053 "in %s transport (%s): %s", tp->name, addr->address,
2054 expand_string_message);
2055 batch_count = tp->batch_max;
2056 }
2057 }
2058
2059 /* Until we reach the batch_max limit, pick off addresses which have the
2060 same characteristics. These are:
2061
2062 same transport
2063 same local part if the transport's configuration contains $local_part
2064 same domain if the transport's configuration contains $domain
2065 same errors address
2066 same additional headers
2067 same headers to be removed
2068 same uid/gid for running the transport
2069 same first host if a host list is set
2070 */
2071
2072 while ((next = *anchor) != NULL && batch_count < tp->batch_max)
2073 {
2074 BOOL ok =
2075 tp == next->transport &&
2076 (!uses_lp || Ustrcmp(next->local_part, addr->local_part) == 0) &&
2077 (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0) &&
2078 same_strings(next->p.errors_address, addr->p.errors_address) &&
2079 same_headers(next->p.extra_headers, addr->p.extra_headers) &&
2080 same_strings(next->p.remove_headers, addr->p.remove_headers) &&
2081 same_ugid(tp, addr, next) &&
2082 ((addr->host_list == NULL && next->host_list == NULL) ||
2083 (addr->host_list != NULL && next->host_list != NULL &&
2084 Ustrcmp(addr->host_list->name, next->host_list->name) == 0));
2085
2086 /* If the transport has a batch_id setting, batch_id will be non-NULL
2087 from the expansion outside the loop. Expand for this address and compare.
2088 Expansion failure makes this address ineligible for batching. */
2089
2090 if (ok && batch_id != NULL)
2091 {
2092 uschar *bid;
2093 address_item *save_nextnext = next->next;
2094 next->next = NULL; /* Expansion for a single address */
2095 deliver_set_expansions(next);
2096 next->next = save_nextnext;
2097 bid = expand_string(tp->batch_id);
2098 deliver_set_expansions(NULL);
2099 if (bid == NULL)
2100 {
2101 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option "
2102 "in %s transport (%s): %s", tp->name, next->address,
2103 expand_string_message);
2104 ok = FALSE;
2105 }
2106 else ok = (Ustrcmp(batch_id, bid) == 0);
2107 }
2108
2109 /* Take address into batch if OK. */
2110
2111 if (ok)
2112 {
2113 *anchor = next->next; /* Include the address */
2114 next->next = NULL;
2115 last->next = next;
2116 last = next;
2117 batch_count++;
2118 }
2119 else anchor = &(next->next); /* Skip the address */
2120 }
2121 }
2122
2123 /* We now have one or more addresses that can be delivered in a batch. Check
2124 whether the transport is prepared to accept a message of this size. If not,
2125 fail them all forthwith. If the expansion fails, or does not yield an
2126 integer, defer delivery. */
2127
2128 if (tp->message_size_limit != NULL)
2129 {
2130 int rc = check_message_size(tp, addr);
2131 if (rc != OK)
2132 {
2133 replicate_status(addr);
2134 while (addr != NULL)
2135 {
2136 addr2 = addr->next;
2137 post_process_one(addr, rc, logflags, DTYPE_TRANSPORT, 0);
2138 addr = addr2;
2139 }
2140 continue; /* With next batch of addresses */
2141 }
2142 }
2143
2144 /* If we are not running the queue, or if forcing, all deliveries will be
2145 attempted. Otherwise, we must respect the retry times for each address. Even
2146 when not doing this, we need to set up the retry key string, and determine
2147 whether a retry record exists, because after a successful delivery, a delete
2148 retry item must be set up. Keep the retry database open only for the duration
2149 of these checks, rather than for all local deliveries, because some local
2150 deliveries (e.g. to pipes) can take a substantial time. */
2151
2152 dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE);
2153 if (dbm_file == NULL)
2154 {
2155 DEBUG(D_deliver|D_retry|D_hints_lookup)
2156 debug_printf("no retry data available\n");
2157 }
2158
2159 addr2 = addr;
2160 addr3 = NULL;
2161 while (addr2 != NULL)
2162 {
2163 BOOL ok = TRUE; /* to deliver this address */
2164 uschar *retry_key;
2165
2166 /* Set up the retry key to include the domain or not, and change its
2167 leading character from "R" to "T". Must make a copy before doing this,
2168 because the old key may be pointed to from a "delete" retry item after
2169 a routing delay. */
2170
2171 retry_key = string_copy(
2172 (tp->retry_use_local_part)? addr2->address_retry_key :
2173 addr2->domain_retry_key);
2174 *retry_key = 'T';
2175
2176 /* Inspect the retry data. If there is no hints file, delivery happens. */
2177
2178 if (dbm_file != NULL)
2179 {
2180 dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key);
2181
2182 /* If there is no retry record, delivery happens. If there is,
2183 remember it exists so it can be deleted after a successful delivery. */
2184
2185 if (retry_record != NULL)
2186 {
2187 setflag(addr2, af_lt_retry_exists);
2188
2189 /* A retry record exists for this address. If queue running and not
2190 forcing, inspect its contents. If the record is too old, or if its
2191 retry time has come, or if it has passed its cutoff time, delivery
2192 will go ahead. */
2193
2194 DEBUG(D_retry)
2195 {
2196 debug_printf("retry record exists: age=%d (max=%d)\n",
2197 (int)(now - retry_record->time_stamp), retry_data_expire);
2198 debug_printf(" time to retry = %d expired = %d\n",
2199 (int)(now - retry_record->next_try), retry_record->expired);
2200 }
2201
2202 if (queue_running && !deliver_force)
2203 {
2204 ok = (now - retry_record->time_stamp > retry_data_expire) ||
2205 (now >= retry_record->next_try) ||
2206 retry_record->expired;
2207
2208 /* If we haven't reached the retry time, there is one more check
2209 to do, which is for the ultimate address timeout. */
2210
2211 if (!ok)
2212 {
2213 retry_config *retry =
2214 retry_find_config(retry_key+2, addr2->domain,
2215 retry_record->basic_errno,
2216 retry_record->more_errno);
2217
2218 DEBUG(D_deliver|D_retry)
2219 debug_printf("retry time not reached for %s: "
2220 "checking ultimate address timeout\n", addr2->address);
2221
2222 if (retry != NULL && retry->rules != NULL)
2223 {
2224 retry_rule *last_rule;
2225 for (last_rule = retry->rules;
2226 last_rule->next != NULL;
2227 last_rule = last_rule->next);
2228 if (now - received_time > last_rule->timeout) ok = TRUE;
2229 }
2230 else ok = TRUE; /* No rule => timed out */
2231
2232 DEBUG(D_deliver|D_retry)
2233 {
2234 if (ok) debug_printf("on queue longer than maximum retry for "
2235 "address - allowing delivery\n");
2236 }
2237 }
2238 }
2239 }
2240 else DEBUG(D_retry) debug_printf("no retry record exists\n");
2241 }
2242
2243 /* This address is to be delivered. Leave it on the chain. */
2244
2245 if (ok)
2246 {
2247 addr3 = addr2;
2248 addr2 = addr2->next;
2249 }
2250
2251 /* This address is to be deferred. Take it out of the chain, and
2252 post-process it as complete. Must take it out of the chain first,
2253 because post processing puts it on another chain. */
2254
2255 else
2256 {
2257 address_item *this = addr2;
2258 this->message = US"Retry time not yet reached";
2259 this->basic_errno = ERRNO_LRETRY;
2260 if (addr3 == NULL) addr2 = addr = addr2->next;
2261 else addr2 = addr3->next = addr2->next;
2262 post_process_one(this, DEFER, logflags, DTYPE_TRANSPORT, 0);
2263 }
2264 }
2265
2266 if (dbm_file != NULL) dbfn_close(dbm_file);
2267
2268 /* If there are no addresses left on the chain, they all deferred. Loop
2269 for the next set of addresses. */
2270
2271 if (addr == NULL) continue;
2272
2273 /* So, finally, we do have some addresses that can be passed to the
2274 transport. Before doing so, set up variables that are relevant to a
2275 single delivery. */
2276
2277 deliver_set_expansions(addr);
2278 delivery_start = time(NULL);
2279 deliver_local(addr, FALSE);
2280 deliver_time = (int)(time(NULL) - delivery_start);
2281
2282 /* If a shadow transport (which must perforce be another local transport), is
2283 defined, and its condition is met, we must pass the message to the shadow
2284 too, but only those addresses that succeeded. We do this by making a new
2285 chain of addresses - also to keep the original chain uncontaminated. We must
2286 use a chain rather than doing it one by one, because the shadow transport may
2287 batch.
2288
2289 NOTE: if the condition fails because of a lookup defer, there is nothing we
2290 can do! */
2291
2292 if (tp->shadow != NULL &&
2293 (tp->shadow_condition == NULL ||
2294 expand_check_condition(tp->shadow_condition, tp->name, US"transport")))
2295 {
2296 transport_instance *stp;
2297 address_item *shadow_addr = NULL;
2298 address_item **last = &shadow_addr;
2299
2300 for (stp = transports; stp != NULL; stp = stp->next)
2301 if (Ustrcmp(stp->name, tp->shadow) == 0) break;
2302
2303 if (stp == NULL)
2304 log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ",
2305 tp->shadow);
2306
2307 /* Pick off the addresses that have succeeded, and make clones. Put into
2308 the shadow_message field a pointer to the shadow_message field of the real
2309 address. */
2310
2311 else for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
2312 {
2313 if (addr2->transport_return != OK) continue;
2314 addr3 = store_get(sizeof(address_item));
2315 *addr3 = *addr2;
2316 addr3->next = NULL;
2317 addr3->shadow_message = (uschar *)(&(addr2->shadow_message));
2318 addr3->transport = stp;
2319 addr3->transport_return = DEFER;
2320 addr3->return_filename = NULL;
2321 addr3->return_file = -1;
2322 *last = addr3;
2323 last = &(addr3->next);
2324 }
2325
2326 /* If we found any addresses to shadow, run the delivery, and stick any
2327 message back into the shadow_message field in the original. */
2328
2329 if (shadow_addr != NULL)
2330 {
2331 int save_count = transport_count;
2332
2333 DEBUG(D_deliver|D_transport)
2334 debug_printf(">>>>>>>>>>>>>>>> Shadow delivery >>>>>>>>>>>>>>>>\n");
2335 deliver_local(shadow_addr, TRUE);
2336
2337 for(; shadow_addr != NULL; shadow_addr = shadow_addr->next)
2338 {
2339 int sresult = shadow_addr->transport_return;
2340 *((uschar **)(shadow_addr->shadow_message)) = (sresult == OK)?
2341 string_sprintf(" ST=%s", stp->name) :
2342 string_sprintf(" ST=%s (%s%s%s)", stp->name,
2343 (shadow_addr->basic_errno <= 0)?
2344 US"" : US strerror(shadow_addr->basic_errno),
2345 (shadow_addr->basic_errno <= 0 || shadow_addr->message == NULL)?
2346 US"" : US": ",
2347 (shadow_addr->message != NULL)? shadow_addr->message :
2348 (shadow_addr->basic_errno <= 0)? US"unknown error" : US"");
2349
2350 DEBUG(D_deliver|D_transport)
2351 debug_printf("%s shadow transport returned %s for %s\n",
2352 stp->name,
2353 (sresult == OK)? "OK" :
2354 (sresult == DEFER)? "DEFER" :
2355 (sresult == FAIL)? "FAIL" :
2356 (sresult == PANIC)? "PANIC" : "?",
2357 shadow_addr->address);
2358 }
2359
2360 DEBUG(D_deliver|D_transport)
2361 debug_printf(">>>>>>>>>>>>>>>> End shadow delivery >>>>>>>>>>>>>>>>\n");
2362
2363 transport_count = save_count; /* Restore original transport count */
2364 }
2365 }
2366
2367 /* Cancel the expansions that were set up for the delivery. */
2368
2369 deliver_set_expansions(NULL);
2370
2371 /* Now we can process the results of the real transport. We must take each
2372 address off the chain first, because post_process_one() puts it on another
2373 chain. */
2374
2375 for (addr2 = addr; addr2 != NULL; addr2 = nextaddr)
2376 {
2377 int result = addr2->transport_return;
2378 nextaddr = addr2->next;
2379
2380 DEBUG(D_deliver|D_transport)
2381 debug_printf("%s transport returned %s for %s\n",
2382 tp->name,
2383 (result == OK)? "OK" :
2384 (result == DEFER)? "DEFER" :
2385 (result == FAIL)? "FAIL" :
2386 (result == PANIC)? "PANIC" : "?",
2387 addr2->address);
2388
2389 /* If there is a retry_record, or if delivery is deferred, build a retry
2390 item for setting a new retry time or deleting the old retry record from
2391 the database. These items are handled all together after all addresses
2392 have been handled (so the database is open just for a short time for
2393 updating). */
2394
2395 if (result == DEFER || testflag(addr2, af_lt_retry_exists))
2396 {
2397 int flags = (result == DEFER)? 0 : rf_delete;
2398 uschar *retry_key = string_copy((tp->retry_use_local_part)?
2399 addr2->address_retry_key : addr2->domain_retry_key);
2400 *retry_key = 'T';
2401 retry_add_item(addr2, retry_key, flags);
2402 }
2403
2404 /* Done with this address */
2405
2406 if (result == OK) addr2->more_errno = deliver_time;
2407 post_process_one(addr2, result, logflags, DTYPE_TRANSPORT, logchar);
2408
2409 /* If a pipe delivery generated text to be sent back, the result may be
2410 changed to FAIL, and we must copy this for subsequent addresses in the
2411 batch. */
2412
2413 if (addr2->transport_return != result)
2414 {
2415 for (addr3 = nextaddr; addr3 != NULL; addr3 = addr3->next)
2416 {
2417 addr3->transport_return = addr2->transport_return;
2418 addr3->basic_errno = addr2->basic_errno;
2419 addr3->message = addr2->message;
2420 }
2421 result = addr2->transport_return;
2422 }
2423
2424 /* Whether or not the result was changed to FAIL, we need to copy the
2425 return_file value from the first address into all the addresses of the
2426 batch, so they are all listed in the error message. */
2427
2428 addr2->return_file = addr->return_file;
2429
2430 /* Change log character for recording successful deliveries. */
2431
2432 if (result == OK) logchar = '-';
2433 }
2434 } /* Loop back for next batch of addresses */
2435 }
2436
2437
2438
2439
2440 /*************************************************
2441 * Sort remote deliveries *
2442 *************************************************/
2443
2444 /* This function is called if remote_sort_domains is set. It arranges that the
2445 chain of addresses for remote deliveries is ordered according to the strings
2446 specified. Try to make this shuffling reasonably efficient by handling
2447 sequences of addresses rather than just single ones.
2448
2449 Arguments: None
2450 Returns: Nothing
2451 */
2452
2453 static void
2454 sort_remote_deliveries(void)
2455 {
2456 int sep = 0;
2457 address_item **aptr = &addr_remote;
2458 uschar *listptr = remote_sort_domains;
2459 uschar *pattern;
2460 uschar patbuf[256];
2461
2462 while (*aptr != NULL &&
2463 (pattern = string_nextinlist(&listptr, &sep, patbuf, sizeof(patbuf)))
2464 != NULL)
2465 {
2466 address_item *moved = NULL;
2467 address_item **bptr = &moved;
2468
2469 while (*aptr != NULL)
2470 {
2471 address_item **next;
2472 deliver_domain = (*aptr)->domain; /* set $domain */
2473 if (match_isinlist(deliver_domain, &pattern, UCHAR_MAX+1,
2474 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
2475 {
2476 aptr = &((*aptr)->next);
2477 continue;
2478 }
2479
2480 next = &((*aptr)->next);
2481 while (*next != NULL &&
2482 (deliver_domain = (*next)->domain, /* Set $domain */
2483 match_isinlist(deliver_domain, &pattern, UCHAR_MAX+1,
2484 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL)) != OK)
2485 next = &((*next)->next);
2486
2487 /* If the batch of non-matchers is at the end, add on any that were
2488 extracted further up the chain, and end this iteration. Otherwise,
2489 extract them from the chain and hang on the moved chain. */
2490
2491 if (*next == NULL)
2492 {
2493 *next = moved;
2494 break;
2495 }
2496
2497 *bptr = *aptr;
2498 *aptr = *next;
2499 *next = NULL;
2500 bptr = next;
2501 aptr = &((*aptr)->next);
2502 }
2503
2504 /* If the loop ended because the final address matched, *aptr will
2505 be NULL. Add on to the end any extracted non-matching addresses. If
2506 *aptr is not NULL, the loop ended via "break" when *next is null, that
2507 is, there was a string of non-matching addresses at the end. In this
2508 case the extracted addresses have already been added on the end. */
2509
2510 if (*aptr == NULL) *aptr = moved;
2511 }
2512
2513 DEBUG(D_deliver)
2514 {
2515 address_item *addr;
2516 debug_printf("remote addresses after sorting:\n");
2517 for (addr = addr_remote; addr != NULL; addr = addr->next)
2518 debug_printf(" %s\n", addr->address);
2519 }
2520 }
2521
2522
2523
2524 /*************************************************
2525 * Read from pipe for remote delivery subprocess *
2526 *************************************************/
2527
2528 /* This function is called when the subprocess is complete, but can also be
2529 called before it is complete, in order to empty a pipe that is full (to prevent
2530 deadlock). It must therefore keep track of its progress in the parlist data
2531 block.
2532
2533 We read the pipe to get the delivery status codes and a possible error message
2534 for each address, optionally preceded by unusability data for the hosts and
2535 also by optional retry data.
2536
2537 Read in large chunks into the big buffer and then scan through, interpreting
2538 the data therein. In most cases, only a single read will be necessary. No
2539 individual item will ever be anywhere near 500 bytes in length, so by ensuring
2540 that we read the next chunk when there is less than 500 bytes left in the
2541 non-final chunk, we can assume each item is complete in store before handling
2542 it. Actually, each item is written using a single write(), which is atomic for
2543 small items (less than PIPE_BUF, which seems to be at least 512 in any Unix) so
2544 even if we are reading while the subprocess is still going, we should never
2545 have only a partial item in the buffer.
2546
2547 Argument:
2548 poffset the offset of the parlist item
2549 eop TRUE if the process has completed
2550
2551 Returns: TRUE if the terminating 'Z' item has been read,
2552 or there has been a disaster (i.e. no more data needed);
2553 FALSE otherwise
2554 */
2555
2556 static BOOL
2557 par_read_pipe(int poffset, BOOL eop)
2558 {
2559 host_item *h;
2560 pardata *p = parlist + poffset;
2561 address_item *addrlist = p->addrlist;
2562 address_item *addr = p->addr;
2563 pid_t pid = p->pid;
2564 int fd = p->fd;
2565 uschar *endptr = big_buffer;
2566 uschar *ptr = endptr;
2567 uschar *msg = p->msg;
2568 BOOL done = p->done;
2569 BOOL unfinished = TRUE;
2570
2571 /* Loop through all items, reading from the pipe when necessary. The pipe
2572 is set up to be non-blocking, but there are two different Unix mechanisms in
2573 use. Exim uses O_NONBLOCK if it is defined. This returns 0 for end of file,
2574 and EAGAIN for no more data. If O_NONBLOCK is not defined, Exim uses O_NDELAY,
2575 which returns 0 for both end of file and no more data. We distinguish the
2576 two cases by taking 0 as end of file only when we know the process has
2577 completed.
2578
2579 Each separate item is written to the pipe in a single write(), and as they are
2580 all short items, the writes will all be atomic and we should never find
2581 ourselves in the position of having read an incomplete item. */
2582
2583 DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n",
2584 (int)p->pid, eop? "ended" : "not ended");
2585
2586 while (!done)
2587 {
2588 retry_item *r, **rp;
2589 int remaining = endptr - ptr;
2590
2591 /* Read (first time) or top up the chars in the buffer if necessary.
2592 There will be only one read if we get all the available data (i.e. don't
2593 fill the buffer completely). */
2594
2595 if (remaining < 500 && unfinished)
2596 {
2597 int len;
2598 int available = big_buffer_size - remaining;
2599
2600 if (remaining > 0) memmove(big_buffer, ptr, remaining);
2601
2602 ptr = big_buffer;
2603 endptr = big_buffer + remaining;
2604 len = read(fd, endptr, available);
2605
2606 DEBUG(D_deliver) debug_printf("read() yielded %d\n", len);
2607
2608 /* If the result is EAGAIN and the process is not complete, just
2609 stop reading any more and process what we have already. */
2610
2611 if (len < 0)
2612 {
2613 if (!eop && errno == EAGAIN) len = 0; else
2614 {
2615 msg = string_sprintf("failed to read pipe from transport process "
2616 "%d for transport %s: %s", pid, addr->transport->driver_name,
2617 strerror(errno));
2618 break;
2619 }
2620 }
2621
2622 /* If the length is zero (eof or no-more-data), just process what we
2623 already have. Note that if the process is still running and we have
2624 read all the data in the pipe (but less that "available") then we
2625 won't read any more, as "unfinished" will get set FALSE. */
2626
2627 endptr += len;
2628 unfinished = len == available;
2629 }
2630
2631 /* If we are at the end of the available data, exit the loop. */
2632
2633 if (ptr >= endptr) break;
2634
2635 /* Handle each possible type of item, assuming the complete item is
2636 available in store. */
2637
2638 switch (*ptr++)
2639 {
2640 /* Host items exist only if any hosts were marked unusable. Match
2641 up by checking the IP address. */
2642
2643 case 'H':
2644 for (h = addrlist->host_list; h != NULL; h = h->next)
2645 {
2646 if (h->address == NULL || Ustrcmp(h->address, ptr+2) != 0) continue;
2647 h->status = ptr[0];
2648 h->why = ptr[1];
2649 }
2650 ptr += 2;
2651 while (*ptr++);
2652 break;
2653
2654 /* Retry items are sent in a preceding R item for each address. This is
2655 kept separate to keep each message short enough to guarantee it won't
2656 be split in the pipe. Hopefully, in the majority of cases, there won't in
2657 fact be any retry items at all.
2658
2659 The complete set of retry items might include an item to delete a
2660 routing retry if there was a previous routing delay. However, routing
2661 retries are also used when a remote transport identifies an address error.
2662 In that case, there may also be an "add" item for the same key. Arrange
2663 that a "delete" item is dropped in favour of an "add" item. */
2664
2665 case 'R':
2666 if (addr == NULL) goto ADDR_MISMATCH;
2667
2668 DEBUG(D_deliver|D_retry)
2669 debug_printf("reading retry information for %s from subprocess\n",
2670 ptr+1);
2671
2672 /* Cut out any "delete" items on the list. */
2673
2674 for (rp = &(addr->retries); (r = *rp) != NULL; rp = &(r->next))
2675 {
2676 if (Ustrcmp(r->key, ptr+1) == 0) /* Found item with same key */
2677 {
2678 if ((r->flags & rf_delete) == 0) break; /* It was not "delete" */
2679 *rp = r->next; /* Excise a delete item */
2680 DEBUG(D_deliver|D_retry)
2681 debug_printf(" existing delete item dropped\n");
2682 }
2683 }
2684
2685 /* We want to add a delete item only if there is no non-delete item;
2686 however we still have to step ptr through the data. */
2687
2688 if (r == NULL || (*ptr & rf_delete) == 0)
2689 {
2690 r = store_get(sizeof(retry_item));
2691 r->next = addr->retries;
2692 addr->retries = r;
2693 r->flags = *ptr++;
2694 r->key = string_copy(ptr);
2695 while (*ptr++);
2696 memcpy(&(r->basic_errno), ptr, sizeof(r->basic_errno));
2697 ptr += sizeof(r->basic_errno);
2698 memcpy(&(r->more_errno), ptr, sizeof(r->more_errno));
2699 ptr += sizeof(r->more_errno);
2700 r->message = (*ptr)? string_copy(ptr) : NULL;
2701 DEBUG(D_deliver|D_retry)
2702 debug_printf(" added %s item\n",
2703 ((r->flags & rf_delete) == 0)? "retry" : "delete");
2704 }
2705
2706 else
2707 {
2708 DEBUG(D_deliver|D_retry)
2709 debug_printf(" delete item not added: non-delete item exists\n");
2710 ptr++;
2711 while(*ptr++);
2712 ptr += sizeof(r->basic_errno) + sizeof(r->more_errno);
2713 }
2714
2715 while(*ptr++);
2716 break;
2717
2718 /* Put the amount of data written into the parlist block */
2719
2720 case 'S':
2721 memcpy(&(p->transport_count), ptr, sizeof(transport_count));
2722 ptr += sizeof(transport_count);
2723 break;
2724
2725 /* Address items are in the order of items on the address chain. We
2726 remember the current address value in case this function is called
2727 several times to empty the pipe in stages. Information about delivery
2728 over TLS is sent in a preceding X item for each address. We don't put
2729 it in with the other info, in order to keep each message short enough to
2730 guarantee it won't be split in the pipe. */
2731
2732 #ifdef SUPPORT_TLS
2733 case 'X':
2734 if (addr == NULL) goto ADDR_MISMATCH; /* Below, in 'A' handler */
2735 addr->cipher = (*ptr)? string_copy(ptr) : NULL;
2736 while (*ptr++);
2737 addr->peerdn = (*ptr)? string_copy(ptr) : NULL;
2738 while (*ptr++);
2739 break;
2740 #endif
2741
2742 case 'A':
2743 if (addr == NULL)
2744 {
2745 ADDR_MISMATCH:
2746 msg = string_sprintf("address count mismatch for data read from pipe "
2747 "for transport process %d for transport %s", pid,
2748 addrlist->transport->driver_name);
2749 done = TRUE;
2750 break;
2751 }
2752
2753 addr->transport_return = *ptr++;
2754 addr->special_action = *ptr++;
2755 memcpy(&(addr->basic_errno), ptr, sizeof(addr->basic_errno));
2756 ptr += sizeof(addr->basic_errno);
2757 memcpy(&(addr->more_errno), ptr, sizeof(addr->more_errno));
2758 ptr += sizeof(addr->more_errno);
2759 memcpy(&(addr->flags), ptr, sizeof(addr->flags));
2760 ptr += sizeof(addr->flags);
2761 addr->message = (*ptr)? string_copy(ptr) : NULL;
2762 while(*ptr++);
2763 addr->user_message = (*ptr)? string_copy(ptr) : NULL;
2764 while(*ptr++);
2765
2766 /* Always two strings for host information, followed by the port number */
2767
2768 if (*ptr != 0)
2769 {
2770 h = store_get(sizeof(host_item));
2771 h->name = string_copy(ptr);
2772 while (*ptr++);
2773 h->address = string_copy(ptr);
2774 while(*ptr++);
2775 memcpy(&(h->port), ptr, sizeof(h->port));
2776 ptr += sizeof(h->port);
2777 addr->host_used = h;
2778 }
2779 else ptr++;
2780
2781 /* Finished with this address */
2782
2783 addr = addr->next;
2784 break;
2785
2786 /* Z marks the logical end of the data. It is followed by '0' if
2787 continue_transport was NULL at the end of transporting, otherwise '1'.
2788 We need to know when it becomes NULL during a delivery down a passed SMTP
2789 channel so that we don't try to pass anything more down it. Of course, for
2790 most normal messages it will remain NULL all the time. */
2791
2792 case 'Z':
2793 if (*ptr == '0')
2794 {
2795 continue_transport = NULL;
2796 continue_hostname = NULL;
2797 }
2798 done = TRUE;
2799 DEBUG(D_deliver) debug_printf("Z%c item read\n", *ptr);
2800 break;
2801
2802 /* Anything else is a disaster. */
2803
2804 default:
2805 msg = string_sprintf("malformed data (%d) read from pipe for transport "
2806 "process %d for transport %s", ptr[-1], pid,
2807 addr->transport->driver_name);
2808 done = TRUE;
2809 break;
2810 }
2811 }
2812
2813 /* The done flag is inspected externally, to determine whether or not to
2814 call the function again when the process finishes. */
2815
2816 p->done = done;
2817
2818 /* If the process hadn't finished, and we haven't seen the end of the data
2819 or suffered a disaster, update the rest of the state, and return FALSE to
2820 indicate "not finished". */
2821
2822 if (!eop && !done)
2823 {
2824 p->addr = addr;
2825 p->msg = msg;
2826 return FALSE;
2827 }
2828
2829 /* Close our end of the pipe, to prevent deadlock if the far end is still
2830 pushing stuff into it. */
2831
2832 close(fd);
2833 p->fd = -1;
2834
2835 /* If we have finished without error, but haven't had data for every address,
2836 something is wrong. */
2837
2838 if (msg == NULL && addr != NULL)
2839 msg = string_sprintf("insufficient address data read from pipe "
2840 "for transport process %d for transport %s", pid,
2841 addr->transport->driver_name);
2842
2843 /* If an error message is set, something has gone wrong in getting back
2844 the delivery data. Put the message into each address and freeze it. */
2845
2846 if (msg != NULL)
2847 {
2848 for (addr = addrlist; addr != NULL; addr = addr->next)
2849 {
2850 addr->transport_return = DEFER;
2851 addr->special_action = SPECIAL_FREEZE;
2852 addr->message = msg;
2853 }
2854 }
2855
2856 /* Return TRUE to indicate we have got all we need from this process, even
2857 if it hasn't actually finished yet. */
2858
2859 return TRUE;
2860 }
2861
2862
2863
2864 /*************************************************
2865 * Post-process a set of remote addresses *
2866 *************************************************/
2867
2868 /* Do what has to be done immediately after a remote delivery for each set of
2869 addresses, then re-write the spool if necessary. Note that post_process_one
2870 puts the address on an appropriate queue; hence we must fish off the next
2871 one first. This function is also called if there is a problem with setting
2872 up a subprocess to do a remote delivery in parallel. In this case, the final
2873 argument contains a message, and the action must be forced to DEFER.
2874
2875 Argument:
2876 addr pointer to chain of address items
2877 logflags flags for logging
2878 msg NULL for normal cases; -> error message for unexpected problems
2879 fallback TRUE if processing fallback hosts
2880
2881 Returns: nothing
2882 */
2883
2884 static void
2885 remote_post_process(address_item *addr, int logflags, uschar *msg,
2886 BOOL fallback)
2887 {
2888 host_item *h;
2889
2890 /* If any host addresses were found to be unusable, add them to the unusable
2891 tree so that subsequent deliveries don't try them. */
2892
2893 for (h = addr->host_list; h != NULL; h = h->next)
2894 {
2895 if (h->address == NULL) continue;
2896 if (h->status >= hstatus_unusable) tree_add_unusable(h);
2897 }
2898
2899 /* Now handle each address on the chain. The transport has placed '=' or '-'
2900 into the special_action field for each successful delivery. */
2901
2902 while (addr != NULL)
2903 {
2904 address_item *next = addr->next;
2905
2906 /* If msg == NULL (normal processing) and the result is DEFER and we are
2907 processing the main hosts and there are fallback hosts available, put the
2908 address on the list for fallback delivery. */
2909
2910 if (addr->transport_return == DEFER &&
2911 addr->fallback_hosts != NULL &&
2912 !fallback &&
2913 msg == NULL)
2914 {
2915 addr->host_list = addr->fallback_hosts;
2916 addr->next = addr_fallback;
2917 addr_fallback = addr;
2918 DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", addr->address);
2919 }
2920
2921 /* If msg is set (=> unexpected problem), set it in the address before
2922 doing the ordinary post processing. */
2923
2924 else
2925 {
2926 if (msg != NULL)
2927 {
2928 addr->message = msg;
2929 addr->transport_return = DEFER;
2930 }
2931 (void)post_process_one(addr, addr->transport_return, logflags,
2932 DTYPE_TRANSPORT, addr->special_action);
2933 }
2934
2935 /* Next address */
2936
2937 addr = next;
2938 }
2939
2940 /* If we have just delivered down a passed SMTP channel, and that was
2941 the last address, the channel will have been closed down. Now that
2942 we have logged that delivery, set continue_sequence to 1 so that
2943 any subsequent deliveries don't get "*" incorrectly logged. */
2944
2945 if (continue_transport == NULL) continue_sequence = 1;
2946 }
2947
2948
2949
2950 /*************************************************
2951 * Wait for one remote delivery subprocess *
2952 *************************************************/
2953
2954 /* This function is called while doing remote deliveries when either the
2955 maximum number of processes exist and we need one to complete so that another
2956 can be created, or when waiting for the last ones to complete. It must wait for
2957 the completion of one subprocess, empty the control block slot, and return a
2958 pointer to the address chain.
2959
2960 Arguments: none
2961 Returns: pointer to the chain of addresses handled by the process;
2962 NULL if no subprocess found - this is an unexpected error
2963 */
2964
2965 static address_item *
2966 par_wait(void)
2967 {
2968 int poffset, status;
2969 address_item *addr, *addrlist;
2970 pid_t pid;
2971
2972 set_process_info("delivering %s: waiting for a remote delivery subprocess "
2973 "to finish", message_id);
2974
2975 /* Loop until either a subprocess completes, or there are no subprocesses in
2976 existence - in which case give an error return. We cannot proceed just by
2977 waiting for a completion, because a subprocess may have filled up its pipe, and
2978 be waiting for it to be emptied. Therefore, if no processes have finished, we
2979 wait for one of the pipes to acquire some data by calling select(), with a
2980 timeout just in case.
2981
2982 The simple approach is just to iterate after reading data from a ready pipe.
2983 This leads to non-ideal behaviour when the subprocess has written its final Z
2984 item, closed the pipe, and is in the process of exiting (the common case). A
2985 call to waitpid() yields nothing completed, but select() shows the pipe ready -
2986 reading it yields EOF, so you end up with busy-waiting until the subprocess has
2987 actually finished.
2988
2989 To avoid this, if all the data that is needed has been read from a subprocess
2990 after select(), an explicit wait() for it is done. We know that all it is doing
2991 is writing to the pipe and then exiting, so the wait should not be long.
2992
2993 The non-blocking waitpid() is to some extent just insurance; if we could
2994 reliably detect end-of-file on the pipe, we could always know when to do a
2995 blocking wait() for a completed process. However, because some systems use
2996 NDELAY, which doesn't distinguish between EOF and pipe empty, it is easier to
2997 use code that functions without the need to recognize EOF.
2998
2999 There's a double loop here just in case we end up with a process that is not in
3000 the list of remote delivery processes. Something has obviously gone wrong if
3001 this is the case. (For example, a process that is incorrectly left over from
3002 routing or local deliveries might be found.) The damage can be minimized by
3003 looping back and looking for another process. If there aren't any, the error
3004 return will happen. */
3005
3006 for (;;) /* Normally we do not repeat this loop */
3007 {
3008 while ((pid = waitpid(-1, &status, WNOHANG)) <= 0)
3009 {
3010 struct timeval tv;
3011 fd_set select_pipes;
3012 int maxpipe, readycount;
3013
3014 /* A return value of -1 can mean several things. If errno != ECHILD, it
3015 either means invalid options (which we discount), or that this process was
3016 interrupted by a signal. Just loop to try the waitpid() again.
3017
3018 If errno == ECHILD, waitpid() is telling us that there are no subprocesses
3019 in existence. This should never happen, and is an unexpected error.
3020 However, there is a nasty complication when running under Linux. If "strace
3021 -f" is being used under Linux to trace this process and its children,
3022 subprocesses are "stolen" from their parents and become the children of the
3023 tracing process. A general wait such as the one we've just obeyed returns
3024 as if there are no children while subprocesses are running. Once a
3025 subprocess completes, it is restored to the parent, and waitpid(-1) finds
3026 it. Thanks to Joachim Wieland for finding all this out and suggesting a
3027 palliative.
3028
3029 This does not happen using "truss" on Solaris, nor (I think) with other
3030 tracing facilities on other OS. It seems to be specific to Linux.
3031
3032 What we do to get round this is to use kill() to see if any of our
3033 subprocesses are still in existence. If kill() gives an OK return, we know
3034 it must be for one of our processes - it can't be for a re-use of the pid,
3035 because if our process had finished, waitpid() would have found it. If any
3036 of our subprocesses are in existence, we proceed to use select() as if
3037 waitpid() had returned zero. I think this is safe. */
3038
3039 if (pid < 0)
3040 {
3041 if (errno != ECHILD) continue; /* Repeats the waitpid() */
3042
3043 DEBUG(D_deliver)
3044 debug_printf("waitpid() returned -1/ECHILD: checking explicitly "
3045 "for process existence\n");
3046
3047 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3048 {
3049 if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0)
3050 {
3051 DEBUG(D_deliver) debug_printf("process %d still exists: assume "
3052 "stolen by strace\n", (int)pid);
3053 break; /* With poffset set */
3054 }
3055 }
3056
3057 if (poffset >= remote_max_parallel)
3058 {
3059 DEBUG(D_deliver) debug_printf("*** no delivery children found\n");
3060 return NULL; /* This is the error return */
3061 }
3062 }
3063
3064 /* A pid value greater than 0 breaks the "while" loop. A negative value has
3065 been handled above. A return value of zero means that there is at least one
3066 subprocess, but there are no completed subprocesses. See if any pipes are
3067 ready with any data for reading. */
3068
3069 DEBUG(D_deliver) debug_printf("selecting on subprocess pipes\n");
3070
3071 maxpipe = 0;
3072 FD_ZERO(&select_pipes);
3073 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3074 {
3075 if (parlist[poffset].pid != 0)
3076 {
3077 int fd = parlist[poffset].fd;
3078 FD_SET(fd, &select_pipes);
3079 if (fd > maxpipe) maxpipe = fd;
3080 }
3081 }
3082
3083 /* Stick in a 60-second timeout, just in case. */
3084
3085 tv.tv_sec = 60;
3086 tv.tv_usec = 0;
3087
3088 readycount = select(maxpipe + 1, (SELECT_ARG2_TYPE *)&select_pipes,
3089 NULL, NULL, &tv);
3090
3091 /* Scan through the pipes and read any that are ready; use the count
3092 returned by select() to stop when there are no more. Select() can return
3093 with no processes (e.g. if interrupted). This shouldn't matter.
3094
3095 If par_read_pipe() returns TRUE, it means that either the terminating Z was
3096 read, or there was a disaster. In either case, we are finished with this
3097 process. Do an explicit wait() for the process and break the main loop if
3098 it succeeds.
3099
3100 It turns out that we have to deal with the case of an interrupted system
3101 call, which can happen on some operating systems if the signal handling is
3102 set up to do that by default. */
3103
3104 for (poffset = 0;
3105 readycount > 0 && poffset < remote_max_parallel;
3106 poffset++)
3107 {
3108 if ((pid = parlist[poffset].pid) != 0 &&
3109 FD_ISSET(parlist[poffset].fd, &select_pipes))
3110 {
3111 readycount--;
3112 if (par_read_pipe(poffset, FALSE)) /* Finished with this pipe */
3113 {
3114 for (;;) /* Loop for signals */
3115 {
3116 pid_t endedpid = waitpid(pid, &status, 0);
3117 if (endedpid == pid) goto PROCESS_DONE;
3118 if (endedpid != (pid_t)(-1) || errno != EINTR)
3119 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return "
3120 "%d (errno = %d) from waitpid() for process %d",
3121 (int)endedpid, errno, (int)pid);
3122 }
3123 }
3124 }
3125 }
3126
3127 /* Now go back and look for a completed subprocess again. */
3128 }
3129
3130 /* A completed process was detected by the non-blocking waitpid(). Find the
3131 data block that corresponds to this subprocess. */
3132
3133 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3134 if (pid == parlist[poffset].pid) break;
3135
3136 /* Found the data block; this is a known remote delivery process. We don't
3137 need to repeat the outer loop. This should be what normally happens. */
3138
3139 if (poffset < remote_max_parallel) break;
3140
3141 /* This situation is an error, but it's probably better to carry on looking
3142 for another process than to give up (as we used to do). */
3143
3144 log_write(0, LOG_MAIN|LOG_PANIC, "Process %d finished: not found in remote "
3145 "transport process list", pid);
3146 } /* End of the "for" loop */
3147
3148 /* Come here when all the data was completely read after a select(), and
3149 the process in pid has been wait()ed for. */
3150
3151 PROCESS_DONE:
3152
3153 DEBUG(D_deliver)
3154 {
3155 if (status == 0)
3156 debug_printf("remote delivery process %d ended\n", (int)pid);
3157 else
3158 debug_printf("remote delivery process %d ended: status=%04x\n", (int)pid,
3159 status);
3160 }
3161
3162 set_process_info("delivering %s", message_id);
3163
3164 /* Get the chain of processed addresses */
3165
3166 addrlist = parlist[poffset].addrlist;
3167
3168 /* If the process did not finish cleanly, record an error and freeze (except
3169 for SIGTERM, SIGKILL and SIGQUIT), and also ensure the journal is not removed,
3170 in case the delivery did actually happen. */
3171
3172 if ((status & 0xffff) != 0)
3173 {
3174 uschar *msg;
3175 int msb = (status >> 8) & 255;
3176 int lsb = status & 255;
3177 int code = (msb == 0)? (lsb & 0x7f) : msb;
3178
3179 msg = string_sprintf("%s transport process returned non-zero status 0x%04x: "
3180 "%s %d",
3181 addrlist->transport->driver_name,
3182 status,
3183 (msb == 0)? "terminated by signal" : "exit code",
3184 code);
3185
3186 if (msb != 0 || (code != SIGTERM && code != SIGKILL && code != SIGQUIT))
3187 addrlist->special_action = SPECIAL_FREEZE;
3188
3189 for (addr = addrlist; addr != NULL; addr = addr->next)
3190 {
3191 addr->transport_return = DEFER;
3192 addr->message = msg;
3193 }
3194
3195 remove_journal = FALSE;
3196 }
3197
3198 /* Else complete reading the pipe to get the result of the delivery, if all
3199 the data has not yet been obtained. */
3200
3201 else if (!parlist[poffset].done) (void)par_read_pipe(poffset, TRUE);
3202
3203 /* Put the data count and return path into globals, mark the data slot unused,
3204 decrement the count of subprocesses, and return the address chain. */
3205
3206 transport_count = parlist[poffset].transport_count;
3207 used_return_path = parlist[poffset].return_path;
3208 parlist[poffset].pid = 0;
3209 parcount--;
3210 return addrlist;
3211 }
3212
3213
3214
3215 /*************************************************
3216 * Wait for subprocesses and post-process *
3217 *************************************************/
3218
3219 /* This function waits for subprocesses until the number that are still running
3220 is below a given threshold. For each complete subprocess, the addresses are
3221 post-processed. If we can't find a running process, there is some shambles.
3222 Better not bomb out, as that might lead to multiple copies of the message. Just
3223 log and proceed as if all done.
3224
3225 Arguments:
3226 max maximum number of subprocesses to leave running
3227 fallback TRUE if processing fallback hosts
3228
3229 Returns: nothing
3230 */
3231
3232 static void
3233 par_reduce(int max, BOOL fallback)
3234 {
3235 while (parcount > max)
3236 {
3237 address_item *doneaddr = par_wait();
3238 if (doneaddr == NULL)
3239 {
3240 log_write(0, LOG_MAIN|LOG_PANIC,
3241 "remote delivery process count got out of step");
3242 parcount = 0;
3243 }
3244 else remote_post_process(doneaddr, LOG_MAIN, NULL, fallback);
3245 }
3246 }
3247
3248
3249
3250
3251 /*************************************************
3252 * Do remote deliveries *
3253 *************************************************/
3254
3255 /* This function is called to process the addresses in addr_remote. We must
3256 pick off the queue all addresses that have the same transport, remote
3257 destination, and errors address, and hand them to the transport in one go,
3258 subject to some configured limitations. If this is a run to continue delivering
3259 to an existing delivery channel, skip all but those addresses that can go to
3260 that channel. The skipped addresses just get deferred.
3261
3262 If mua_wrapper is set, all addresses must be able to be sent in a single
3263 transaction. If not, this function yields FALSE.
3264
3265 In Exim 4, remote deliveries are always done in separate processes, even
3266 if remote_max_parallel = 1 or if there's only one delivery to do. The reason
3267 is so that the base process can retain privilege. This makes the
3268 implementation of fallback transports feasible (though not initially done.)
3269
3270 We create up to the configured number of subprocesses, each of which passes
3271 back the delivery state via a pipe. (However, when sending down an existing
3272 connection, remote_max_parallel is forced to 1.)
3273
3274 Arguments:
3275 fallback TRUE if processing fallback hosts
3276
3277 Returns: TRUE normally
3278 FALSE if mua_wrapper is set and the addresses cannot all be sent
3279 in one transaction
3280 */
3281
3282 static BOOL
3283 do_remote_deliveries(BOOL fallback)
3284 {
3285 int parmax;
3286 int delivery_count;
3287 int poffset;
3288
3289 parcount = 0; /* Number of executing subprocesses */
3290
3291 /* When sending down an existing channel, only do one delivery at a time.
3292 We use a local variable (parmax) to hold the maximum number of processes;
3293 this gets reduced from remote_max_parallel if we can't create enough pipes. */
3294
3295 if (continue_transport != NULL) remote_max_parallel = 1;
3296 parmax = remote_max_parallel;
3297
3298 /* If the data for keeping a list of processes hasn't yet been
3299 set up, do so. */
3300
3301 if (parlist == NULL)
3302 {
3303 parlist = store_get(remote_max_parallel * sizeof(pardata));
3304 for (poffset = 0; poffset < remote_max_parallel; poffset++)
3305 parlist[poffset].pid = 0;
3306 }
3307
3308 /* Now loop for each remote delivery */
3309
3310 for (delivery_count = 0; addr_remote != NULL; delivery_count++)
3311 {
3312 pid_t pid;
3313 uid_t uid;
3314 gid_t gid;
3315 int pfd[2];
3316 int address_count = 1;
3317 int address_count_max;
3318 BOOL multi_domain;
3319 BOOL use_initgroups;
3320 BOOL pipe_done = FALSE;
3321 transport_instance *tp;
3322 address_item **anchor = &addr_remote;
3323 address_item *addr = addr_remote;
3324 address_item *last = addr;
3325 address_item *next;
3326
3327 /* Pull the first address right off the list. */
3328
3329 addr_remote = addr->next;
3330 addr->next = NULL;
3331
3332 DEBUG(D_deliver|D_transport)
3333 debug_printf("--------> %s <--------\n", addr->address);
3334
3335 /* If no transport has been set, there has been a big screw-up somewhere. */
3336
3337 if ((tp = addr->transport) == NULL)
3338 {
3339 disable_logging = FALSE; /* Jic */
3340 remote_post_process(addr, LOG_MAIN|LOG_PANIC,
3341 US"No transport set by router", fallback);
3342 continue;
3343 }
3344
3345 /* Check that this base address hasn't previously been delivered to this
3346 transport. The check is necessary at this point to handle homonymic addresses
3347 correctly in cases where the pattern of redirection changes between delivery
3348 attempts. Non-homonymic previous delivery is detected earlier, at routing
3349 time. */
3350
3351 if (previously_transported(addr)) continue;
3352
3353 /* Force failure if the message is too big. */
3354
3355 if (tp->message_size_limit != NULL)
3356 {
3357 int rc = check_message_size(tp, addr);
3358 if (rc != OK)
3359 {
3360 addr->transport_return = rc;
3361 remote_post_process(addr, LOG_MAIN, NULL, fallback);
3362 continue;
3363 }
3364 }
3365
3366 /* Get the flag which specifies whether the transport can handle different
3367 domains that nevertheless resolve to the same set of hosts. */
3368
3369 multi_domain = tp->multi_domain;
3370
3371 /* Get the maximum it can handle in one envelope, with zero meaning
3372 unlimited, which is forced for the MUA wrapper case. */
3373
3374 address_count_max = tp->max_addresses;
3375 if (address_count_max == 0 || mua_wrapper) address_count_max = 999999;
3376
3377
3378 /************************************************************************/
3379 /***** This is slightly experimental code, but should be safe. *****/
3380
3381 /* The address_count_max value is the maximum number of addresses that the
3382 transport can send in one envelope. However, the transport must be capable of
3383 dealing with any number of addresses. If the number it gets exceeds its
3384 envelope limitation, it must send multiple copies of the message. This can be
3385 done over a single connection for SMTP, so uses less resources than making
3386 multiple connections. On the other hand, if remote_max_parallel is greater
3387 than one, it is perhaps a good idea to use parallel processing to move the
3388 message faster, even if that results in multiple simultaneous connections to
3389 the same host.
3390
3391 How can we come to some compromise between these two ideals? What we do is to
3392 limit the number of addresses passed to a single instance of a transport to
3393 the greater of (a) its address limit (rcpt_max for SMTP) and (b) the total
3394 number of addresses routed to remote transports divided by
3395 remote_max_parallel. For example, if the message has 100 remote recipients,
3396 remote max parallel is 2, and rcpt_max is 10, we'd never send more than 50 at
3397 once. But if rcpt_max is 100, we could send up to 100.
3398
3399 Of course, not all the remotely addresses in a message are going to go to the
3400 same set of hosts (except in smarthost configurations), so this is just a
3401 heuristic way of dividing up the work.
3402
3403 Furthermore (1), because this may not be wanted in some cases, and also to
3404 cope with really pathological cases, there is also a limit to the number of
3405 messages that are sent over one connection. This is the same limit that is
3406 used when sending several different messages over the same connection.
3407 Continue_sequence is set when in this situation, to the number sent so
3408 far, including this message.
3409
3410 Furthermore (2), when somebody explicitly sets the maximum value to 1, it
3411 is probably because they are using VERP, in which case they want to pass only
3412 one address at a time to the transport, in order to be able to use
3413 $local_part and $domain in constructing a new return path. We could test for
3414 the use of these variables, but as it is so likely they will be used when the
3415 maximum is 1, we don't bother. Just leave the value alone. */
3416
3417 if (address_count_max != 1 &&
3418 address_count_max < remote_delivery_count/remote_max_parallel)
3419 {
3420 int new_max = remote_delivery_count/remote_max_parallel;
3421 int message_max = tp->connection_max_messages;
3422 if (connection_max_messages >= 0) message_max = connection_max_messages;
3423 message_max -= continue_sequence - 1;
3424 if (message_max > 0 && new_max > address_count_max * message_max)
3425 new_max = address_count_max * message_max;
3426 address_count_max = new_max;
3427 }
3428
3429 /************************************************************************/
3430
3431
3432 /* Pick off all addresses which have the same transport, errors address,
3433 destination, and extra headers. In some cases they point to the same host
3434 list, but we also need to check for identical host lists generated from
3435 entirely different domains. The host list pointers can be NULL in the case
3436 where the hosts are defined in the transport. There is also a configured
3437 maximum limit of addresses that can be handled at once (see comments above
3438 for how it is computed). */
3439
3440 while ((next = *anchor) != NULL && address_count < address_count_max)
3441 {
3442 if ((multi_domain || Ustrcmp(next->domain, addr->domain) == 0)
3443 &&
3444 tp == next->transport
3445 &&
3446 same_hosts(next->host_list, addr->host_list)
3447 &&
3448 same_strings(next->p.errors_address, addr->p.errors_address)
3449 &&
3450 same_headers(next->p.extra_headers, addr->p.extra_headers