CHUNKING: pipeline data right after the BDAT command
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
80fea873 5/* Copyright (c) University of Cambridge 1995 - 2016 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* General functions concerned with transportation, and generic options for all
9transports. */
10
11
12#include "exim.h"
13
a8530a10
TK
14#ifdef HAVE_LINUX_SENDFILE
15#include <sys/sendfile.h>
16#endif
059ec3d9
PH
17
18/* Structure for keeping list of addresses that have been added to
19Envelope-To:, in order to avoid duplication. */
20
21struct aci {
22 struct aci *next;
23 address_item *ptr;
24 };
25
26
27/* Static data for write_chunk() */
28
29static uschar *chunk_ptr; /* chunk pointer */
30static uschar *nl_check; /* string to look for at line start */
31static int nl_check_length; /* length of same */
32static uschar *nl_escape; /* string to insert */
33static int nl_escape_length; /* length of same */
34static int nl_partial_match; /* length matched at chunk end */
35
36
37/* Generic options for transports, all of which live inside transport_instance
38data blocks and which therefore have the opt_public flag set. Note that there
39are other options living inside this structure which can be set only from
40certain transports. */
41
42optionlist optionlist_transports[] = {
43 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
44 (void *)offsetof(transport_instance, expand_gid) },
45 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
46 (void *)offsetof(transport_instance, expand_uid) },
47 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
48 (void *)offsetof(transport_instance, rewrite_existflags) },
49 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
50 (void *)offsetof(transport_instance, rewrite_rules) },
51 { "*set_group", opt_bool|opt_hidden|opt_public,
52 (void *)offsetof(transport_instance, gid_set) },
53 { "*set_user", opt_bool|opt_hidden|opt_public,
54 (void *)offsetof(transport_instance, uid_set) },
55 { "body_only", opt_bool|opt_public,
56 (void *)offsetof(transport_instance, body_only) },
57 { "current_directory", opt_stringptr|opt_public,
58 (void *)offsetof(transport_instance, current_dir) },
59 { "debug_print", opt_stringptr | opt_public,
60 (void *)offsetof(transport_instance, debug_string) },
61 { "delivery_date_add", opt_bool|opt_public,
62 (void *)(offsetof(transport_instance, delivery_date_add)) },
63 { "disable_logging", opt_bool|opt_public,
64 (void *)(offsetof(transport_instance, disable_logging)) },
65 { "driver", opt_stringptr|opt_public,
66 (void *)offsetof(transport_instance, driver_name) },
67 { "envelope_to_add", opt_bool|opt_public,
68 (void *)(offsetof(transport_instance, envelope_to_add)) },
0cbf2b82 69#ifndef DISABLE_EVENT
774ef2d7
JH
70 { "event_action", opt_stringptr | opt_public,
71 (void *)offsetof(transport_instance, event_action) },
72#endif
059ec3d9
PH
73 { "group", opt_expand_gid|opt_public,
74 (void *)offsetof(transport_instance, gid) },
846726c5 75 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
76 (void *)offsetof(transport_instance, add_headers) },
77 { "headers_only", opt_bool|opt_public,
78 (void *)offsetof(transport_instance, headers_only) },
846726c5 79 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
80 (void *)offsetof(transport_instance, remove_headers) },
81 { "headers_rewrite", opt_rewrite|opt_public,
82 (void *)offsetof(transport_instance, headers_rewrite) },
83 { "home_directory", opt_stringptr|opt_public,
84 (void *)offsetof(transport_instance, home_dir) },
85 { "initgroups", opt_bool|opt_public,
86 (void *)offsetof(transport_instance, initgroups) },
fa41615d
JH
87 { "max_parallel", opt_stringptr|opt_public,
88 (void *)offsetof(transport_instance, max_parallel) },
059ec3d9
PH
89 { "message_size_limit", opt_stringptr|opt_public,
90 (void *)offsetof(transport_instance, message_size_limit) },
91 { "rcpt_include_affixes", opt_bool|opt_public,
92 (void *)offsetof(transport_instance, rcpt_include_affixes) },
93 { "retry_use_local_part", opt_bool|opt_public,
94 (void *)offsetof(transport_instance, retry_use_local_part) },
95 { "return_path", opt_stringptr|opt_public,
96 (void *)(offsetof(transport_instance, return_path)) },
97 { "return_path_add", opt_bool|opt_public,
98 (void *)(offsetof(transport_instance, return_path_add)) },
99 { "shadow_condition", opt_stringptr|opt_public,
100 (void *)offsetof(transport_instance, shadow_condition) },
101 { "shadow_transport", opt_stringptr|opt_public,
102 (void *)offsetof(transport_instance, shadow) },
103 { "transport_filter", opt_stringptr|opt_public,
104 (void *)offsetof(transport_instance, filter_command) },
105 { "transport_filter_timeout", opt_time|opt_public,
106 (void *)offsetof(transport_instance, filter_timeout) },
107 { "user", opt_expand_uid|opt_public,
108 (void *)offsetof(transport_instance, uid) }
109};
110
c0b9d3e8 111int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9
PH
112
113
c0b9d3e8
JH
114void
115readconf_options_transports(void)
116{
117struct transport_info * ti;
118
383832ef 119readconf_options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
c0b9d3e8
JH
120
121for (ti = transports_available; ti->driver_name[0]; ti++)
4945f557 122 {
383832ef
HSHR
123 macro_create(string_sprintf("_DRIVER_TRANSPORT_%T", ti->driver_name), US"y", FALSE, TRUE);
124 readconf_options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
4945f557 125 }
383832ef 126}
c0b9d3e8 127
059ec3d9
PH
128/*************************************************
129* Initialize transport list *
130*************************************************/
131
132/* Read the transports section of the configuration file, and set up a chain of
133transport instances according to its contents. Each transport has generic
134options and may also have its own private options. This function is only ever
135called when transports == NULL. We use generic code in readconf to do most of
136the work. */
137
138void
139transport_init(void)
140{
141transport_instance *t;
142
143readconf_driver_init(US"transport",
144 (driver_instance **)(&transports), /* chain anchor */
145 (driver_info *)transports_available, /* available drivers */
146 sizeof(transport_info), /* size of info block */
147 &transport_defaults, /* default values for generic options */
148 sizeof(transport_instance), /* size of instance block */
149 optionlist_transports, /* generic options */
150 optionlist_transports_size);
151
152/* Now scan the configured transports and check inconsistencies. A shadow
153transport is permitted only for local transports. */
154
7b4c8c1f 155for (t = transports; t; t = t->next)
059ec3d9 156 {
7b4c8c1f
JH
157 if (!t->info->local && t->shadow)
158 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
159 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
160
161 if (t->body_only && t->headers_only)
162 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
163 "%s transport: body_only and headers_only are mutually exclusive",
164 t->name);
165 }
166}
167
168
169
170/*************************************************
171* Write block of data *
172*************************************************/
173
174/* Subroutine called by write_chunk() and at the end of the message actually
175to write a data block. Also called directly by some transports to write
176additional data to the file descriptor (e.g. prefix, suffix).
177
178If a transport wants data transfers to be timed, it sets a non-zero value in
179transport_write_timeout. A non-zero transport_write_timeout causes a timer to
180be set for each block of data written from here. If time runs out, then write()
181fails and provokes an error return. The caller can then inspect sigalrm_seen to
182check for a timeout.
183
184On some systems, if a quota is exceeded during the write, the yield is the
185number of bytes written rather than an immediate error code. This also happens
186on some systems in other cases, for example a pipe that goes away because the
187other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
188get the error codes the first time.
189
190The write() function is also interruptible; the Solaris 2.6 man page says:
191
192 If write() is interrupted by a signal before it writes any
193 data, it will return -1 with errno set to EINTR.
194
195 If write() is interrupted by a signal after it successfully
196 writes some data, it will return the number of bytes written.
197
198To handle these cases, we want to restart the write() to output the remainder
199of the data after a non-negative return from write(), except after a timeout.
200In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
201proper error then occurs. In principle, after an interruption, the second
202write() could suffer the same fate, but we do not want to continue for
203evermore, so stick a maximum repetition count on the loop to act as a
204longstop.
205
206Arguments:
207 fd file descriptor to write to
208 block block of bytes to write
209 len number of bytes to write
210
211Returns: TRUE on success, FALSE on failure (with errno preserved);
212 transport_count is incremented by the number of bytes written
213*/
214
215BOOL
216transport_write_block(int fd, uschar *block, int len)
217{
218int i, rc, save_errno;
958541e9
PH
219int local_timeout = transport_write_timeout;
220
221/* This loop is for handling incomplete writes and other retries. In most
222normal cases, it is only ever executed once. */
059ec3d9
PH
223
224for (i = 0; i < 100; i++)
225 {
226 DEBUG(D_transport)
227 debug_printf("writing data block fd=%d size=%d timeout=%d\n",
958541e9 228 fd, len, local_timeout);
059ec3d9 229
958541e9
PH
230 /* This code makes use of alarm() in order to implement the timeout. This
231 isn't a very tidy way of doing things. Using non-blocking I/O with select()
232 provides a neater approach. However, I don't know how to do this when TLS is
233 in use. */
059ec3d9 234
958541e9
PH
235 if (transport_write_timeout <= 0) /* No timeout wanted */
236 {
237 #ifdef SUPPORT_TLS
817d9f57 238 if (tls_out.active == fd) rc = tls_write(FALSE, block, len); else
958541e9
PH
239 #endif
240 rc = write(fd, block, len);
241 save_errno = errno;
242 }
059ec3d9 243
958541e9 244 /* Timeout wanted. */
059ec3d9 245
958541e9 246 else
059ec3d9 247 {
958541e9 248 alarm(local_timeout);
59932f7d
JH
249#ifdef SUPPORT_TLS
250 if (tls_out.active == fd)
251 rc = tls_write(FALSE, block, len);
252 else
253#endif
254 rc = write(fd, block, len);
958541e9
PH
255 save_errno = errno;
256 local_timeout = alarm(0);
059ec3d9
PH
257 if (sigalrm_seen)
258 {
259 errno = ETIMEDOUT;
260 return FALSE;
261 }
262 }
263
264 /* Hopefully, the most common case is success, so test that first. */
265
266 if (rc == len) { transport_count += len; return TRUE; }
267
958541e9
PH
268 /* A non-negative return code is an incomplete write. Try again for the rest
269 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
270
271 if (rc >= 0)
272 {
273 len -= rc;
274 block += rc;
275 transport_count += rc;
276 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 277 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
278 }
279
280 /* A negative return code with an EINTR error is another form of
281 incomplete write, zero bytes having been written */
282
283 if (save_errno == EINTR)
284 {
285 DEBUG(D_transport)
286 debug_printf("write interrupted before anything written\n");
958541e9 287 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
288 }
289
290 /* A response of EAGAIN from write() is likely only in the case of writing
291 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
292
293 if (save_errno == EAGAIN)
294 {
295 DEBUG(D_transport)
296 debug_printf("write temporarily locked out, waiting 1 sec\n");
297 sleep(1);
958541e9
PH
298
299 /* Before continuing to try another write, check that we haven't run out of
300 time. */
301
302 CHECK_TIMEOUT:
303 if (transport_write_timeout > 0 && local_timeout <= 0)
304 {
305 errno = ETIMEDOUT;
306 return FALSE;
307 }
059ec3d9
PH
308 continue;
309 }
310
311 /* Otherwise there's been an error */
312
313 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
314 strerror(save_errno));
315 errno = save_errno;
316 return FALSE;
317 }
318
319/* We've tried and tried and tried but still failed */
320
321errno = ERRNO_WRITEINCOMPLETE;
322return FALSE;
323}
324
325
326
327
328/*************************************************
329* Write formatted string *
330*************************************************/
331
332/* This is called by various transports. It is a convenience function.
333
334Arguments:
335 fd file descriptor
336 format string format
337 ... arguments for format
338
339Returns: the yield of transport_write_block()
340*/
341
342BOOL
1ba28e2b 343transport_write_string(int fd, const char *format, ...)
059ec3d9
PH
344{
345va_list ap;
346va_start(ap, format);
347if (!string_vformat(big_buffer, big_buffer_size, format, ap))
348 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
349va_end(ap);
350return transport_write_block(fd, big_buffer, Ustrlen(big_buffer));
351}
352
353
354
355
356/*************************************************
357* Write character chunk *
358*************************************************/
359
360/* Subroutine used by transport_write_message() to scan character chunks for
361newlines and act appropriately. The object is to minimise the number of writes.
362The output byte stream is buffered up in deliver_out_buffer, which is written
363only when it gets full, thus minimizing write operations and TCP packets.
364
365Static data is used to handle the case when the last character of the previous
366chunk was NL, or matched part of the data that has to be escaped.
367
368Arguments:
369 fd file descript to write to
370 chunk pointer to data to write
371 len length of data to write
6d5c916c 372 tctx transport context - processing to be done during output
059ec3d9
PH
373
374In addition, the static nl_xxx variables must be set as required.
375
376Returns: TRUE on success, FALSE on failure (with errno preserved)
377*/
378
379static BOOL
6d5c916c 380write_chunk(int fd, transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
381{
382uschar *start = chunk;
383uschar *end = chunk + len;
ff5aac2b 384uschar *ptr;
059ec3d9
PH
385int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
386
387/* The assumption is made that the check string will never stretch over move
388than one chunk since the only time there are partial matches is when copying
389the body in large buffers. There is always enough room in the buffer for an
390escape string, since the loop below ensures this for each character it
391processes, and it won't have stuck in the escape string if it left a partial
392match. */
393
394if (nl_partial_match >= 0)
395 {
396 if (nl_check_length > 0 && len >= nl_check_length &&
397 Ustrncmp(start, nl_check + nl_partial_match,
398 nl_check_length - nl_partial_match) == 0)
399 {
400 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
401 chunk_ptr += nl_escape_length;
402 start += nl_check_length - nl_partial_match;
403 }
404
405 /* The partial match was a false one. Insert the characters carried over
406 from the previous chunk. */
407
408 else if (nl_partial_match > 0)
409 {
410 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
411 chunk_ptr += nl_partial_match;
412 }
413
414 nl_partial_match = -1;
415 }
416
417/* Now process the characters in the chunk. Whenever we hit a newline we check
418for possible escaping. The code for the non-NL route should be as fast as
419possible. */
420
421for (ptr = start; ptr < end; ptr++)
422 {
6d5c916c 423 int ch, len;
059ec3d9
PH
424
425 /* Flush the buffer if it has reached the threshold - we want to leave enough
426 room for the next uschar, plus a possible extra CR for an LF, plus the escape
427 string. */
428
6d5c916c 429 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 430 {
58fc5fb2
JH
431 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
432
6d5c916c
JH
433 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
434 from previous SMTP commands. */
435
436 if (tctx && tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2 437 {
d2aa036b 438 if ( tctx->chunk_cb(tctx, (unsigned)len, 0) != OK
58fc5fb2 439 || !transport_write_block(fd, deliver_out_buffer, len)
d2aa036b 440 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
58fc5fb2
JH
441 )
442 return FALSE;
443 }
444 else
445 if (!transport_write_block(fd, deliver_out_buffer, len))
6d5c916c 446 return FALSE;
059ec3d9
PH
447 chunk_ptr = deliver_out_buffer;
448 }
449
450 if ((ch = *ptr) == '\n')
451 {
452 int left = end - ptr - 1; /* count of chars left after NL */
453
454 /* Insert CR before NL if required */
455
6d5c916c 456 if (tctx && tctx->options & topt_use_crlf) *chunk_ptr++ = '\r';
059ec3d9 457 *chunk_ptr++ = '\n';
332f5cf3 458 transport_newlines++;
059ec3d9
PH
459
460 /* The check_string test (formerly "from hack") replaces the specific
461 string at the start of a line with an escape string (e.g. "From " becomes
462 ">From " or "." becomes "..". It is a case-sensitive test. The length
463 check above ensures there is always enough room to insert this string. */
464
465 if (nl_check_length > 0)
466 {
467 if (left >= nl_check_length &&
468 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
469 {
470 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
471 chunk_ptr += nl_escape_length;
472 ptr += nl_check_length;
473 }
474
475 /* Handle the case when there isn't enough left to match the whole
476 check string, but there may be a partial match. We remember how many
477 characters matched, and finish processing this chunk. */
478
479 else if (left <= 0) nl_partial_match = 0;
480
481 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
482 {
483 nl_partial_match = left;
484 ptr = end;
485 }
486 }
487 }
488
489 /* Not a NL character */
490
491 else *chunk_ptr++ = ch;
492 }
493
494return TRUE;
495}
496
497
498
499
500/*************************************************
501* Generate address for RCPT TO *
502*************************************************/
503
504/* This function puts together an address for RCPT to, using the caseful
505version of the local part and the caseful version of the domain. If there is no
506prefix or suffix, or if affixes are to be retained, we can just use the
507original address. Otherwise, if there is a prefix but no suffix we can use a
508pointer into the original address. If there is a suffix, however, we have to
509build a new string.
510
511Arguments:
512 addr the address item
513 include_affixes TRUE if affixes are to be included
514
515Returns: a string
516*/
517
518uschar *
519transport_rcpt_address(address_item *addr, BOOL include_affixes)
520{
521uschar *at;
522int plen, slen;
523
524if (include_affixes)
525 {
526 setflag(addr, af_include_affixes); /* Affects logged => line */
527 return addr->address;
528 }
529
530if (addr->suffix == NULL)
531 {
532 if (addr->prefix == NULL) return addr->address;
533 return addr->address + Ustrlen(addr->prefix);
534 }
535
536at = Ustrrchr(addr->address, '@');
537plen = (addr->prefix == NULL)? 0 : Ustrlen(addr->prefix);
538slen = Ustrlen(addr->suffix);
539
540return string_sprintf("%.*s@%s", (at - addr->address - plen - slen),
541 addr->address + plen, at + 1);
542}
543
544
545/*************************************************
546* Output Envelope-To: address & scan duplicates *
547*************************************************/
548
549/* This function is called from internal_transport_write_message() below, when
550generating an Envelope-To: header line. It checks for duplicates of the given
551address and its ancestors. When one is found, this function calls itself
552recursively, to output the envelope address of the duplicate.
553
554We want to avoid duplication in the list, which can arise for example when
555A->B,C and then both B and C alias to D. This can also happen when there are
556unseen drivers in use. So a list of addresses that have been output is kept in
557the plist variable.
558
559It is also possible to have loops in the address ancestry/duplication graph,
560for example if there are two top level addresses A and B and we have A->B,C and
561B->A. To break the loop, we use a list of processed addresses in the dlist
562variable.
563
564After handling duplication, this function outputs the progenitor of the given
565address.
566
567Arguments:
568 p the address we are interested in
569 pplist address of anchor of the list of addresses not to output
570 pdlist address of anchor of the list of processed addresses
571 first TRUE if this is the first address; set it FALSE afterwards
572 fd the file descriptor to write to
6d5c916c 573 tctx transport context - processing to be done during output
059ec3d9
PH
574
575Returns: FALSE if writing failed
576*/
577
578static BOOL
579write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
6d5c916c 580 BOOL *first, int fd, transport_ctx * tctx)
059ec3d9
PH
581{
582address_item *pp;
583struct aci *ppp;
584
585/* Do nothing if we have already handled this address. If not, remember it
586so that we don't handle it again. */
587
ff5aac2b 588for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9
PH
589
590ppp = store_get(sizeof(struct aci));
591ppp->next = *pdlist;
592*pdlist = ppp;
593ppp->ptr = p;
594
595/* Now scan up the ancestry, checking for duplicates at each generation. */
596
597for (pp = p;; pp = pp->parent)
598 {
599 address_item *dup;
ff5aac2b
JH
600 for (dup = addr_duplicate; dup; dup = dup->next)
601 if (dup->dupof == pp) /* a dup of our address */
6d5c916c 602 if (!write_env_to(dup, pplist, pdlist, first, fd, tctx))
ff5aac2b
JH
603 return FALSE;
604 if (!pp->parent) break;
059ec3d9
PH
605 }
606
607/* Check to see if we have already output the progenitor. */
608
ff5aac2b
JH
609for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
610if (ppp) return TRUE;
059ec3d9
PH
611
612/* Remember what we have output, and output it. */
613
614ppp = store_get(sizeof(struct aci));
615ppp->next = *pplist;
616*pplist = ppp;
617ppp->ptr = pp;
618
6d5c916c 619if (!*first && !write_chunk(fd, tctx, US",\n ", 3)) return FALSE;
059ec3d9 620*first = FALSE;
6d5c916c 621return write_chunk(fd, tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
622}
623
624
625
626
4c04137d 627/* Add/remove/rewrite headers, and send them plus the empty-line separator.
511a6c14
JH
628
629Globals:
630 header_list
631
632Arguments:
633 addr (chain of) addresses (for extra headers), or NULL;
634 only the first address is used
635 fd file descriptor to write the message to
44bc8f0c 636 tctx transport context
59932f7d 637 sendfn function for output (transport or verify)
511a6c14
JH
638
639Returns: TRUE on success; FALSE on failure.
640*/
641BOOL
6d5c916c
JH
642transport_headers_send(int fd, transport_ctx * tctx,
643 BOOL (*sendfn)(int fd, transport_ctx * tctx, uschar * s, int len))
511a6c14
JH
644{
645header_line *h;
6b46ecc6 646const uschar *list;
6d5c916c
JH
647transport_instance * tblock = tctx ? tctx->tblock : NULL;
648address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
649
650/* Then the message's headers. Don't write any that are flagged as "old";
651that means they were rewritten, or are a record of envelope rewriting, or
652were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
653match any entries therein. It is a colon-sep list; expand the items
654separately and squash any empty ones.
d43cbe25 655Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 656
ff5aac2b 657for (h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14
JH
658 {
659 int i;
511a6c14
JH
660 BOOL include_header = TRUE;
661
fb6833e0 662 list = tblock ? tblock->remove_headers : NULL;
d43cbe25 663 for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
664 {
665 if (list)
666 {
667 int sep = ':'; /* This is specified as a colon-separated list */
668 uschar *s, *ss;
8bc732e8 669 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 670 {
76146973
JH
671 int len;
672
673 if (i == 0)
674 if (!(s = expand_string(s)) && !expand_string_forcedfail)
675 {
676 errno = ERRNO_CHHEADER_FAIL;
677 return FALSE;
678 }
d4ff61d1 679 len = s ? Ustrlen(s) : 0;
511a6c14
JH
680 if (strncmpic(h->text, s, len) != 0) continue;
681 ss = h->text + len;
682 while (*ss == ' ' || *ss == '\t') ss++;
683 if (*ss == ':') break;
684 }
6b46ecc6 685 if (s) { include_header = FALSE; break; }
511a6c14 686 }
6b46ecc6 687 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
688 }
689
690 /* If this header is to be output, try to rewrite it if there are rewriting
691 rules. */
692
693 if (include_header)
694 {
fb6833e0 695 if (tblock && tblock->rewrite_rules)
511a6c14
JH
696 {
697 void *reset_point = store_get(0);
698 header_line *hh;
699
6b46ecc6
JH
700 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
701 tblock->rewrite_existflags, FALSE)))
511a6c14 702 {
6d5c916c 703 if (!sendfn(fd, tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
704 store_reset(reset_point);
705 continue; /* With the next header line */
706 }
707 }
708
709 /* Either no rewriting rules, or it didn't get rewritten */
710
6d5c916c 711 if (!sendfn(fd, tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
712 }
713
714 /* Header removed */
715
716 else
717 {
718 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
719 }
720 }
721
722/* Add on any address-specific headers. If there are multiple addresses,
723they will all have the same headers in order to be batched. The headers
724are chained in reverse order of adding (so several addresses from the
725same alias might share some of them) but we want to output them in the
726opposite order. This is a bit tedious, but there shouldn't be very many
727of them. We just walk the list twice, reversing the pointers each time,
728but on the second time, write out the items.
729
730Headers added to an address by a router are guaranteed to end with a newline.
731*/
732
733if (addr)
734 {
735 int i;
d43cbe25 736 header_line *hprev = addr->prop.extra_headers;
511a6c14
JH
737 header_line *hnext;
738 for (i = 0; i < 2; i++)
ff5aac2b 739 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
740 {
741 hnext = h->next;
742 h->next = hprev;
743 hprev = h;
744 if (i == 1)
745 {
6d5c916c 746 if (!sendfn(fd, tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
747 DEBUG(D_transport)
748 debug_printf("added header line(s):\n%s---\n", h->text);
749 }
750 }
511a6c14
JH
751 }
752
76146973
JH
753/* If a string containing additional headers exists it is a newline-sep
754list. Expand each item and write out the result. This is done last so that
755if it (deliberately or accidentally) isn't in header format, it won't mess
756up any other headers. An empty string or a forced expansion failure are
757noops. An added header string from a transport may not end with a newline;
758add one if it does not. */
511a6c14 759
fb6833e0 760if (tblock && (list = CUS tblock->add_headers))
511a6c14 761 {
76146973
JH
762 int sep = '\n';
763 uschar * s;
764
6b46ecc6 765 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 766 if ((s = expand_string(s)))
76146973
JH
767 {
768 int len = Ustrlen(s);
769 if (len > 0)
511a6c14 770 {
6d5c916c
JH
771 if (!sendfn(fd, tctx, s, len)) return FALSE;
772 if (s[len-1] != '\n' && !sendfn(fd, tctx, US"\n", 1))
76146973
JH
773 return FALSE;
774 DEBUG(D_transport)
775 {
776 debug_printf("added header line:\n%s", s);
777 if (s[len-1] != '\n') debug_printf("\n");
778 debug_printf("---\n");
779 }
511a6c14
JH
780 }
781 }
ff5aac2b
JH
782 else if (!expand_string_forcedfail)
783 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
784 }
785
786/* Separate headers from body with a blank line */
787
6d5c916c 788return sendfn(fd, tctx, US"\n", 1);
511a6c14
JH
789}
790
791
059ec3d9
PH
792/*************************************************
793* Write the message *
794*************************************************/
795
796/* This function writes the message to the given file descriptor. The headers
797are in the in-store data structure, and the rest of the message is in the open
798file descriptor deliver_datafile. Make sure we start it at the beginning.
799
800. If add_return_path is TRUE, a "return-path:" header is added to the message,
801 containing the envelope sender's address.
802
803. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
804 giving the top-level envelope address that caused this delivery to happen.
805
806. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
807 message. It gives the time and date that delivery took place.
808
809. If check_string is not null, the start of each line is checked for that
810 string. If it is found, it is replaced by escape_string. This used to be
811 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
812
813. If use_crlf is true, newlines are turned into CRLF (SMTP output).
814
815The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
816any writing or reading error, leaving the code in errno intact. Error exits
817can include timeouts for certain transports, which are requested by setting
818transport_write_timeout non-zero.
819
820Arguments:
059ec3d9 821 fd file descriptor to write the message to
6b46ecc6
JH
822 tctx
823 addr (chain of) addresses (for extra headers), or NULL;
824 only the first address is used
825 tblock optional transport instance block (NULL signifies NULL/0):
826 add_headers a string containing one or more headers to add; it is
827 expanded, and must be in correct RFC 822 format as
828 it is transmitted verbatim; NULL => no additions,
829 and so does empty string or forced expansion fail
830 remove_headers a colon-separated list of headers to remove, or NULL
831 rewrite_rules chain of header rewriting rules
832 rewrite_existflags flags for the rewriting rules
833 options bit-wise options:
834 add_return_path if TRUE, add a "return-path" header
835 add_envelope_to if TRUE, add a "envelope-to" header
836 add_delivery_date if TRUE, add a "delivery-date" header
837 use_crlf if TRUE, turn NL into CR LF
838 end_dot if TRUE, send a terminating "." line at the end
839 no_headers if TRUE, omit the headers
840 no_body if TRUE, omit the body
2d14f397
JH
841 check_string a string to check for at the start of lines, or NULL
842 escape_string a string to insert in front of any check string
843 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
844 it is used when returning messages to their senders,
845 and is approximate rather than exact, owing to chunk
846 buffering
059ec3d9
PH
847
848Returns: TRUE on success; FALSE (with errno) on failure.
849 In addition, the global variable transport_count
850 is incremented by the number of bytes written.
851*/
852
853static BOOL
6b46ecc6 854internal_transport_write_message(int fd, transport_ctx * tctx, int size_limit)
059ec3d9 855{
059ec3d9 856int len;
059ec3d9
PH
857
858/* Initialize pointer in output buffer. */
859
860chunk_ptr = deliver_out_buffer;
861
862/* Set up the data for start-of-line data checking and escaping */
863
864nl_partial_match = -1;
6b46ecc6 865if (tctx->check_string && tctx->escape_string)
059ec3d9 866 {
6b46ecc6 867 nl_check = tctx->check_string;
059ec3d9 868 nl_check_length = Ustrlen(nl_check);
6b46ecc6 869 nl_escape = tctx->escape_string;
059ec3d9
PH
870 nl_escape_length = Ustrlen(nl_escape);
871 }
ff5aac2b
JH
872else
873 nl_check_length = nl_escape_length = 0;
059ec3d9
PH
874
875/* Whether the escaping mechanism is applied to headers or not is controlled by
876an option (set for SMTP, not otherwise). Negate the length if not wanted till
877after the headers. */
878
6b46ecc6 879if (!(tctx->options & topt_escape_headers))
ff5aac2b 880 nl_check_length = -nl_check_length;
059ec3d9
PH
881
882/* Write the headers if required, including any that have to be added. If there
883are header rewriting rules, apply them. */
884
6b46ecc6 885if (!(tctx->options & topt_no_headers))
059ec3d9
PH
886 {
887 /* Add return-path: if requested. */
888
6b46ecc6 889 if (tctx->options & topt_add_return_path)
059ec3d9
PH
890 {
891 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 892 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 893 return_path);
6d5c916c 894 if (!write_chunk(fd, tctx, buffer, n)) return FALSE;
059ec3d9
PH
895 }
896
897 /* Add envelope-to: if requested */
898
6b46ecc6 899 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
900 {
901 BOOL first = TRUE;
902 address_item *p;
903 struct aci *plist = NULL;
904 struct aci *dlist = NULL;
905 void *reset_point = store_get(0);
906
6d5c916c 907 if (!write_chunk(fd, tctx, US"Envelope-to: ", 13)) return FALSE;
059ec3d9
PH
908
909 /* Pick up from all the addresses. The plist and dlist variables are
910 anchors for lists of addresses already handled; they have to be defined at
4c04137d 911 this level because write_env_to() calls itself recursively. */
059ec3d9 912
6b46ecc6 913 for (p = tctx->addr; p; p = p->next)
6d5c916c 914 if (!write_env_to(p, &plist, &dlist, &first, fd, tctx))
ff5aac2b 915 return FALSE;
059ec3d9
PH
916
917 /* Add a final newline and reset the store used for tracking duplicates */
918
6d5c916c 919 if (!write_chunk(fd, tctx, US"\n", 1)) return FALSE;
059ec3d9
PH
920 store_reset(reset_point);
921 }
922
923 /* Add delivery-date: if requested. */
924
6b46ecc6 925 if (tctx->options & topt_add_delivery_date)
059ec3d9
PH
926 {
927 uschar buffer[100];
ff5aac2b 928 int n = sprintf(CS buffer, "Delivery-date: %s\n", tod_stamp(tod_full));
6d5c916c 929 if (!write_chunk(fd, tctx, buffer, n)) return FALSE;
059ec3d9
PH
930 }
931
932 /* Then the message's headers. Don't write any that are flagged as "old";
933 that means they were rewritten, or are a record of envelope rewriting, or
934 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 935 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 936 addr is not NULL. */
6b46ecc6 937
6d5c916c 938 if (!transport_headers_send(fd, tctx, &write_chunk))
511a6c14 939 return FALSE;
059ec3d9
PH
940 }
941
58fc5fb2
JH
942/* When doing RFC3030 CHUNKING output, work out how much data would be in a
943last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
944(optimally, all of the headers - but it does not matter if we already had to
945flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 946(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
947that further use of write_chunk() will not prepend BDATs.
948The first BDAT written will also first flush any outstanding MAIL and RCPT
949commands which were buffered thans to PIPELINING.
950Commands go out (using a send()) from a different buffer to data (using a
951write()). They might not end up in the same TCP segment, which is
952suboptimal. */
59932f7d 953
65de12cc 954if (tctx->options & topt_use_bdat)
59932f7d 955 {
e027f545 956 off_t fsize;
4dc2379a 957 int hsize, size = 0;
e027f545
JH
958
959 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
960 hsize = 0;
65de12cc 961 if (!(tctx->options & topt_no_body))
59932f7d
JH
962 {
963 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
964 fsize -= SPOOL_DATA_START_OFFSET;
965 if (size_limit > 0 && fsize > size_limit)
966 fsize = size_limit;
e027f545 967 size = hsize + fsize;
65de12cc 968 if (tctx->options & topt_use_crlf)
59932f7d
JH
969 size += body_linecount; /* account for CRLF-expansion */
970 }
971
e027f545
JH
972 /* If the message is large, emit first a non-LAST chunk with just the
973 headers, and reap the command responses. This lets us error out early
974 on RCPT rejects rather than sending megabytes of data. Include headers
975 on the assumption they are cheap enough and some clever implementations
976 might errorcheck them too, on-the-fly, and reject that chunk. */
977
978 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
979 {
58fc5fb2 980 DEBUG(D_transport)
4c04137d 981 debug_printf("sending small initial BDAT; hsize=%d\n", hsize);
d2aa036b 982 if ( tctx->chunk_cb(tctx, hsize, 0) != OK
e027f545 983 || !transport_write_block(fd, deliver_out_buffer, hsize)
d2aa036b 984 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
e027f545
JH
985 )
986 return FALSE;
987 chunk_ptr = deliver_out_buffer;
988 size -= hsize;
989 }
990
d2aa036b
JH
991 /* Emit a LAST datachunk command, and unmark the context for further
992 BDAT commands. */
59932f7d 993
d2aa036b 994 if (tctx->chunk_cb(tctx, size, tc_chunk_last) != OK)
511a6c14 995 return FALSE;
6d5c916c 996 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
997 }
998
999/* If the body is required, ensure that the data for check strings (formerly
1000the "from hack") is enabled by negating the length if necessary. (It will be
1001negative in cases where it isn't to apply to the headers). Then ensure the body
1002is positioned at the start of its file (following the message id), then write
1003it, applying the size limit if required. */
1004
6b46ecc6 1005if (!(tctx->options & topt_no_body))
059ec3d9 1006 {
e027f545
JH
1007 int size = size_limit;
1008
059ec3d9
PH
1009 nl_check_length = abs(nl_check_length);
1010 nl_partial_match = 0;
d4ff61d1
JH
1011 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1012 return FALSE;
59932f7d
JH
1013 while ( (len = MAX(DELIVER_IN_BUFFER_SIZE, size)) > 0
1014 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1015 {
6d5c916c 1016 if (!write_chunk(fd, tctx, deliver_in_buffer, len))
59932f7d 1017 return FALSE;
e027f545 1018 size -= len;
059ec3d9
PH
1019 }
1020
059ec3d9
PH
1021 /* A read error on the body will have left len == -1 and errno set. */
1022
1023 if (len != 0) return FALSE;
c0940526 1024 }
059ec3d9 1025
c0940526 1026/* Finished with the check string */
059ec3d9 1027
c0940526
PP
1028nl_check_length = nl_escape_length = 0;
1029
1030/* If requested, add a terminating "." line (SMTP output). */
1031
6d5c916c 1032if (tctx->options & topt_end_dot && !write_chunk(fd, tctx, US".\n", 2))
c0940526 1033 return FALSE;
059ec3d9
PH
1034
1035/* Write out any remaining data in the buffer before returning. */
1036
1037return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
1038 transport_write_block(fd, deliver_out_buffer, len);
1039}
1040
1041
80a47a2c 1042#ifndef DISABLE_DKIM
fb2274d4 1043
4cd12fe9
TK
1044/***************************************************************************************************
1045* External interface to write the message, while signing it with DKIM and/or Domainkeys *
1046***************************************************************************************************/
fb2274d4 1047
077ba943
JH
1048/* This function is a wrapper around transport_write_message().
1049 It is only called from the smtp transport if DKIM or Domainkeys support
1050 is compiled in. The function sets up a replacement fd into a -K file,
1051 then calls the normal function. This way, the exact bits that exim would
1052 have put "on the wire" will end up in the file (except for TLS
1053 encapsulation, which is the very very last thing). When we are done
1054 signing the file, send the signed message down the original fd (or TLS fd).
1055
1056Arguments:
ff5aac2b
JH
1057 as for internal_transport_write_message() above, with additional arguments
1058 for DKIM.
f7572e5a
TK
1059
1060Returns: TRUE on success; FALSE (with errno) for any failure
1061*/
1062
1063BOOL
6b46ecc6 1064dkim_transport_write_message(int out_fd, transport_ctx * tctx,
b9df1829 1065 struct ob_dkim * dkim, const uschar ** err)
f7572e5a 1066{
077ba943
JH
1067int dkim_fd;
1068int save_errno = 0;
1069BOOL rc;
41313d92 1070uschar * dkim_spool_name;
b9df1829
JH
1071uschar * dkim_signature = NULL;
1072int sread = 0, wwritten = 0, siglen = 0, options;
ff5aac2b 1073off_t k_file_size;
b9df1829 1074const uschar * errstr;
077ba943
JH
1075
1076/* If we can't sign, just call the original function. */
1077
ff5aac2b 1078if (!(dkim->dkim_private_key && dkim->dkim_domain && dkim->dkim_selector))
6b46ecc6 1079 return transport_write_message(out_fd, tctx, 0);
077ba943 1080
41313d92
JH
1081dkim_spool_name = spool_fname(US"input", message_subdir, message_id,
1082 string_sprintf("-%d-K", (int)getpid()));
077ba943
JH
1083
1084if ((dkim_fd = Uopen(dkim_spool_name, O_RDWR|O_CREAT|O_TRUNC, SPOOL_MODE)) < 0)
1085 {
1086 /* Can't create spool file. Ugh. */
1087 rc = FALSE;
1088 save_errno = errno;
b9df1829 1089 *err = string_sprintf("dkim spoolfile create: %s", strerror(errno));
077ba943 1090 goto CLEANUP;
4cd12fe9
TK
1091 }
1092
e983e85a
JH
1093/* Call original function to write the -K file; does the CRLF expansion
1094(but, in the CHUNKING case, not dot-stuffing and dot-termination). */
f7572e5a 1095
48f1c853 1096options = tctx->options;
65de12cc 1097tctx->options &= ~topt_use_bdat;
6b46ecc6 1098rc = transport_write_message(dkim_fd, tctx, 0);
48f1c853 1099tctx->options = options;
f7572e5a 1100
077ba943
JH
1101/* Save error state. We must clean up before returning. */
1102if (!rc)
1103 {
1104 save_errno = errno;
1105 goto CLEANUP;
1106 }
f7572e5a 1107
48f1c853 1108/* Rewind file and feed it to the goats^W DKIM lib */
e983e85a 1109dkim->dot_stuffed = !!(options & topt_end_dot);
48f1c853 1110lseek(dkim_fd, 0, SEEK_SET);
b9df1829 1111if ((dkim_signature = dkim_exim_sign(dkim_fd, dkim, &errstr)))
48f1c853
JH
1112 siglen = Ustrlen(dkim_signature);
1113else if (dkim->dkim_strict)
077ba943 1114 {
48f1c853
JH
1115 uschar *dkim_strict_result = expand_string(dkim->dkim_strict);
1116 if (dkim_strict_result)
1117 if ( (strcmpic(dkim->dkim_strict,US"1") == 0) ||
1118 (strcmpic(dkim->dkim_strict,US"true") == 0) )
077ba943 1119 {
48f1c853
JH
1120 /* Set errno to something halfway meaningful */
1121 save_errno = EACCES;
1122 log_write(0, LOG_MAIN, "DKIM: message could not be signed,"
1123 " and dkim_strict is set. Deferring message delivery.");
b9df1829 1124 *err = errstr;
48f1c853
JH
1125 rc = FALSE;
1126 goto CLEANUP;
f7572e5a 1127 }
e520153e 1128 }
ff5aac2b 1129
48f1c853
JH
1130#ifndef HAVE_LINUX_SENDFILE
1131if (options & topt_use_bdat)
6b46ecc6 1132#endif
48f1c853
JH
1133 k_file_size = lseek(dkim_fd, 0, SEEK_END); /* Fetch file size */
1134
1135if (options & topt_use_bdat)
e520153e 1136 {
48f1c853
JH
1137
1138 /* On big messages output a precursor chunk to get any pipelined
1139 MAIL & RCPT commands flushed, then reap the responses so we can
1140 error out on RCPT rejects before sending megabytes. */
1141
1142 if (siglen + k_file_size > DELIVER_OUT_BUFFER_SIZE && siglen > 0)
077ba943 1143 {
d2aa036b 1144 if ( tctx->chunk_cb(tctx, siglen, 0) != OK
48f1c853 1145 || !transport_write_block(out_fd, dkim_signature, siglen)
d2aa036b 1146 || tctx->chunk_cb(tctx, 0, tc_reap_prev) != OK
48f1c853
JH
1147 )
1148 goto err;
1149 siglen = 0;
f7572e5a 1150 }
48f1c853 1151
d2aa036b
JH
1152 /* Send the BDAT command for the entire message, as a single LAST-marked
1153 chunk. */
1154
1155 if (tctx->chunk_cb(tctx, siglen + k_file_size, tc_chunk_last) != OK)
48f1c853 1156 goto err;
4cd12fe9 1157 }
f7572e5a 1158
48f1c853
JH
1159if(siglen > 0 && !transport_write_block(out_fd, dkim_signature, siglen))
1160 goto err;
1161
f7572e5a 1162#ifdef HAVE_LINUX_SENDFILE
077ba943
JH
1163/* We can use sendfile() to shove the file contents
1164 to the socket. However only if we don't use TLS,
1165 as then there's another layer of indirection
1166 before the data finally hits the socket. */
ff5aac2b 1167if (tls_out.active != out_fd)
077ba943
JH
1168 {
1169 ssize_t copied = 0;
1170 off_t offset = 0;
e8bc7fca
JH
1171
1172 /* Rewind file */
1173 lseek(dkim_fd, 0, SEEK_SET);
1174
ff5aac2b
JH
1175 while(copied >= 0 && offset < k_file_size)
1176 copied = sendfile(out_fd, dkim_fd, &offset, k_file_size - offset);
077ba943 1177 if (copied < 0)
48f1c853 1178 goto err;
077ba943 1179 }
e8bc7fca
JH
1180else
1181
f7572e5a
TK
1182#endif
1183
077ba943 1184 {
e8bc7fca
JH
1185 /* Rewind file */
1186 lseek(dkim_fd, 0, SEEK_SET);
077ba943 1187
e8bc7fca 1188 /* Send file down the original fd */
ff5aac2b 1189 while((sread = read(dkim_fd, deliver_out_buffer, DELIVER_OUT_BUFFER_SIZE)) >0)
f7572e5a 1190 {
4dc2379a 1191 uschar * p = deliver_out_buffer;
e8bc7fca
JH
1192 /* write the chunk */
1193
1194 while (sread)
1195 {
077ba943 1196#ifdef SUPPORT_TLS
ff5aac2b 1197 wwritten = tls_out.active == out_fd
4dc2379a
JH
1198 ? tls_write(FALSE, p, sread)
1199 : write(out_fd, CS p, sread);
077ba943 1200#else
4dc2379a 1201 wwritten = write(out_fd, CS p, sread);
077ba943 1202#endif
e8bc7fca 1203 if (wwritten == -1)
48f1c853 1204 goto err;
e8bc7fca
JH
1205 p += wwritten;
1206 sread -= wwritten;
f7572e5a 1207 }
f7572e5a
TK
1208 }
1209
e8bc7fca
JH
1210 if (sread == -1)
1211 {
1212 save_errno = errno;
1213 rc = FALSE;
1214 }
077ba943 1215 }
f7572e5a 1216
077ba943 1217CLEANUP:
48f1c853
JH
1218 /* unlink -K file */
1219 (void)close(dkim_fd);
1220 Uunlink(dkim_spool_name);
1221 errno = save_errno;
1222 return rc;
1223
1224err:
1225 save_errno = errno;
1226 rc = FALSE;
1227 goto CLEANUP;
f7572e5a 1228}
80a47a2c 1229
f7572e5a
TK
1230#endif
1231
1232
1233
059ec3d9
PH
1234/*************************************************
1235* External interface to write the message *
1236*************************************************/
1237
1238/* If there is no filtering required, call the internal function above to do
1239the real work, passing over all the arguments from this function. Otherwise,
1240set up a filtering process, fork another process to call the internal function
1241to write to the filter, and in this process just suck from the filter and write
1242down the given fd. At the end, tidy up the pipes and the processes.
1243
1244Arguments: as for internal_transport_write_message() above
1245
1246Returns: TRUE on success; FALSE (with errno) for any failure
1247 transport_count is incremented by the number of bytes written
1248*/
1249
1250BOOL
6b46ecc6 1251transport_write_message(int fd, transport_ctx * tctx, int size_limit)
059ec3d9 1252{
059ec3d9
PH
1253BOOL last_filter_was_NL = TRUE;
1254int rc, len, yield, fd_read, fd_write, save_errno;
806c3df9 1255int pfd[2] = {-1, -1};
059ec3d9 1256pid_t filter_pid, write_pid;
6d5c916c 1257static transport_ctx dummy_tctx = {0};
6b46ecc6
JH
1258
1259if (!tctx) tctx = &dummy_tctx;
059ec3d9 1260
2e2a30b4
PH
1261transport_filter_timed_out = FALSE;
1262
059ec3d9
PH
1263/* If there is no filter command set up, call the internal function that does
1264the actual work, passing it the incoming fd, and return its result. */
1265
6ff55e50
JH
1266if ( !transport_filter_argv
1267 || !*transport_filter_argv
1268 || !**transport_filter_argv
1269 )
6b46ecc6 1270 return internal_transport_write_message(fd, tctx, size_limit);
059ec3d9
PH
1271
1272/* Otherwise the message must be written to a filter process and read back
1273before being written to the incoming fd. First set up the special processing to
1274be done during the copying. */
1275
059ec3d9
PH
1276nl_partial_match = -1;
1277
6b46ecc6 1278if (tctx->check_string && tctx->escape_string)
059ec3d9 1279 {
6b46ecc6 1280 nl_check = tctx->check_string;
059ec3d9 1281 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1282 nl_escape = tctx->escape_string;
059ec3d9
PH
1283 nl_escape_length = Ustrlen(nl_escape);
1284 }
1285else nl_check_length = nl_escape_length = 0;
1286
1287/* Start up a subprocess to run the command. Ensure that our main fd will
1288be closed when the subprocess execs, but remove the flag afterwards.
1289(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1290process to deliver another message.) We get back stdin/stdout file descriptors.
1291If the process creation failed, give an error return. */
1292
1293fd_read = -1;
1294fd_write = -1;
1295save_errno = 0;
1296yield = FALSE;
1297write_pid = (pid_t)(-1);
1298
92b0827a
JH
1299 {
1300 int bits = fcntl(fd, F_GETFD);
1301 (void)fcntl(fd, F_SETFD, bits | FD_CLOEXEC);
1302 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
1303 &fd_write, &fd_read, FALSE);
1304 (void)fcntl(fd, F_SETFD, bits & ~FD_CLOEXEC);
1305 }
059ec3d9
PH
1306if (filter_pid < 0) goto TIDY_UP; /* errno set */
1307
1308DEBUG(D_transport)
fb6833e0 1309 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1310 (int)filter_pid, fd_write, fd_read);
1311
1312/* Fork subprocess to write the message to the filter, and return the result
1313via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1314smtp dots, or check string processing. */
1315
1316if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
1317if ((write_pid = fork()) == 0)
1318 {
1319 BOOL rc;
f1e894f3
PH
1320 (void)close(fd_read);
1321 (void)close(pfd[pipe_read]);
059ec3d9 1322 nl_check_length = nl_escape_length = 0;
6b46ecc6 1323
fb6833e0 1324 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1325 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6
JH
1326
1327 rc = internal_transport_write_message(fd_write, tctx, size_limit);
1328
059ec3d9 1329 save_errno = errno;
1ac6b2e7
JH
1330 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1331 != sizeof(BOOL)
1332 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1333 != sizeof(int)
6b46ecc6 1334 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1ac6b2e7
JH
1335 != sizeof(int)
1336 )
1337 rc = FALSE; /* compiler quietening */
059ec3d9
PH
1338 _exit(0);
1339 }
1340save_errno = errno;
1341
1342/* Parent process: close our copy of the writing subprocess' pipes. */
1343
f1e894f3
PH
1344(void)close(pfd[pipe_write]);
1345(void)close(fd_write);
059ec3d9
PH
1346fd_write = -1;
1347
1348/* Writing process creation failed */
1349
1350if (write_pid < 0)
1351 {
1352 errno = save_errno; /* restore */
1353 goto TIDY_UP;
1354 }
1355
1356/* When testing, let the subprocess get going */
1357
1358if (running_in_test_harness) millisleep(250);
1359
1360DEBUG(D_transport)
1361 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1362
1363/* Copy the message from the filter to the output fd. A read error leaves len
1364== -1 and errno set. We need to apply a timeout to the read, to cope with
1365the case when the filter gets stuck, but it can be quite a long one. The
1366default is 5m, but this is now configurable. */
1367
1368DEBUG(D_transport) debug_printf("copying from the filter\n");
1369
1370/* Copy the output of the filter, remembering if the last character was NL. If
1371no data is returned, that counts as "ended with NL" (default setting of the
1372variable is TRUE). */
1373
1374chunk_ptr = deliver_out_buffer;
1375
1376for (;;)
1377 {
1378 sigalrm_seen = FALSE;
1379 alarm(transport_filter_timeout);
1380 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
1381 alarm(0);
1382 if (sigalrm_seen)
1383 {
1384 errno = ETIMEDOUT;
2e2a30b4 1385 transport_filter_timed_out = TRUE;
059ec3d9
PH
1386 goto TIDY_UP;
1387 }
1388
1389 /* If the read was successful, write the block down the original fd,
1390 remembering whether it ends in \n or not. */
1391
1392 if (len > 0)
1393 {
6d5c916c 1394 if (!write_chunk(fd, tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1395 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1396 }
1397
1398 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1399
1400 else
1401 {
1402 if (len == 0) yield = TRUE;
1403 break;
1404 }
1405 }
1406
1407/* Tidying up code. If yield = FALSE there has been an error and errno is set
1408to something. Ensure the pipes are all closed and the processes are removed. If
1409there has been an error, kill the processes before waiting for them, just to be
1410sure. Also apply a paranoia timeout. */
1411
1412TIDY_UP:
1413save_errno = errno;
1414
f1e894f3
PH
1415(void)close(fd_read);
1416if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1417
1418if (!yield)
1419 {
1420 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1421 if (write_pid > 0) kill(write_pid, SIGKILL);
1422 }
1423
1424/* Wait for the filter process to complete. */
1425
1426DEBUG(D_transport) debug_printf("waiting for filter process\n");
1427if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1428 {
1429 yield = FALSE;
1430 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1431 tctx->addr->more_errno = rc;
059ec3d9
PH
1432 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1433 }
1434
1435/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1436read the results from its pipe, provided we haven't already had a filter
35af9f61 1437process failure. */
059ec3d9
PH
1438
1439DEBUG(D_transport) debug_printf("waiting for writing process\n");
1440if (write_pid > 0)
1441 {
35af9f61
PH
1442 rc = child_close(write_pid, 30);
1443 if (yield)
8e669ac1 1444 if (rc == 0)
35af9f61
PH
1445 {
1446 BOOL ok;
4dc2379a
JH
1447 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
1448 {
1449 DEBUG(D_transport)
1450 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1451 save_errno = ERRNO_FILTER_FAIL;
1452 yield = FALSE;
1453 }
1454 else if (!ok)
35af9f61 1455 {
4dc2379a 1456 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
6b46ecc6 1457 dummy = read(pfd[pipe_read], (void *)&(tctx->addr->more_errno), sizeof(int));
35af9f61
PH
1458 yield = FALSE;
1459 }
1460 }
1461 else
059ec3d9 1462 {
059ec3d9 1463 yield = FALSE;
35af9f61 1464 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1465 tctx->addr->more_errno = rc;
35af9f61 1466 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1467 }
059ec3d9 1468 }
f1e894f3 1469(void)close(pfd[pipe_read]);
059ec3d9
PH
1470
1471/* If there have been no problems we can now add the terminating "." if this is
1472SMTP output, turning off escaping beforehand. If the last character from the
1473filter was not NL, insert a NL to make the SMTP protocol work. */
1474
1475if (yield)
1476 {
1477 nl_check_length = nl_escape_length = 0;
6b46ecc6 1478 if ( tctx->options & topt_end_dot
ff5aac2b 1479 && ( last_filter_was_NL
6d5c916c
JH
1480 ? !write_chunk(fd, tctx, US".\n", 2)
1481 : !write_chunk(fd, tctx, US"\n.\n", 3)
ff5aac2b 1482 ) )
059ec3d9 1483 yield = FALSE;
059ec3d9
PH
1484
1485 /* Write out any remaining data in the buffer. */
1486
1487 else
ff5aac2b
JH
1488 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
1489 || transport_write_block(fd, deliver_out_buffer, len);
059ec3d9 1490 }
ff5aac2b
JH
1491else
1492 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1493
1494DEBUG(D_transport)
1495 {
1496 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1497 if (!yield)
6b46ecc6 1498 debug_printf("errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1499 }
1500
1501return yield;
1502}
1503
1504
1505
1506
1507
1508/*************************************************
1509* Update waiting database *
1510*************************************************/
1511
1512/* This is called when an address is deferred by remote transports that are
1513capable of sending more than one message over one connection. A database is
1514maintained for each transport, keeping track of which messages are waiting for
1515which hosts. The transport can then consult this when eventually a successful
1516delivery happens, and if it finds that another message is waiting for the same
1517host, it can fire up a new process to deal with it using the same connection.
1518
1519The database records are keyed by host name. They can get full if there are
1520lots of messages waiting, and so there is a continuation mechanism for them.
1521
1522Each record contains a list of message ids, packed end to end without any
1523zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1524in this record, and the sequence field says if there are any other records for
1525this host. If the sequence field is 0, there are none. If it is 1, then another
1526record with the name <hostname>:0 exists; if it is 2, then two other records
1527with sequence numbers 0 and 1 exist, and so on.
1528
1529Currently, an exhaustive search of all continuation records has to be done to
1530determine whether to add a message id to a given record. This shouldn't be
1531too bad except in extreme cases. I can't figure out a *simple* way of doing
1532better.
1533
1534Old records should eventually get swept up by the exim_tidydb utility.
1535
1536Arguments:
f6c332bd 1537 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1538 tpname name of the transport
1539
1540Returns: nothing
1541*/
1542
1543void
1544transport_update_waiting(host_item *hostlist, uschar *tpname)
1545{
55414b25 1546const uschar *prevname = US"";
059ec3d9
PH
1547host_item *host;
1548open_db dbblock;
1549open_db *dbm_file;
1550
7a0743eb
PH
1551DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1552
059ec3d9
PH
1553/* Open the database for this transport */
1554
36d295f1
JH
1555if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
1556 O_RDWR, &dbblock, TRUE)))
1557 return;
059ec3d9
PH
1558
1559/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1560that the message id is in each host record. */
059ec3d9 1561
36d295f1 1562for (host = hostlist; host; host = host->next)
059ec3d9
PH
1563 {
1564 BOOL already = FALSE;
1565 dbdata_wait *host_record;
1566 uschar *s;
1567 int i, host_length;
36d295f1 1568 uschar buffer[256];
059ec3d9 1569
059ec3d9
PH
1570 /* Skip if this is the same host as we just processed; otherwise remember
1571 the name for next time. */
1572
1573 if (Ustrcmp(prevname, host->name) == 0) continue;
1574 prevname = host->name;
1575
1576 /* Look up the host record; if there isn't one, make an empty one. */
1577
36d295f1 1578 if (!(host_record = dbfn_read(dbm_file, host->name)))
059ec3d9
PH
1579 {
1580 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH);
1581 host_record->count = host_record->sequence = 0;
1582 }
1583
1584 /* Compute the current length */
1585
1586 host_length = host_record->count * MESSAGE_ID_LENGTH;
1587
1588 /* Search the record to see if the current message is already in it. */
1589
1590 for (s = host_record->text; s < host_record->text + host_length;
1591 s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1592 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1593 { already = TRUE; break; }
059ec3d9
PH
1594
1595 /* If we haven't found this message in the main record, search any
1596 continuation records that exist. */
1597
1598 for (i = host_record->sequence - 1; i >= 0 && !already; i--)
1599 {
1600 dbdata_wait *cont;
1601 sprintf(CS buffer, "%.200s:%d", host->name, i);
36d295f1 1602 if ((cont = dbfn_read(dbm_file, buffer)))
059ec3d9
PH
1603 {
1604 int clen = cont->count * MESSAGE_ID_LENGTH;
1605 for (s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
059ec3d9
PH
1606 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1607 { already = TRUE; break; }
059ec3d9
PH
1608 }
1609 }
1610
1611 /* If this message is already in a record, no need to update. */
1612
7a0743eb
PH
1613 if (already)
1614 {
1615 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1616 continue;
1617 }
059ec3d9
PH
1618
1619
1620 /* If this record is full, write it out with a new name constructed
1621 from the sequence number, increase the sequence number, and empty
1622 the record. */
1623
1624 if (host_record->count >= WAIT_NAME_MAX)
1625 {
1626 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1627 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
1628 host_record->sequence++;
1629 host_record->count = 0;
1630 host_length = 0;
1631 }
1632
1633 /* If this record is not full, increase the size of the record to
1634 allow for one new message id. */
1635
1636 else
1637 {
1638 dbdata_wait *newr =
1639 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH);
1640 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1641 host_record = newr;
1642 }
1643
1644 /* Now add the new name on the end */
1645
1646 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1647 host_record->count++;
1648 host_length += MESSAGE_ID_LENGTH;
1649
1650 /* Update the database */
1651
1652 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1653 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1654 }
1655
1656/* All now done */
1657
1658dbfn_close(dbm_file);
1659}
1660
1661
1662
1663
1664/*************************************************
1665* Test for waiting messages *
1666*************************************************/
1667
1668/* This function is called by a remote transport which uses the previous
1669function to remember which messages are waiting for which remote hosts. It's
1670called after a successful delivery and its job is to check whether there is
1671another message waiting for the same host. However, it doesn't do this if the
1672current continue sequence is greater than the maximum supplied as an argument,
1673or greater than the global connection_max_messages, which, if set, overrides.
1674
1675Arguments:
1676 transport_name name of the transport
1677 hostname name of the host
1678 local_message_max maximum number of messages down one connection
1679 as set by the caller transport
1680 new_message_id set to the message id of a waiting message
1681 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1682 oicf_func function to call to validate if it is ok to send
1683 to this message_id from the current instance.
1684 oicf_data opaque data for oicf_func
059ec3d9
PH
1685
1686Returns: TRUE if new_message_id set; FALSE otherwise
1687*/
1688
a39bd74d
JB
1689typedef struct msgq_s
1690{
1691 uschar message_id [MESSAGE_ID_LENGTH + 1];
1692 BOOL bKeep;
1693} msgq_t;
1694
059ec3d9 1695BOOL
55414b25 1696transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1697 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1698{
1699dbdata_wait *host_record;
0539a19d 1700int host_length;
059ec3d9
PH
1701open_db dbblock;
1702open_db *dbm_file;
059ec3d9 1703
a39bd74d 1704int i;
a39bd74d 1705struct stat statbuf;
a39bd74d 1706
059ec3d9
PH
1707*more = FALSE;
1708
1709DEBUG(D_transport)
1710 {
1711 debug_printf("transport_check_waiting entered\n");
1712 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1713 continue_sequence, local_message_max, connection_max_messages);
1714 }
1715
1716/* Do nothing if we have hit the maximum number that can be send down one
1717connection. */
1718
1719if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1720if (local_message_max > 0 && continue_sequence >= local_message_max)
1721 {
1722 DEBUG(D_transport)
1723 debug_printf("max messages for one connection reached: returning\n");
1724 return FALSE;
1725 }
1726
1727/* Open the waiting information database. */
1728
36d295f1
JH
1729if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
1730 O_RDWR, &dbblock, TRUE)))
1731 return FALSE;
059ec3d9
PH
1732
1733/* See if there is a record for this host; if not, there's nothing to do. */
1734
789f8a4f 1735if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1736 {
1737 dbfn_close(dbm_file);
1738 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1739 return FALSE;
1740 }
1741
1742/* If the data in the record looks corrupt, just log something and
1743don't try to use it. */
1744
1745if (host_record->count > WAIT_NAME_MAX)
1746 {
1747 dbfn_close(dbm_file);
1748 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1749 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1750 return FALSE;
1751 }
1752
1753/* Scan the message ids in the record from the end towards the beginning,
1754until one is found for which a spool file actually exists. If the record gets
1755emptied, delete it and continue with any continuation records that may exist.
1756*/
1757
a39bd74d
JB
1758/* For Bug 1141, I refactored this major portion of the routine, it is risky
1759but the 1 off will remain without it. This code now allows me to SKIP over
1760a message I do not want to send out on this run. */
059ec3d9 1761
a39bd74d
JB
1762host_length = host_record->count * MESSAGE_ID_LENGTH;
1763
1764while (1)
059ec3d9 1765 {
789f8a4f
JH
1766 msgq_t *msgq;
1767 int msgq_count = 0;
1768 int msgq_actual = 0;
1769 BOOL bFound = FALSE;
1770 BOOL bContinuation = FALSE;
1771
a39bd74d 1772 /* create an array to read entire message queue into memory for processing */
059ec3d9 1773
40c90bca 1774 msgq = store_malloc(sizeof(msgq_t) * host_record->count);
a39bd74d
JB
1775 msgq_count = host_record->count;
1776 msgq_actual = msgq_count;
059ec3d9 1777
a39bd74d 1778 for (i = 0; i < host_record->count; ++i)
059ec3d9 1779 {
a39bd74d
JB
1780 msgq[i].bKeep = TRUE;
1781
1782 Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1783 MESSAGE_ID_LENGTH);
a39bd74d
JB
1784 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1785 }
1786
1787 /* first thing remove current message id if it exists */
059ec3d9 1788
a39bd74d
JB
1789 for (i = 0; i < msgq_count; ++i)
1790 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1791 {
1792 msgq[i].bKeep = FALSE;
1793 break;
1794 }
1795
1796 /* now find the next acceptable message_id */
1797
a39bd74d
JB
1798 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1799 {
41313d92
JH
1800 uschar subdir[2];
1801
1802 subdir[0] = split_spool_directory ? msgq[i].message_id[5] : 0;
1803 subdir[1] = 0;
059ec3d9 1804
41313d92
JH
1805 if (Ustat(spool_fname(US"input", subdir, msgq[i].message_id, US"-D"),
1806 &statbuf) != 0)
a39bd74d
JB
1807 msgq[i].bKeep = FALSE;
1808 else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data))
059ec3d9 1809 {
a39bd74d
JB
1810 Ustrcpy(new_message_id, msgq[i].message_id);
1811 msgq[i].bKeep = FALSE;
1812 bFound = TRUE;
059ec3d9
PH
1813 break;
1814 }
1815 }
1816
a39bd74d
JB
1817 /* re-count */
1818 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1819 if (msgq[i].bKeep)
1820 msgq_actual++;
1821
1822 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1823 memory queue */
a39bd74d
JB
1824
1825 if (msgq_actual <= 0)
1826 {
1827 host_length = 0;
1828 host_record->count = 0;
1829 }
1830 else
1831 {
1832 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1833 host_record->count = msgq_actual;
1834
1835 if (msgq_actual < msgq_count)
1836 {
1837 int new_count;
1838 for (new_count = 0, i = 0; i < msgq_count; ++i)
1839 if (msgq[i].bKeep)
1840 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1841 msgq[i].message_id, MESSAGE_ID_LENGTH);
1842
1843 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1844 }
1845 }
1846
36d295f1 1847 /* Check for a continuation record. */
a39bd74d 1848
059ec3d9
PH
1849 while (host_length <= 0)
1850 {
1851 int i;
a39bd74d 1852 dbdata_wait * newr = NULL;
36d295f1 1853 uschar buffer[256];
059ec3d9
PH
1854
1855 /* Search for a continuation */
1856
a39bd74d 1857 for (i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1858 {
1859 sprintf(CS buffer, "%.200s:%d", hostname, i);
1860 newr = dbfn_read(dbm_file, buffer);
1861 }
1862
1863 /* If no continuation, delete the current and break the loop */
1864
a39bd74d 1865 if (!newr)
059ec3d9
PH
1866 {
1867 dbfn_delete(dbm_file, hostname);
1868 break;
1869 }
1870
1871 /* Else replace the current with the continuation */
1872
1873 dbfn_delete(dbm_file, buffer);
1874 host_record = newr;
1875 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1876
a39bd74d
JB
1877 bContinuation = TRUE;
1878 }
059ec3d9 1879
789f8a4f
JH
1880 if (bFound) /* Usual exit from main loop */
1881 {
40c90bca 1882 store_free (msgq);
a39bd74d 1883 break;
789f8a4f 1884 }
059ec3d9
PH
1885
1886 /* If host_length <= 0 we have emptied a record and not found a good message,
1887 and there are no continuation records. Otherwise there is a continuation
1888 record to process. */
1889
1890 if (host_length <= 0)
1891 {
1892 dbfn_close(dbm_file);
1893 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1894 return FALSE;
1895 }
a39bd74d
JB
1896
1897 /* we were not able to find an acceptable message, nor was there a
1898 * continuation record. So bug out, outer logic will clean this up.
1899 */
1900
1901 if (!bContinuation)
1902 {
789f8a4f 1903 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1904 dbfn_close(dbm_file);
1905 return FALSE;
1906 }
a39bd74d 1907
40c90bca 1908 store_free(msgq);
789f8a4f 1909 } /* we need to process a continuation record */
059ec3d9
PH
1910
1911/* Control gets here when an existing message has been encountered; its
1912id is in new_message_id, and host_length is the revised length of the
1913host record. If it is zero, the record has been removed. Update the
1914record if required, close the database, and return TRUE. */
1915
1916if (host_length > 0)
1917 {
1918 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1919
059ec3d9
PH
1920 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1921 *more = TRUE;
1922 }
1923
1924dbfn_close(dbm_file);
1925return TRUE;
1926}
1927
059ec3d9
PH
1928/*************************************************
1929* Deliver waiting message down same socket *
1930*************************************************/
1931
57cc2785
JH
1932/* Just the regain-root-privilege exec portion */
1933void
1934transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
1935 const uschar *hostaddress, uschar *id, int socket_fd)
1936{
57cc2785
JH
1937int i = 20;
1938const uschar **argv;
1939
1940/* Set up the calling arguments; use the standard function for the basics,
1941but we have a number of extras that may be added. */
1942
1943argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
1944
1945if (smtp_authenticated) argv[i++] = US"-MCA";
1946if (smtp_peer_options & PEER_OFFERED_CHUNKING) argv[i++] = US"-MCK";
1947if (smtp_peer_options & PEER_OFFERED_DSN) argv[i++] = US"-MCD";
1948if (smtp_peer_options & PEER_OFFERED_PIPE) argv[i++] = US"-MCP";
1949if (smtp_peer_options & PEER_OFFERED_SIZE) argv[i++] = US"-MCS";
1950#ifdef SUPPORT_TLS
1951if (smtp_peer_options & PEER_OFFERED_TLS)
1952 if (tls_out.active >= 0 || continue_proxy_cipher)
1953 {
1954 argv[i++] = US"-MCt";
1955 argv[i++] = sending_ip_address;
1956 argv[i++] = string_sprintf("%d", sending_port);
1957 argv[i++] = tls_out.active >= 0 ? tls_out.cipher : continue_proxy_cipher;
1958 }
1959 else
1960 argv[i++] = US"-MCT";
1961#endif
1962
1963if (queue_run_pid != (pid_t)0)
1964 {
1965 argv[i++] = US"-MCQ";
1966 argv[i++] = string_sprintf("%d", queue_run_pid);
1967 argv[i++] = string_sprintf("%d", queue_run_pipe);
1968 }
1969
1970argv[i++] = US"-MC";
1971argv[i++] = US transport_name;
1972argv[i++] = US hostname;
1973argv[i++] = US hostaddress;
1974argv[i++] = string_sprintf("%d", continue_sequence + 1);
1975argv[i++] = id;
1976argv[i++] = NULL;
1977
1978/* Arrange for the channel to be on stdin. */
1979
1980if (socket_fd != 0)
1981 {
1982 (void)dup2(socket_fd, 0);
1983 (void)close(socket_fd);
1984 }
1985
1986DEBUG(D_exec) debug_print_argv(argv);
1987exim_nullstd(); /* Ensure std{out,err} exist */
1988execv(CS argv[0], (char *const *)argv);
1989
1990DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
1991_exit(errno); /* Note: must be _exit(), NOT exit() */
1992}
1993
1994
1995
059ec3d9
PH
1996/* Fork a new exim process to deliver the message, and do a re-exec, both to
1997get a clean delivery process, and to regain root privilege in cases where it
1998has been given away.
1999
2000Arguments:
2001 transport_name to pass to the new process
2002 hostname ditto
2003 hostaddress ditto
2004 id the new message to process
2005 socket_fd the connected socket
2006
2007Returns: FALSE if fork fails; TRUE otherwise
2008*/
2009
2010BOOL
55414b25
JH
2011transport_pass_socket(const uschar *transport_name, const uschar *hostname,
2012 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
2013{
2014pid_t pid;
2015int status;
2016
2017DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
2018
2019if ((pid = fork()) == 0)
2020 {
059ec3d9
PH
2021 /* Disconnect entirely from the parent process. If we are running in the
2022 test harness, wait for a bit to allow the previous process time to finish,
2023 write the log, etc., so that the output is always in the same order for
2024 automatic comparison. */
2025
2026 if ((pid = fork()) != 0) _exit(EXIT_SUCCESS);
ed0e9820 2027 if (running_in_test_harness) sleep(1);
059ec3d9 2028
57cc2785
JH
2029 transport_do_pass_socket(transport_name, hostname, hostaddress,
2030 id, socket_fd);
059ec3d9
PH
2031 }
2032
2033/* If the process creation succeeded, wait for the first-level child, which
2034immediately exits, leaving the second level process entirely disconnected from
2035this one. */
2036
2037if (pid > 0)
2038 {
2039 int rc;
2040 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
2041 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded\n");
2042 return TRUE;
2043 }
2044else
2045 {
2046 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
2047 strerror(errno));
2048 return FALSE;
2049 }
2050}
2051
2052
2053
2054/*************************************************
2055* Set up direct (non-shell) command *
2056*************************************************/
2057
2058/* This function is called when a command line is to be parsed and executed
2059directly, without the use of /bin/sh. It is called by the pipe transport,
2060the queryprogram router, and also from the main delivery code when setting up a
2061transport filter process. The code for ETRN also makes use of this; in that
2062case, no addresses are passed.
2063
2064Arguments:
2065 argvptr pointer to anchor for argv vector
55414b25 2066 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
2067 expand_arguments true if expansion is to occur
2068 expand_failed error value to set if expansion fails; not relevant if
2069 addr == NULL
2070 addr chain of addresses, or NULL
2071 etext text for use in error messages
2072 errptr where to put error message if addr is NULL;
2073 otherwise it is put in the first address
2074
2075Returns: TRUE if all went well; otherwise an error will be
2076 set in the first address and FALSE returned
2077*/
2078
2079BOOL
55414b25
JH
2080transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2081 BOOL expand_arguments, int expand_failed, address_item *addr,
2082 uschar *etext, uschar **errptr)
059ec3d9
PH
2083{
2084address_item *ad;
55414b25 2085const uschar **argv;
059ec3d9
PH
2086uschar *s, *ss;
2087int address_count = 0;
2088int argcount = 0;
2089int i, max_args;
2090
2091/* Get store in which to build an argument list. Count the number of addresses
2092supplied, and allow for that many arguments, plus an additional 60, which
2093should be enough for anybody. Multiple addresses happen only when the local
2094delivery batch option is set. */
2095
2096for (ad = addr; ad != NULL; ad = ad->next) address_count++;
2097max_args = address_count + 60;
2098*argvptr = argv = store_get((max_args+1)*sizeof(uschar *));
2099
2100/* Split the command up into arguments terminated by white space. Lose
2101trailing space at the start and end. Double-quoted arguments can contain \\ and
2102\" escapes and so can be handled by the standard function; single-quoted
2103arguments are verbatim. Copy each argument into a new string. */
2104
2105s = cmd;
2106while (isspace(*s)) s++;
2107
2108while (*s != 0 && argcount < max_args)
2109 {
2110 if (*s == '\'')
2111 {
2112 ss = s + 1;
2113 while (*ss != 0 && *ss != '\'') ss++;
2114 argv[argcount++] = ss = store_get(ss - s++);
2115 while (*s != 0 && *s != '\'') *ss++ = *s++;
2116 if (*s != 0) s++;
2117 *ss++ = 0;
2118 }
55414b25 2119 else argv[argcount++] = string_copy(string_dequote(CUSS &s));
059ec3d9
PH
2120 while (isspace(*s)) s++;
2121 }
2122
2123argv[argcount] = (uschar *)0;
2124
2125/* If *s != 0 we have run out of argument slots. */
2126
2127if (*s != 0)
2128 {
2129 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2130 "%s", cmd, etext);
2131 if (addr != NULL)
2132 {
2133 addr->transport_return = FAIL;
2134 addr->message = msg;
2135 }
2136 else *errptr = msg;
2137 return FALSE;
2138 }
2139
2140/* Expand each individual argument if required. Expansion happens for pipes set
2141up in filter files and with directly-supplied commands. It does not happen if
2142the pipe comes from a traditional .forward file. A failing expansion is a big
2143disaster if the command came from Exim's configuration; if it came from a user
2144it is just a normal failure. The expand_failed value is used as the error value
2145to cater for these two cases.
2146
2147An argument consisting just of the text "$pipe_addresses" is treated specially.
2148It is not passed to the general expansion function. Instead, it is replaced by
2149a number of arguments, one for each address. This avoids problems with shell
2150metacharacters and spaces in addresses.
2151
2152If the parent of the top address has an original part of "system-filter", this
2153pipe was set up by the system filter, and we can permit the expansion of
2154$recipients. */
2155
2156DEBUG(D_transport)
2157 {
2158 debug_printf("direct command:\n");
2159 for (i = 0; argv[i] != (uschar *)0; i++)
2160 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2161 }
2162
2163if (expand_arguments)
2164 {
2165 BOOL allow_dollar_recipients = addr != NULL &&
2166 addr->parent != NULL &&
2167 Ustrcmp(addr->parent->address, "system-filter") == 0;
2168
2169 for (i = 0; argv[i] != (uschar *)0; i++)
2170 {
2171
2172 /* Handle special fudge for passing an address list */
2173
2174 if (addr != NULL &&
2175 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2176 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2177 {
2178 int additional;
2179
2180 if (argcount + address_count - 1 > max_args)
2181 {
2182 addr->transport_return = FAIL;
2183 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2184 "in %s", cmd, etext);
2185 return FALSE;
2186 }
2187
2188 additional = address_count - 1;
2189 if (additional > 0)
2190 memmove(argv + i + 1 + additional, argv + i + 1,
2191 (argcount - i)*sizeof(uschar *));
2192
09792322
NK
2193 for (ad = addr; ad != NULL; ad = ad->next) {
2194 argv[i++] = ad->address;
2195 argcount++;
2196 }
2197
700d22f3
PP
2198 /* Subtract one since we replace $pipe_addresses */
2199 argcount--;
2200 i--;
09792322
NK
2201 }
2202
2203 /* Handle special case of $address_pipe when af_force_command is set */
2204
2205 else if (addr != NULL && testflag(addr,af_force_command) &&
2206 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2207 Ustrcmp(argv[i], "${address_pipe}") == 0))
2208 {
2209 int address_pipe_i;
2210 int address_pipe_argcount = 0;
2211 int address_pipe_max_args;
2212 uschar **address_pipe_argv;
2213
2214 /* We can never have more then the argv we will be loading into */
2215 address_pipe_max_args = max_args - argcount + 1;
2216
2217 DEBUG(D_transport)
700d22f3 2218 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2219
2220 /* We allocate an additional for (uschar *)0 */
2221 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *));
2222
2223 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2224 s = expand_string(addr->local_part + 1);
2225
700d22f3
PP
2226 if (s == NULL || *s == '\0')
2227 {
2228 addr->transport_return = FAIL;
2229 addr->message = string_sprintf("Expansion of \"%s\" "
2230 "from command \"%s\" in %s failed: %s",
2231 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2232 return FALSE;
700d22f3 2233 }
09792322
NK
2234
2235 while (isspace(*s)) s++; /* strip leading space */
2236
2237 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2238 {
2239 if (*s == '\'')
2240 {
2241 ss = s + 1;
2242 while (*ss != 0 && *ss != '\'') ss++;
2243 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++);
2244 while (*s != 0 && *s != '\'') *ss++ = *s++;
2245 if (*s != 0) s++;
2246 *ss++ = 0;
2247 }
55414b25
JH
2248 else address_pipe_argv[address_pipe_argcount++] =
2249 string_copy(string_dequote(CUSS &s));
09792322
NK
2250 while (isspace(*s)) s++; /* strip space after arg */
2251 }
2252
2253 address_pipe_argv[address_pipe_argcount] = (uschar *)0;
2254
2255 /* If *s != 0 we have run out of argument slots. */
2256 if (*s != 0)
2257 {
2258 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2259 "\"%s\" in %s", addr->local_part + 1, etext);
2260 if (addr != NULL)
2261 {
2262 addr->transport_return = FAIL;
2263 addr->message = msg;
2264 }
2265 else *errptr = msg;
2266 return FALSE;
2267 }
2268
700d22f3
PP
2269 /* address_pipe_argcount - 1
2270 * because we are replacing $address_pipe in the argument list
2271 * with the first thing it expands to */
09792322
NK
2272 if (argcount + address_pipe_argcount - 1 > max_args)
2273 {
2274 addr->transport_return = FAIL;
2275 addr->message = string_sprintf("Too many arguments to command "
2276 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2277 return FALSE;
2278 }
2279
2280 /* If we are not just able to replace the slot that contained
2281 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2282 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2283 * Visually if address_pipe_argcount == 2:
2284 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2285 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2286 */
2287 if (address_pipe_argcount > 1)
2288 memmove(
4c04137d 2289 /* current position + additional args */
09792322
NK
2290 argv + i + address_pipe_argcount,
2291 /* current position + 1 (for the (uschar *)0 at the end) */
2292 argv + i + 1,
2293 /* -1 for the (uschar *)0 at the end)*/
2294 (argcount - i)*sizeof(uschar *)
2295 );
2296
2297 /* Now we fill in the slots we just moved argv out of
2298 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2299 */
700d22f3
PP
2300 for (address_pipe_i = 0;
2301 address_pipe_argv[address_pipe_i] != (uschar *)0;
2302 address_pipe_i++)
2303 {
2304 argv[i++] = address_pipe_argv[address_pipe_i];
2305 argcount++;
09792322
NK
2306 }
2307
700d22f3
PP
2308 /* Subtract one since we replace $address_pipe */
2309 argcount--;
2310 i--;
059ec3d9
PH
2311 }
2312
2313 /* Handle normal expansion string */
2314
2315 else
2316 {
55414b25 2317 const uschar *expanded_arg;
059ec3d9 2318 enable_dollar_recipients = allow_dollar_recipients;
55414b25 2319 expanded_arg = expand_cstring(argv[i]);
059ec3d9
PH
2320 enable_dollar_recipients = FALSE;
2321
2322 if (expanded_arg == NULL)
2323 {
2324 uschar *msg = string_sprintf("Expansion of \"%s\" "
2325 "from command \"%s\" in %s failed: %s",
2326 argv[i], cmd, etext, expand_string_message);
2327 if (addr != NULL)
2328 {
2329 addr->transport_return = expand_failed;
2330 addr->message = msg;
2331 }
2332 else *errptr = msg;
2333 return FALSE;
2334 }
2335 argv[i] = expanded_arg;
2336 }
2337 }
2338
2339 DEBUG(D_transport)
2340 {
2341 debug_printf("direct command after expansion:\n");
2342 for (i = 0; argv[i] != (uschar *)0; i++)
2343 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2344 }
2345 }
2346
2347return TRUE;
2348}
2349
511a6c14
JH
2350/* vi: aw ai sw=2
2351*/
059ec3d9 2352/* End of transport.c */