Avoid parsing cost for auto-macro creates
[exim.git] / src / src / transport.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
80fea873 5/* Copyright (c) University of Cambridge 1995 - 2016 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* General functions concerned with transportation, and generic options for all
9transports. */
10
11
12#include "exim.h"
13
a8530a10
TK
14#ifdef HAVE_LINUX_SENDFILE
15#include <sys/sendfile.h>
16#endif
059ec3d9
PH
17
18/* Structure for keeping list of addresses that have been added to
19Envelope-To:, in order to avoid duplication. */
20
21struct aci {
22 struct aci *next;
23 address_item *ptr;
24 };
25
26
27/* Static data for write_chunk() */
28
29static uschar *chunk_ptr; /* chunk pointer */
30static uschar *nl_check; /* string to look for at line start */
31static int nl_check_length; /* length of same */
32static uschar *nl_escape; /* string to insert */
33static int nl_escape_length; /* length of same */
34static int nl_partial_match; /* length matched at chunk end */
35
36
37/* Generic options for transports, all of which live inside transport_instance
38data blocks and which therefore have the opt_public flag set. Note that there
39are other options living inside this structure which can be set only from
40certain transports. */
41
42optionlist optionlist_transports[] = {
43 { "*expand_group", opt_stringptr|opt_hidden|opt_public,
44 (void *)offsetof(transport_instance, expand_gid) },
45 { "*expand_user", opt_stringptr|opt_hidden|opt_public,
46 (void *)offsetof(transport_instance, expand_uid) },
47 { "*headers_rewrite_flags", opt_int|opt_public|opt_hidden,
48 (void *)offsetof(transport_instance, rewrite_existflags) },
49 { "*headers_rewrite_rules", opt_void|opt_public|opt_hidden,
50 (void *)offsetof(transport_instance, rewrite_rules) },
51 { "*set_group", opt_bool|opt_hidden|opt_public,
52 (void *)offsetof(transport_instance, gid_set) },
53 { "*set_user", opt_bool|opt_hidden|opt_public,
54 (void *)offsetof(transport_instance, uid_set) },
55 { "body_only", opt_bool|opt_public,
56 (void *)offsetof(transport_instance, body_only) },
57 { "current_directory", opt_stringptr|opt_public,
58 (void *)offsetof(transport_instance, current_dir) },
59 { "debug_print", opt_stringptr | opt_public,
60 (void *)offsetof(transport_instance, debug_string) },
61 { "delivery_date_add", opt_bool|opt_public,
62 (void *)(offsetof(transport_instance, delivery_date_add)) },
63 { "disable_logging", opt_bool|opt_public,
64 (void *)(offsetof(transport_instance, disable_logging)) },
65 { "driver", opt_stringptr|opt_public,
66 (void *)offsetof(transport_instance, driver_name) },
67 { "envelope_to_add", opt_bool|opt_public,
68 (void *)(offsetof(transport_instance, envelope_to_add)) },
0cbf2b82 69#ifndef DISABLE_EVENT
774ef2d7
JH
70 { "event_action", opt_stringptr | opt_public,
71 (void *)offsetof(transport_instance, event_action) },
72#endif
059ec3d9
PH
73 { "group", opt_expand_gid|opt_public,
74 (void *)offsetof(transport_instance, gid) },
846726c5 75 { "headers_add", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
76 (void *)offsetof(transport_instance, add_headers) },
77 { "headers_only", opt_bool|opt_public,
78 (void *)offsetof(transport_instance, headers_only) },
846726c5 79 { "headers_remove", opt_stringptr|opt_public|opt_rep_str,
059ec3d9
PH
80 (void *)offsetof(transport_instance, remove_headers) },
81 { "headers_rewrite", opt_rewrite|opt_public,
82 (void *)offsetof(transport_instance, headers_rewrite) },
83 { "home_directory", opt_stringptr|opt_public,
84 (void *)offsetof(transport_instance, home_dir) },
85 { "initgroups", opt_bool|opt_public,
86 (void *)offsetof(transport_instance, initgroups) },
fa41615d
JH
87 { "max_parallel", opt_stringptr|opt_public,
88 (void *)offsetof(transport_instance, max_parallel) },
059ec3d9
PH
89 { "message_size_limit", opt_stringptr|opt_public,
90 (void *)offsetof(transport_instance, message_size_limit) },
91 { "rcpt_include_affixes", opt_bool|opt_public,
92 (void *)offsetof(transport_instance, rcpt_include_affixes) },
93 { "retry_use_local_part", opt_bool|opt_public,
94 (void *)offsetof(transport_instance, retry_use_local_part) },
95 { "return_path", opt_stringptr|opt_public,
96 (void *)(offsetof(transport_instance, return_path)) },
97 { "return_path_add", opt_bool|opt_public,
98 (void *)(offsetof(transport_instance, return_path_add)) },
99 { "shadow_condition", opt_stringptr|opt_public,
100 (void *)offsetof(transport_instance, shadow_condition) },
101 { "shadow_transport", opt_stringptr|opt_public,
102 (void *)offsetof(transport_instance, shadow) },
103 { "transport_filter", opt_stringptr|opt_public,
104 (void *)offsetof(transport_instance, filter_command) },
105 { "transport_filter_timeout", opt_time|opt_public,
106 (void *)offsetof(transport_instance, filter_timeout) },
107 { "user", opt_expand_uid|opt_public,
108 (void *)offsetof(transport_instance, uid) }
109};
110
c0b9d3e8 111int optionlist_transports_size = nelem(optionlist_transports);
059ec3d9
PH
112
113
c0b9d3e8
JH
114void
115readconf_options_transports(void)
116{
117struct transport_info * ti;
118
119readconf_options_from_list(optionlist_transports, nelem(optionlist_transports), US"TP");
120
121for (ti = transports_available; ti->driver_name[0]; ti++)
4945f557 122 {
c193398d 123 macro_create(string_sprintf("_DRVR_TPT_%T", ti->driver_name), US"y", FALSE);
c0b9d3e8 124 readconf_options_from_list(ti->options, (unsigned)*ti->options_count, ti->driver_name);
4945f557 125 }
c0b9d3e8
JH
126}
127
059ec3d9
PH
128/*************************************************
129* Initialize transport list *
130*************************************************/
131
132/* Read the transports section of the configuration file, and set up a chain of
133transport instances according to its contents. Each transport has generic
134options and may also have its own private options. This function is only ever
135called when transports == NULL. We use generic code in readconf to do most of
136the work. */
137
138void
139transport_init(void)
140{
141transport_instance *t;
142
143readconf_driver_init(US"transport",
144 (driver_instance **)(&transports), /* chain anchor */
145 (driver_info *)transports_available, /* available drivers */
146 sizeof(transport_info), /* size of info block */
147 &transport_defaults, /* default values for generic options */
148 sizeof(transport_instance), /* size of instance block */
149 optionlist_transports, /* generic options */
150 optionlist_transports_size);
151
152/* Now scan the configured transports and check inconsistencies. A shadow
153transport is permitted only for local transports. */
154
7b4c8c1f 155for (t = transports; t; t = t->next)
059ec3d9 156 {
7b4c8c1f
JH
157 if (!t->info->local && t->shadow)
158 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
159 "shadow transport not allowed on non-local transport %s", t->name);
059ec3d9
PH
160
161 if (t->body_only && t->headers_only)
162 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
163 "%s transport: body_only and headers_only are mutually exclusive",
164 t->name);
165 }
166}
167
168
169
170/*************************************************
171* Write block of data *
172*************************************************/
173
174/* Subroutine called by write_chunk() and at the end of the message actually
175to write a data block. Also called directly by some transports to write
176additional data to the file descriptor (e.g. prefix, suffix).
177
178If a transport wants data transfers to be timed, it sets a non-zero value in
179transport_write_timeout. A non-zero transport_write_timeout causes a timer to
180be set for each block of data written from here. If time runs out, then write()
181fails and provokes an error return. The caller can then inspect sigalrm_seen to
182check for a timeout.
183
184On some systems, if a quota is exceeded during the write, the yield is the
185number of bytes written rather than an immediate error code. This also happens
186on some systems in other cases, for example a pipe that goes away because the
187other end's process terminates (Linux). On other systems, (e.g. Solaris 2) you
188get the error codes the first time.
189
190The write() function is also interruptible; the Solaris 2.6 man page says:
191
192 If write() is interrupted by a signal before it writes any
193 data, it will return -1 with errno set to EINTR.
194
195 If write() is interrupted by a signal after it successfully
196 writes some data, it will return the number of bytes written.
197
198To handle these cases, we want to restart the write() to output the remainder
199of the data after a non-negative return from write(), except after a timeout.
200In the error cases (EDQUOT, EPIPE) no bytes get written the second time, and a
201proper error then occurs. In principle, after an interruption, the second
202write() could suffer the same fate, but we do not want to continue for
203evermore, so stick a maximum repetition count on the loop to act as a
204longstop.
205
206Arguments:
207 fd file descriptor to write to
208 block block of bytes to write
209 len number of bytes to write
210
211Returns: TRUE on success, FALSE on failure (with errno preserved);
212 transport_count is incremented by the number of bytes written
213*/
214
215BOOL
216transport_write_block(int fd, uschar *block, int len)
217{
218int i, rc, save_errno;
958541e9
PH
219int local_timeout = transport_write_timeout;
220
221/* This loop is for handling incomplete writes and other retries. In most
222normal cases, it is only ever executed once. */
059ec3d9
PH
223
224for (i = 0; i < 100; i++)
225 {
226 DEBUG(D_transport)
227 debug_printf("writing data block fd=%d size=%d timeout=%d\n",
958541e9 228 fd, len, local_timeout);
059ec3d9 229
958541e9
PH
230 /* This code makes use of alarm() in order to implement the timeout. This
231 isn't a very tidy way of doing things. Using non-blocking I/O with select()
232 provides a neater approach. However, I don't know how to do this when TLS is
233 in use. */
059ec3d9 234
958541e9
PH
235 if (transport_write_timeout <= 0) /* No timeout wanted */
236 {
237 #ifdef SUPPORT_TLS
817d9f57 238 if (tls_out.active == fd) rc = tls_write(FALSE, block, len); else
958541e9
PH
239 #endif
240 rc = write(fd, block, len);
241 save_errno = errno;
242 }
059ec3d9 243
958541e9 244 /* Timeout wanted. */
059ec3d9 245
958541e9 246 else
059ec3d9 247 {
958541e9 248 alarm(local_timeout);
59932f7d
JH
249#ifdef SUPPORT_TLS
250 if (tls_out.active == fd)
251 rc = tls_write(FALSE, block, len);
252 else
253#endif
254 rc = write(fd, block, len);
958541e9
PH
255 save_errno = errno;
256 local_timeout = alarm(0);
059ec3d9
PH
257 if (sigalrm_seen)
258 {
259 errno = ETIMEDOUT;
260 return FALSE;
261 }
262 }
263
264 /* Hopefully, the most common case is success, so test that first. */
265
266 if (rc == len) { transport_count += len; return TRUE; }
267
958541e9
PH
268 /* A non-negative return code is an incomplete write. Try again for the rest
269 of the block. If we have exactly hit the timeout, give up. */
059ec3d9
PH
270
271 if (rc >= 0)
272 {
273 len -= rc;
274 block += rc;
275 transport_count += rc;
276 DEBUG(D_transport) debug_printf("write incomplete (%d)\n", rc);
958541e9 277 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
278 }
279
280 /* A negative return code with an EINTR error is another form of
281 incomplete write, zero bytes having been written */
282
283 if (save_errno == EINTR)
284 {
285 DEBUG(D_transport)
286 debug_printf("write interrupted before anything written\n");
958541e9 287 goto CHECK_TIMEOUT; /* A few lines below */
059ec3d9
PH
288 }
289
290 /* A response of EAGAIN from write() is likely only in the case of writing
291 to a FIFO that is not swallowing the data as fast as Exim is writing it. */
292
293 if (save_errno == EAGAIN)
294 {
295 DEBUG(D_transport)
296 debug_printf("write temporarily locked out, waiting 1 sec\n");
297 sleep(1);
958541e9
PH
298
299 /* Before continuing to try another write, check that we haven't run out of
300 time. */
301
302 CHECK_TIMEOUT:
303 if (transport_write_timeout > 0 && local_timeout <= 0)
304 {
305 errno = ETIMEDOUT;
306 return FALSE;
307 }
059ec3d9
PH
308 continue;
309 }
310
311 /* Otherwise there's been an error */
312
313 DEBUG(D_transport) debug_printf("writing error %d: %s\n", save_errno,
314 strerror(save_errno));
315 errno = save_errno;
316 return FALSE;
317 }
318
319/* We've tried and tried and tried but still failed */
320
321errno = ERRNO_WRITEINCOMPLETE;
322return FALSE;
323}
324
325
326
327
328/*************************************************
329* Write formatted string *
330*************************************************/
331
332/* This is called by various transports. It is a convenience function.
333
334Arguments:
335 fd file descriptor
336 format string format
337 ... arguments for format
338
339Returns: the yield of transport_write_block()
340*/
341
342BOOL
1ba28e2b 343transport_write_string(int fd, const char *format, ...)
059ec3d9
PH
344{
345va_list ap;
346va_start(ap, format);
347if (!string_vformat(big_buffer, big_buffer_size, format, ap))
348 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "overlong formatted string in transport");
349va_end(ap);
350return transport_write_block(fd, big_buffer, Ustrlen(big_buffer));
351}
352
353
354
355
356/*************************************************
357* Write character chunk *
358*************************************************/
359
360/* Subroutine used by transport_write_message() to scan character chunks for
361newlines and act appropriately. The object is to minimise the number of writes.
362The output byte stream is buffered up in deliver_out_buffer, which is written
363only when it gets full, thus minimizing write operations and TCP packets.
364
365Static data is used to handle the case when the last character of the previous
366chunk was NL, or matched part of the data that has to be escaped.
367
368Arguments:
369 fd file descript to write to
370 chunk pointer to data to write
371 len length of data to write
6d5c916c 372 tctx transport context - processing to be done during output
059ec3d9
PH
373
374In addition, the static nl_xxx variables must be set as required.
375
376Returns: TRUE on success, FALSE on failure (with errno preserved)
377*/
378
379static BOOL
6d5c916c 380write_chunk(int fd, transport_ctx * tctx, uschar *chunk, int len)
059ec3d9
PH
381{
382uschar *start = chunk;
383uschar *end = chunk + len;
ff5aac2b 384uschar *ptr;
059ec3d9
PH
385int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
386
387/* The assumption is made that the check string will never stretch over move
388than one chunk since the only time there are partial matches is when copying
389the body in large buffers. There is always enough room in the buffer for an
390escape string, since the loop below ensures this for each character it
391processes, and it won't have stuck in the escape string if it left a partial
392match. */
393
394if (nl_partial_match >= 0)
395 {
396 if (nl_check_length > 0 && len >= nl_check_length &&
397 Ustrncmp(start, nl_check + nl_partial_match,
398 nl_check_length - nl_partial_match) == 0)
399 {
400 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
401 chunk_ptr += nl_escape_length;
402 start += nl_check_length - nl_partial_match;
403 }
404
405 /* The partial match was a false one. Insert the characters carried over
406 from the previous chunk. */
407
408 else if (nl_partial_match > 0)
409 {
410 Ustrncpy(chunk_ptr, nl_check, nl_partial_match);
411 chunk_ptr += nl_partial_match;
412 }
413
414 nl_partial_match = -1;
415 }
416
417/* Now process the characters in the chunk. Whenever we hit a newline we check
418for possible escaping. The code for the non-NL route should be as fast as
419possible. */
420
421for (ptr = start; ptr < end; ptr++)
422 {
6d5c916c 423 int ch, len;
059ec3d9
PH
424
425 /* Flush the buffer if it has reached the threshold - we want to leave enough
426 room for the next uschar, plus a possible extra CR for an LF, plus the escape
427 string. */
428
6d5c916c 429 if ((len = chunk_ptr - deliver_out_buffer) > mlen)
059ec3d9 430 {
58fc5fb2
JH
431 DEBUG(D_transport) debug_printf("flushing headers buffer\n");
432
6d5c916c
JH
433 /* If CHUNKING, prefix with BDAT (size) NON-LAST. Also, reap responses
434 from previous SMTP commands. */
435
436 if (tctx && tctx->options & topt_use_bdat && tctx->chunk_cb)
58fc5fb2
JH
437 {
438 if ( tctx->chunk_cb(fd, tctx, (unsigned)len, 0) != OK
439 || !transport_write_block(fd, deliver_out_buffer, len)
440 || tctx->chunk_cb(fd, tctx, 0, tc_reap_prev) != OK
441 )
442 return FALSE;
443 }
444 else
445 if (!transport_write_block(fd, deliver_out_buffer, len))
6d5c916c 446 return FALSE;
059ec3d9
PH
447 chunk_ptr = deliver_out_buffer;
448 }
449
450 if ((ch = *ptr) == '\n')
451 {
452 int left = end - ptr - 1; /* count of chars left after NL */
453
454 /* Insert CR before NL if required */
455
6d5c916c 456 if (tctx && tctx->options & topt_use_crlf) *chunk_ptr++ = '\r';
059ec3d9 457 *chunk_ptr++ = '\n';
332f5cf3 458 transport_newlines++;
059ec3d9
PH
459
460 /* The check_string test (formerly "from hack") replaces the specific
461 string at the start of a line with an escape string (e.g. "From " becomes
462 ">From " or "." becomes "..". It is a case-sensitive test. The length
463 check above ensures there is always enough room to insert this string. */
464
465 if (nl_check_length > 0)
466 {
467 if (left >= nl_check_length &&
468 Ustrncmp(ptr+1, nl_check, nl_check_length) == 0)
469 {
470 Ustrncpy(chunk_ptr, nl_escape, nl_escape_length);
471 chunk_ptr += nl_escape_length;
472 ptr += nl_check_length;
473 }
474
475 /* Handle the case when there isn't enough left to match the whole
476 check string, but there may be a partial match. We remember how many
477 characters matched, and finish processing this chunk. */
478
479 else if (left <= 0) nl_partial_match = 0;
480
481 else if (Ustrncmp(ptr+1, nl_check, left) == 0)
482 {
483 nl_partial_match = left;
484 ptr = end;
485 }
486 }
487 }
488
489 /* Not a NL character */
490
491 else *chunk_ptr++ = ch;
492 }
493
494return TRUE;
495}
496
497
498
499
500/*************************************************
501* Generate address for RCPT TO *
502*************************************************/
503
504/* This function puts together an address for RCPT to, using the caseful
505version of the local part and the caseful version of the domain. If there is no
506prefix or suffix, or if affixes are to be retained, we can just use the
507original address. Otherwise, if there is a prefix but no suffix we can use a
508pointer into the original address. If there is a suffix, however, we have to
509build a new string.
510
511Arguments:
512 addr the address item
513 include_affixes TRUE if affixes are to be included
514
515Returns: a string
516*/
517
518uschar *
519transport_rcpt_address(address_item *addr, BOOL include_affixes)
520{
521uschar *at;
522int plen, slen;
523
524if (include_affixes)
525 {
526 setflag(addr, af_include_affixes); /* Affects logged => line */
527 return addr->address;
528 }
529
530if (addr->suffix == NULL)
531 {
532 if (addr->prefix == NULL) return addr->address;
533 return addr->address + Ustrlen(addr->prefix);
534 }
535
536at = Ustrrchr(addr->address, '@');
537plen = (addr->prefix == NULL)? 0 : Ustrlen(addr->prefix);
538slen = Ustrlen(addr->suffix);
539
540return string_sprintf("%.*s@%s", (at - addr->address - plen - slen),
541 addr->address + plen, at + 1);
542}
543
544
545/*************************************************
546* Output Envelope-To: address & scan duplicates *
547*************************************************/
548
549/* This function is called from internal_transport_write_message() below, when
550generating an Envelope-To: header line. It checks for duplicates of the given
551address and its ancestors. When one is found, this function calls itself
552recursively, to output the envelope address of the duplicate.
553
554We want to avoid duplication in the list, which can arise for example when
555A->B,C and then both B and C alias to D. This can also happen when there are
556unseen drivers in use. So a list of addresses that have been output is kept in
557the plist variable.
558
559It is also possible to have loops in the address ancestry/duplication graph,
560for example if there are two top level addresses A and B and we have A->B,C and
561B->A. To break the loop, we use a list of processed addresses in the dlist
562variable.
563
564After handling duplication, this function outputs the progenitor of the given
565address.
566
567Arguments:
568 p the address we are interested in
569 pplist address of anchor of the list of addresses not to output
570 pdlist address of anchor of the list of processed addresses
571 first TRUE if this is the first address; set it FALSE afterwards
572 fd the file descriptor to write to
6d5c916c 573 tctx transport context - processing to be done during output
059ec3d9
PH
574
575Returns: FALSE if writing failed
576*/
577
578static BOOL
579write_env_to(address_item *p, struct aci **pplist, struct aci **pdlist,
6d5c916c 580 BOOL *first, int fd, transport_ctx * tctx)
059ec3d9
PH
581{
582address_item *pp;
583struct aci *ppp;
584
585/* Do nothing if we have already handled this address. If not, remember it
586so that we don't handle it again. */
587
ff5aac2b 588for (ppp = *pdlist; ppp; ppp = ppp->next) if (p == ppp->ptr) return TRUE;
059ec3d9
PH
589
590ppp = store_get(sizeof(struct aci));
591ppp->next = *pdlist;
592*pdlist = ppp;
593ppp->ptr = p;
594
595/* Now scan up the ancestry, checking for duplicates at each generation. */
596
597for (pp = p;; pp = pp->parent)
598 {
599 address_item *dup;
ff5aac2b
JH
600 for (dup = addr_duplicate; dup; dup = dup->next)
601 if (dup->dupof == pp) /* a dup of our address */
6d5c916c 602 if (!write_env_to(dup, pplist, pdlist, first, fd, tctx))
ff5aac2b
JH
603 return FALSE;
604 if (!pp->parent) break;
059ec3d9
PH
605 }
606
607/* Check to see if we have already output the progenitor. */
608
ff5aac2b
JH
609for (ppp = *pplist; ppp; ppp = ppp->next) if (pp == ppp->ptr) break;
610if (ppp) return TRUE;
059ec3d9
PH
611
612/* Remember what we have output, and output it. */
613
614ppp = store_get(sizeof(struct aci));
615ppp->next = *pplist;
616*pplist = ppp;
617ppp->ptr = pp;
618
6d5c916c 619if (!*first && !write_chunk(fd, tctx, US",\n ", 3)) return FALSE;
059ec3d9 620*first = FALSE;
6d5c916c 621return write_chunk(fd, tctx, pp->address, Ustrlen(pp->address));
059ec3d9
PH
622}
623
624
625
626
511a6c14
JH
627/* Add/remove/rewwrite headers, and send them plus the empty-line sparator.
628
629Globals:
630 header_list
631
632Arguments:
633 addr (chain of) addresses (for extra headers), or NULL;
634 only the first address is used
635 fd file descriptor to write the message to
44bc8f0c 636 tctx transport context
59932f7d 637 sendfn function for output (transport or verify)
511a6c14
JH
638
639Returns: TRUE on success; FALSE on failure.
640*/
641BOOL
6d5c916c
JH
642transport_headers_send(int fd, transport_ctx * tctx,
643 BOOL (*sendfn)(int fd, transport_ctx * tctx, uschar * s, int len))
511a6c14
JH
644{
645header_line *h;
6b46ecc6 646const uschar *list;
6d5c916c
JH
647transport_instance * tblock = tctx ? tctx->tblock : NULL;
648address_item * addr = tctx ? tctx->addr : NULL;
511a6c14
JH
649
650/* Then the message's headers. Don't write any that are flagged as "old";
651that means they were rewritten, or are a record of envelope rewriting, or
652were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
76146973
JH
653match any entries therein. It is a colon-sep list; expand the items
654separately and squash any empty ones.
d43cbe25 655Then check addr->prop.remove_headers too, provided that addr is not NULL. */
511a6c14 656
ff5aac2b 657for (h = header_list; h; h = h->next) if (h->type != htype_old)
511a6c14
JH
658 {
659 int i;
511a6c14
JH
660 BOOL include_header = TRUE;
661
fb6833e0 662 list = tblock ? tblock->remove_headers : NULL;
d43cbe25 663 for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */
511a6c14
JH
664 {
665 if (list)
666 {
667 int sep = ':'; /* This is specified as a colon-separated list */
668 uschar *s, *ss;
8bc732e8 669 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
511a6c14 670 {
76146973
JH
671 int len;
672
673 if (i == 0)
674 if (!(s = expand_string(s)) && !expand_string_forcedfail)
675 {
676 errno = ERRNO_CHHEADER_FAIL;
677 return FALSE;
678 }
d4ff61d1 679 len = s ? Ustrlen(s) : 0;
511a6c14
JH
680 if (strncmpic(h->text, s, len) != 0) continue;
681 ss = h->text + len;
682 while (*ss == ' ' || *ss == '\t') ss++;
683 if (*ss == ':') break;
684 }
6b46ecc6 685 if (s) { include_header = FALSE; break; }
511a6c14 686 }
6b46ecc6 687 if (addr) list = addr->prop.remove_headers;
511a6c14
JH
688 }
689
690 /* If this header is to be output, try to rewrite it if there are rewriting
691 rules. */
692
693 if (include_header)
694 {
fb6833e0 695 if (tblock && tblock->rewrite_rules)
511a6c14
JH
696 {
697 void *reset_point = store_get(0);
698 header_line *hh;
699
6b46ecc6
JH
700 if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
701 tblock->rewrite_existflags, FALSE)))
511a6c14 702 {
6d5c916c 703 if (!sendfn(fd, tctx, hh->text, hh->slen)) return FALSE;
511a6c14
JH
704 store_reset(reset_point);
705 continue; /* With the next header line */
706 }
707 }
708
709 /* Either no rewriting rules, or it didn't get rewritten */
710
6d5c916c 711 if (!sendfn(fd, tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
712 }
713
714 /* Header removed */
715
716 else
717 {
718 DEBUG(D_transport) debug_printf("removed header line:\n%s---\n", h->text);
719 }
720 }
721
722/* Add on any address-specific headers. If there are multiple addresses,
723they will all have the same headers in order to be batched. The headers
724are chained in reverse order of adding (so several addresses from the
725same alias might share some of them) but we want to output them in the
726opposite order. This is a bit tedious, but there shouldn't be very many
727of them. We just walk the list twice, reversing the pointers each time,
728but on the second time, write out the items.
729
730Headers added to an address by a router are guaranteed to end with a newline.
731*/
732
733if (addr)
734 {
735 int i;
d43cbe25 736 header_line *hprev = addr->prop.extra_headers;
511a6c14
JH
737 header_line *hnext;
738 for (i = 0; i < 2; i++)
ff5aac2b 739 for (h = hprev, hprev = NULL; h; h = hnext)
511a6c14
JH
740 {
741 hnext = h->next;
742 h->next = hprev;
743 hprev = h;
744 if (i == 1)
745 {
6d5c916c 746 if (!sendfn(fd, tctx, h->text, h->slen)) return FALSE;
511a6c14
JH
747 DEBUG(D_transport)
748 debug_printf("added header line(s):\n%s---\n", h->text);
749 }
750 }
511a6c14
JH
751 }
752
76146973
JH
753/* If a string containing additional headers exists it is a newline-sep
754list. Expand each item and write out the result. This is done last so that
755if it (deliberately or accidentally) isn't in header format, it won't mess
756up any other headers. An empty string or a forced expansion failure are
757noops. An added header string from a transport may not end with a newline;
758add one if it does not. */
511a6c14 759
fb6833e0 760if (tblock && (list = CUS tblock->add_headers))
511a6c14 761 {
76146973
JH
762 int sep = '\n';
763 uschar * s;
764
6b46ecc6 765 while ((s = string_nextinlist(&list, &sep, NULL, 0)))
ff5aac2b 766 if ((s = expand_string(s)))
76146973
JH
767 {
768 int len = Ustrlen(s);
769 if (len > 0)
511a6c14 770 {
6d5c916c
JH
771 if (!sendfn(fd, tctx, s, len)) return FALSE;
772 if (s[len-1] != '\n' && !sendfn(fd, tctx, US"\n", 1))
76146973
JH
773 return FALSE;
774 DEBUG(D_transport)
775 {
776 debug_printf("added header line:\n%s", s);
777 if (s[len-1] != '\n') debug_printf("\n");
778 debug_printf("---\n");
779 }
511a6c14
JH
780 }
781 }
ff5aac2b
JH
782 else if (!expand_string_forcedfail)
783 { errno = ERRNO_CHHEADER_FAIL; return FALSE; }
511a6c14
JH
784 }
785
786/* Separate headers from body with a blank line */
787
6d5c916c 788return sendfn(fd, tctx, US"\n", 1);
511a6c14
JH
789}
790
791
059ec3d9
PH
792/*************************************************
793* Write the message *
794*************************************************/
795
796/* This function writes the message to the given file descriptor. The headers
797are in the in-store data structure, and the rest of the message is in the open
798file descriptor deliver_datafile. Make sure we start it at the beginning.
799
800. If add_return_path is TRUE, a "return-path:" header is added to the message,
801 containing the envelope sender's address.
802
803. If add_envelope_to is TRUE, a "envelope-to:" header is added to the message,
804 giving the top-level envelope address that caused this delivery to happen.
805
806. If add_delivery_date is TRUE, a "delivery-date:" header is added to the
807 message. It gives the time and date that delivery took place.
808
809. If check_string is not null, the start of each line is checked for that
810 string. If it is found, it is replaced by escape_string. This used to be
811 the "from hack" for files, and "smtp_dots" for escaping SMTP dots.
812
813. If use_crlf is true, newlines are turned into CRLF (SMTP output).
814
815The yield is TRUE if all went well, and FALSE if not. Exit *immediately* after
816any writing or reading error, leaving the code in errno intact. Error exits
817can include timeouts for certain transports, which are requested by setting
818transport_write_timeout non-zero.
819
820Arguments:
059ec3d9 821 fd file descriptor to write the message to
6b46ecc6
JH
822 tctx
823 addr (chain of) addresses (for extra headers), or NULL;
824 only the first address is used
825 tblock optional transport instance block (NULL signifies NULL/0):
826 add_headers a string containing one or more headers to add; it is
827 expanded, and must be in correct RFC 822 format as
828 it is transmitted verbatim; NULL => no additions,
829 and so does empty string or forced expansion fail
830 remove_headers a colon-separated list of headers to remove, or NULL
831 rewrite_rules chain of header rewriting rules
832 rewrite_existflags flags for the rewriting rules
833 options bit-wise options:
834 add_return_path if TRUE, add a "return-path" header
835 add_envelope_to if TRUE, add a "envelope-to" header
836 add_delivery_date if TRUE, add a "delivery-date" header
837 use_crlf if TRUE, turn NL into CR LF
838 end_dot if TRUE, send a terminating "." line at the end
839 no_headers if TRUE, omit the headers
840 no_body if TRUE, omit the body
2d14f397
JH
841 check_string a string to check for at the start of lines, or NULL
842 escape_string a string to insert in front of any check string
843 size_limit if > 0, this is a limit to the size of message written;
6b46ecc6
JH
844 it is used when returning messages to their senders,
845 and is approximate rather than exact, owing to chunk
846 buffering
059ec3d9
PH
847
848Returns: TRUE on success; FALSE (with errno) on failure.
849 In addition, the global variable transport_count
850 is incremented by the number of bytes written.
851*/
852
853static BOOL
6b46ecc6 854internal_transport_write_message(int fd, transport_ctx * tctx, int size_limit)
059ec3d9 855{
059ec3d9 856int len;
059ec3d9
PH
857
858/* Initialize pointer in output buffer. */
859
860chunk_ptr = deliver_out_buffer;
861
862/* Set up the data for start-of-line data checking and escaping */
863
864nl_partial_match = -1;
6b46ecc6 865if (tctx->check_string && tctx->escape_string)
059ec3d9 866 {
6b46ecc6 867 nl_check = tctx->check_string;
059ec3d9 868 nl_check_length = Ustrlen(nl_check);
6b46ecc6 869 nl_escape = tctx->escape_string;
059ec3d9
PH
870 nl_escape_length = Ustrlen(nl_escape);
871 }
ff5aac2b
JH
872else
873 nl_check_length = nl_escape_length = 0;
059ec3d9
PH
874
875/* Whether the escaping mechanism is applied to headers or not is controlled by
876an option (set for SMTP, not otherwise). Negate the length if not wanted till
877after the headers. */
878
6b46ecc6 879if (!(tctx->options & topt_escape_headers))
ff5aac2b 880 nl_check_length = -nl_check_length;
059ec3d9
PH
881
882/* Write the headers if required, including any that have to be added. If there
883are header rewriting rules, apply them. */
884
6b46ecc6 885if (!(tctx->options & topt_no_headers))
059ec3d9
PH
886 {
887 /* Add return-path: if requested. */
888
6b46ecc6 889 if (tctx->options & topt_add_return_path)
059ec3d9
PH
890 {
891 uschar buffer[ADDRESS_MAXLENGTH + 20];
ff5aac2b 892 int n = sprintf(CS buffer, "Return-path: <%.*s>\n", ADDRESS_MAXLENGTH,
059ec3d9 893 return_path);
6d5c916c 894 if (!write_chunk(fd, tctx, buffer, n)) return FALSE;
059ec3d9
PH
895 }
896
897 /* Add envelope-to: if requested */
898
6b46ecc6 899 if (tctx->options & topt_add_envelope_to)
059ec3d9
PH
900 {
901 BOOL first = TRUE;
902 address_item *p;
903 struct aci *plist = NULL;
904 struct aci *dlist = NULL;
905 void *reset_point = store_get(0);
906
6d5c916c 907 if (!write_chunk(fd, tctx, US"Envelope-to: ", 13)) return FALSE;
059ec3d9
PH
908
909 /* Pick up from all the addresses. The plist and dlist variables are
910 anchors for lists of addresses already handled; they have to be defined at
911 this level becuase write_env_to() calls itself recursively. */
912
6b46ecc6 913 for (p = tctx->addr; p; p = p->next)
6d5c916c 914 if (!write_env_to(p, &plist, &dlist, &first, fd, tctx))
ff5aac2b 915 return FALSE;
059ec3d9
PH
916
917 /* Add a final newline and reset the store used for tracking duplicates */
918
6d5c916c 919 if (!write_chunk(fd, tctx, US"\n", 1)) return FALSE;
059ec3d9
PH
920 store_reset(reset_point);
921 }
922
923 /* Add delivery-date: if requested. */
924
6b46ecc6 925 if (tctx->options & topt_add_delivery_date)
059ec3d9
PH
926 {
927 uschar buffer[100];
ff5aac2b 928 int n = sprintf(CS buffer, "Delivery-date: %s\n", tod_stamp(tod_full));
6d5c916c 929 if (!write_chunk(fd, tctx, buffer, n)) return FALSE;
059ec3d9
PH
930 }
931
932 /* Then the message's headers. Don't write any that are flagged as "old";
933 that means they were rewritten, or are a record of envelope rewriting, or
934 were removed (e.g. Bcc). If remove_headers is not null, skip any headers that
d43cbe25 935 match any entries therein. Then check addr->prop.remove_headers too, provided that
059ec3d9 936 addr is not NULL. */
6b46ecc6 937
6d5c916c 938 if (!transport_headers_send(fd, tctx, &write_chunk))
511a6c14 939 return FALSE;
059ec3d9
PH
940 }
941
58fc5fb2
JH
942/* When doing RFC3030 CHUNKING output, work out how much data would be in a
943last-BDAT, consisting of the current write_chunk() output buffer fill
59932f7d
JH
944(optimally, all of the headers - but it does not matter if we already had to
945flush that buffer with non-last BDAT prependix) plus the amount of body data
58fc5fb2 946(as expanded for CRLF lines). Then create and write BDAT(s), and ensure
e027f545
JH
947that further use of write_chunk() will not prepend BDATs.
948The first BDAT written will also first flush any outstanding MAIL and RCPT
949commands which were buffered thans to PIPELINING.
950Commands go out (using a send()) from a different buffer to data (using a
951write()). They might not end up in the same TCP segment, which is
952suboptimal. */
59932f7d 953
65de12cc 954if (tctx->options & topt_use_bdat)
59932f7d 955 {
e027f545 956 off_t fsize;
4dc2379a 957 int hsize, size = 0;
e027f545
JH
958
959 if ((hsize = chunk_ptr - deliver_out_buffer) < 0)
960 hsize = 0;
65de12cc 961 if (!(tctx->options & topt_no_body))
59932f7d
JH
962 {
963 if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
964 fsize -= SPOOL_DATA_START_OFFSET;
965 if (size_limit > 0 && fsize > size_limit)
966 fsize = size_limit;
e027f545 967 size = hsize + fsize;
65de12cc 968 if (tctx->options & topt_use_crlf)
59932f7d
JH
969 size += body_linecount; /* account for CRLF-expansion */
970 }
971
e027f545
JH
972 /* If the message is large, emit first a non-LAST chunk with just the
973 headers, and reap the command responses. This lets us error out early
974 on RCPT rejects rather than sending megabytes of data. Include headers
975 on the assumption they are cheap enough and some clever implementations
976 might errorcheck them too, on-the-fly, and reject that chunk. */
977
978 if (size > DELIVER_OUT_BUFFER_SIZE && hsize > 0)
979 {
58fc5fb2
JH
980 DEBUG(D_transport)
981 debug_printf("sending small initial BDAT; hssize=%d\n", hsize);
e027f545
JH
982 if ( tctx->chunk_cb(fd, tctx, hsize, 0) != OK
983 || !transport_write_block(fd, deliver_out_buffer, hsize)
984 || tctx->chunk_cb(fd, tctx, 0, tc_reap_prev) != OK
985 )
986 return FALSE;
987 chunk_ptr = deliver_out_buffer;
988 size -= hsize;
989 }
990
991 /* Emit a LAST datachunk command. */
59932f7d 992
e027f545 993 if (tctx->chunk_cb(fd, tctx, size, tc_chunk_last) != OK)
511a6c14 994 return FALSE;
59932f7d 995
6d5c916c 996 tctx->options &= ~topt_use_bdat;
059ec3d9
PH
997 }
998
999/* If the body is required, ensure that the data for check strings (formerly
1000the "from hack") is enabled by negating the length if necessary. (It will be
1001negative in cases where it isn't to apply to the headers). Then ensure the body
1002is positioned at the start of its file (following the message id), then write
1003it, applying the size limit if required. */
1004
6b46ecc6 1005if (!(tctx->options & topt_no_body))
059ec3d9 1006 {
e027f545
JH
1007 int size = size_limit;
1008
059ec3d9
PH
1009 nl_check_length = abs(nl_check_length);
1010 nl_partial_match = 0;
d4ff61d1
JH
1011 if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
1012 return FALSE;
59932f7d
JH
1013 while ( (len = MAX(DELIVER_IN_BUFFER_SIZE, size)) > 0
1014 && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
059ec3d9 1015 {
6d5c916c 1016 if (!write_chunk(fd, tctx, deliver_in_buffer, len))
59932f7d 1017 return FALSE;
e027f545 1018 size -= len;
059ec3d9
PH
1019 }
1020
059ec3d9
PH
1021 /* A read error on the body will have left len == -1 and errno set. */
1022
1023 if (len != 0) return FALSE;
c0940526 1024 }
059ec3d9 1025
c0940526 1026/* Finished with the check string */
059ec3d9 1027
c0940526
PP
1028nl_check_length = nl_escape_length = 0;
1029
1030/* If requested, add a terminating "." line (SMTP output). */
1031
6d5c916c 1032if (tctx->options & topt_end_dot && !write_chunk(fd, tctx, US".\n", 2))
c0940526 1033 return FALSE;
059ec3d9
PH
1034
1035/* Write out any remaining data in the buffer before returning. */
1036
1037return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
1038 transport_write_block(fd, deliver_out_buffer, len);
1039}
1040
1041
80a47a2c 1042#ifndef DISABLE_DKIM
fb2274d4 1043
4cd12fe9
TK
1044/***************************************************************************************************
1045* External interface to write the message, while signing it with DKIM and/or Domainkeys *
1046***************************************************************************************************/
fb2274d4 1047
077ba943
JH
1048/* This function is a wrapper around transport_write_message().
1049 It is only called from the smtp transport if DKIM or Domainkeys support
1050 is compiled in. The function sets up a replacement fd into a -K file,
1051 then calls the normal function. This way, the exact bits that exim would
1052 have put "on the wire" will end up in the file (except for TLS
1053 encapsulation, which is the very very last thing). When we are done
1054 signing the file, send the signed message down the original fd (or TLS fd).
1055
1056Arguments:
ff5aac2b
JH
1057 as for internal_transport_write_message() above, with additional arguments
1058 for DKIM.
f7572e5a
TK
1059
1060Returns: TRUE on success; FALSE (with errno) for any failure
1061*/
1062
1063BOOL
6b46ecc6
JH
1064dkim_transport_write_message(int out_fd, transport_ctx * tctx,
1065 struct ob_dkim * dkim)
f7572e5a 1066{
077ba943
JH
1067int dkim_fd;
1068int save_errno = 0;
1069BOOL rc;
41313d92 1070uschar * dkim_spool_name;
077ba943
JH
1071int sread = 0;
1072int wwritten = 0;
1073uschar *dkim_signature = NULL;
48f1c853 1074int siglen = 0;
ff5aac2b 1075off_t k_file_size;
48f1c853 1076int options;
077ba943
JH
1077
1078/* If we can't sign, just call the original function. */
1079
ff5aac2b 1080if (!(dkim->dkim_private_key && dkim->dkim_domain && dkim->dkim_selector))
6b46ecc6 1081 return transport_write_message(out_fd, tctx, 0);
077ba943 1082
41313d92
JH
1083dkim_spool_name = spool_fname(US"input", message_subdir, message_id,
1084 string_sprintf("-%d-K", (int)getpid()));
077ba943
JH
1085
1086if ((dkim_fd = Uopen(dkim_spool_name, O_RDWR|O_CREAT|O_TRUNC, SPOOL_MODE)) < 0)
1087 {
1088 /* Can't create spool file. Ugh. */
1089 rc = FALSE;
1090 save_errno = errno;
1091 goto CLEANUP;
4cd12fe9
TK
1092 }
1093
e983e85a
JH
1094/* Call original function to write the -K file; does the CRLF expansion
1095(but, in the CHUNKING case, not dot-stuffing and dot-termination). */
f7572e5a 1096
48f1c853 1097options = tctx->options;
65de12cc 1098tctx->options &= ~topt_use_bdat;
6b46ecc6 1099rc = transport_write_message(dkim_fd, tctx, 0);
48f1c853 1100tctx->options = options;
f7572e5a 1101
077ba943
JH
1102/* Save error state. We must clean up before returning. */
1103if (!rc)
1104 {
1105 save_errno = errno;
1106 goto CLEANUP;
1107 }
f7572e5a 1108
48f1c853 1109/* Rewind file and feed it to the goats^W DKIM lib */
e983e85a 1110dkim->dot_stuffed = !!(options & topt_end_dot);
48f1c853 1111lseek(dkim_fd, 0, SEEK_SET);
e983e85a 1112if ((dkim_signature = dkim_exim_sign(dkim_fd, dkim)))
48f1c853
JH
1113 siglen = Ustrlen(dkim_signature);
1114else if (dkim->dkim_strict)
077ba943 1115 {
48f1c853
JH
1116 uschar *dkim_strict_result = expand_string(dkim->dkim_strict);
1117 if (dkim_strict_result)
1118 if ( (strcmpic(dkim->dkim_strict,US"1") == 0) ||
1119 (strcmpic(dkim->dkim_strict,US"true") == 0) )
077ba943 1120 {
48f1c853
JH
1121 /* Set errno to something halfway meaningful */
1122 save_errno = EACCES;
1123 log_write(0, LOG_MAIN, "DKIM: message could not be signed,"
1124 " and dkim_strict is set. Deferring message delivery.");
1125 rc = FALSE;
1126 goto CLEANUP;
f7572e5a 1127 }
e520153e 1128 }
ff5aac2b 1129
48f1c853
JH
1130#ifndef HAVE_LINUX_SENDFILE
1131if (options & topt_use_bdat)
6b46ecc6 1132#endif
48f1c853
JH
1133 k_file_size = lseek(dkim_fd, 0, SEEK_END); /* Fetch file size */
1134
1135if (options & topt_use_bdat)
e520153e 1136 {
48f1c853
JH
1137
1138 /* On big messages output a precursor chunk to get any pipelined
1139 MAIL & RCPT commands flushed, then reap the responses so we can
1140 error out on RCPT rejects before sending megabytes. */
1141
1142 if (siglen + k_file_size > DELIVER_OUT_BUFFER_SIZE && siglen > 0)
077ba943 1143 {
48f1c853
JH
1144 if ( tctx->chunk_cb(out_fd, tctx, siglen, 0) != OK
1145 || !transport_write_block(out_fd, dkim_signature, siglen)
1146 || tctx->chunk_cb(out_fd, tctx, 0, tc_reap_prev) != OK
1147 )
1148 goto err;
1149 siglen = 0;
f7572e5a 1150 }
48f1c853
JH
1151
1152 if (tctx->chunk_cb(out_fd, tctx, siglen + k_file_size, tc_chunk_last) != OK)
1153 goto err;
4cd12fe9 1154 }
f7572e5a 1155
48f1c853
JH
1156if(siglen > 0 && !transport_write_block(out_fd, dkim_signature, siglen))
1157 goto err;
1158
f7572e5a 1159#ifdef HAVE_LINUX_SENDFILE
077ba943
JH
1160/* We can use sendfile() to shove the file contents
1161 to the socket. However only if we don't use TLS,
1162 as then there's another layer of indirection
1163 before the data finally hits the socket. */
ff5aac2b 1164if (tls_out.active != out_fd)
077ba943
JH
1165 {
1166 ssize_t copied = 0;
1167 off_t offset = 0;
e8bc7fca
JH
1168
1169 /* Rewind file */
1170 lseek(dkim_fd, 0, SEEK_SET);
1171
ff5aac2b
JH
1172 while(copied >= 0 && offset < k_file_size)
1173 copied = sendfile(out_fd, dkim_fd, &offset, k_file_size - offset);
077ba943 1174 if (copied < 0)
48f1c853 1175 goto err;
077ba943 1176 }
e8bc7fca
JH
1177else
1178
f7572e5a
TK
1179#endif
1180
077ba943 1181 {
e8bc7fca
JH
1182 /* Rewind file */
1183 lseek(dkim_fd, 0, SEEK_SET);
077ba943 1184
e8bc7fca 1185 /* Send file down the original fd */
ff5aac2b 1186 while((sread = read(dkim_fd, deliver_out_buffer, DELIVER_OUT_BUFFER_SIZE)) >0)
f7572e5a 1187 {
4dc2379a 1188 uschar * p = deliver_out_buffer;
e8bc7fca
JH
1189 /* write the chunk */
1190
1191 while (sread)
1192 {
077ba943 1193#ifdef SUPPORT_TLS
ff5aac2b 1194 wwritten = tls_out.active == out_fd
4dc2379a
JH
1195 ? tls_write(FALSE, p, sread)
1196 : write(out_fd, CS p, sread);
077ba943 1197#else
4dc2379a 1198 wwritten = write(out_fd, CS p, sread);
077ba943 1199#endif
e8bc7fca 1200 if (wwritten == -1)
48f1c853 1201 goto err;
e8bc7fca
JH
1202 p += wwritten;
1203 sread -= wwritten;
f7572e5a 1204 }
f7572e5a
TK
1205 }
1206
e8bc7fca
JH
1207 if (sread == -1)
1208 {
1209 save_errno = errno;
1210 rc = FALSE;
1211 }
077ba943 1212 }
f7572e5a 1213
077ba943 1214CLEANUP:
48f1c853
JH
1215 /* unlink -K file */
1216 (void)close(dkim_fd);
1217 Uunlink(dkim_spool_name);
1218 errno = save_errno;
1219 return rc;
1220
1221err:
1222 save_errno = errno;
1223 rc = FALSE;
1224 goto CLEANUP;
f7572e5a 1225}
80a47a2c 1226
f7572e5a
TK
1227#endif
1228
1229
1230
059ec3d9
PH
1231/*************************************************
1232* External interface to write the message *
1233*************************************************/
1234
1235/* If there is no filtering required, call the internal function above to do
1236the real work, passing over all the arguments from this function. Otherwise,
1237set up a filtering process, fork another process to call the internal function
1238to write to the filter, and in this process just suck from the filter and write
1239down the given fd. At the end, tidy up the pipes and the processes.
1240
6b46ecc6 1241XXX
059ec3d9
PH
1242Arguments: as for internal_transport_write_message() above
1243
1244Returns: TRUE on success; FALSE (with errno) for any failure
1245 transport_count is incremented by the number of bytes written
1246*/
1247
1248BOOL
6b46ecc6 1249transport_write_message(int fd, transport_ctx * tctx, int size_limit)
059ec3d9 1250{
059ec3d9
PH
1251BOOL last_filter_was_NL = TRUE;
1252int rc, len, yield, fd_read, fd_write, save_errno;
806c3df9 1253int pfd[2] = {-1, -1};
059ec3d9 1254pid_t filter_pid, write_pid;
6d5c916c 1255static transport_ctx dummy_tctx = {0};
6b46ecc6
JH
1256
1257if (!tctx) tctx = &dummy_tctx;
059ec3d9 1258
2e2a30b4
PH
1259transport_filter_timed_out = FALSE;
1260
059ec3d9
PH
1261/* If there is no filter command set up, call the internal function that does
1262the actual work, passing it the incoming fd, and return its result. */
1263
6ff55e50
JH
1264if ( !transport_filter_argv
1265 || !*transport_filter_argv
1266 || !**transport_filter_argv
1267 )
6b46ecc6 1268 return internal_transport_write_message(fd, tctx, size_limit);
059ec3d9
PH
1269
1270/* Otherwise the message must be written to a filter process and read back
1271before being written to the incoming fd. First set up the special processing to
1272be done during the copying. */
1273
059ec3d9
PH
1274nl_partial_match = -1;
1275
6b46ecc6 1276if (tctx->check_string && tctx->escape_string)
059ec3d9 1277 {
6b46ecc6 1278 nl_check = tctx->check_string;
059ec3d9 1279 nl_check_length = Ustrlen(nl_check);
6b46ecc6 1280 nl_escape = tctx->escape_string;
059ec3d9
PH
1281 nl_escape_length = Ustrlen(nl_escape);
1282 }
1283else nl_check_length = nl_escape_length = 0;
1284
1285/* Start up a subprocess to run the command. Ensure that our main fd will
1286be closed when the subprocess execs, but remove the flag afterwards.
1287(Otherwise, if this is a TCP/IP socket, it can't get passed on to another
1288process to deliver another message.) We get back stdin/stdout file descriptors.
1289If the process creation failed, give an error return. */
1290
1291fd_read = -1;
1292fd_write = -1;
1293save_errno = 0;
1294yield = FALSE;
1295write_pid = (pid_t)(-1);
1296
92b0827a
JH
1297 {
1298 int bits = fcntl(fd, F_GETFD);
1299 (void)fcntl(fd, F_SETFD, bits | FD_CLOEXEC);
1300 filter_pid = child_open(USS transport_filter_argv, NULL, 077,
1301 &fd_write, &fd_read, FALSE);
1302 (void)fcntl(fd, F_SETFD, bits & ~FD_CLOEXEC);
1303 }
059ec3d9
PH
1304if (filter_pid < 0) goto TIDY_UP; /* errno set */
1305
1306DEBUG(D_transport)
fb6833e0 1307 debug_printf("process %d running as transport filter: fd_write=%d fd_read=%d\n",
059ec3d9
PH
1308 (int)filter_pid, fd_write, fd_read);
1309
1310/* Fork subprocess to write the message to the filter, and return the result
1311via a(nother) pipe. While writing to the filter, we do not do the CRLF,
1312smtp dots, or check string processing. */
1313
1314if (pipe(pfd) != 0) goto TIDY_UP; /* errno set */
1315if ((write_pid = fork()) == 0)
1316 {
1317 BOOL rc;
f1e894f3
PH
1318 (void)close(fd_read);
1319 (void)close(pfd[pipe_read]);
059ec3d9 1320 nl_check_length = nl_escape_length = 0;
6b46ecc6 1321
fb6833e0 1322 tctx->check_string = tctx->escape_string = NULL;
65de12cc 1323 tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
6b46ecc6
JH
1324
1325 rc = internal_transport_write_message(fd_write, tctx, size_limit);
1326
059ec3d9 1327 save_errno = errno;
1ac6b2e7
JH
1328 if ( write(pfd[pipe_write], (void *)&rc, sizeof(BOOL))
1329 != sizeof(BOOL)
1330 || write(pfd[pipe_write], (void *)&save_errno, sizeof(int))
1331 != sizeof(int)
6b46ecc6 1332 || write(pfd[pipe_write], (void *)&tctx->addr->more_errno, sizeof(int))
1ac6b2e7
JH
1333 != sizeof(int)
1334 )
1335 rc = FALSE; /* compiler quietening */
059ec3d9
PH
1336 _exit(0);
1337 }
1338save_errno = errno;
1339
1340/* Parent process: close our copy of the writing subprocess' pipes. */
1341
f1e894f3
PH
1342(void)close(pfd[pipe_write]);
1343(void)close(fd_write);
059ec3d9
PH
1344fd_write = -1;
1345
1346/* Writing process creation failed */
1347
1348if (write_pid < 0)
1349 {
1350 errno = save_errno; /* restore */
1351 goto TIDY_UP;
1352 }
1353
1354/* When testing, let the subprocess get going */
1355
1356if (running_in_test_harness) millisleep(250);
1357
1358DEBUG(D_transport)
1359 debug_printf("process %d writing to transport filter\n", (int)write_pid);
1360
1361/* Copy the message from the filter to the output fd. A read error leaves len
1362== -1 and errno set. We need to apply a timeout to the read, to cope with
1363the case when the filter gets stuck, but it can be quite a long one. The
1364default is 5m, but this is now configurable. */
1365
1366DEBUG(D_transport) debug_printf("copying from the filter\n");
1367
1368/* Copy the output of the filter, remembering if the last character was NL. If
1369no data is returned, that counts as "ended with NL" (default setting of the
1370variable is TRUE). */
1371
1372chunk_ptr = deliver_out_buffer;
1373
1374for (;;)
1375 {
1376 sigalrm_seen = FALSE;
1377 alarm(transport_filter_timeout);
1378 len = read(fd_read, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE);
1379 alarm(0);
1380 if (sigalrm_seen)
1381 {
1382 errno = ETIMEDOUT;
2e2a30b4 1383 transport_filter_timed_out = TRUE;
059ec3d9
PH
1384 goto TIDY_UP;
1385 }
1386
1387 /* If the read was successful, write the block down the original fd,
1388 remembering whether it ends in \n or not. */
1389
1390 if (len > 0)
1391 {
6d5c916c 1392 if (!write_chunk(fd, tctx, deliver_in_buffer, len)) goto TIDY_UP;
059ec3d9
PH
1393 last_filter_was_NL = (deliver_in_buffer[len-1] == '\n');
1394 }
1395
1396 /* Otherwise, break the loop. If we have hit EOF, set yield = TRUE. */
1397
1398 else
1399 {
1400 if (len == 0) yield = TRUE;
1401 break;
1402 }
1403 }
1404
1405/* Tidying up code. If yield = FALSE there has been an error and errno is set
1406to something. Ensure the pipes are all closed and the processes are removed. If
1407there has been an error, kill the processes before waiting for them, just to be
1408sure. Also apply a paranoia timeout. */
1409
1410TIDY_UP:
1411save_errno = errno;
1412
f1e894f3
PH
1413(void)close(fd_read);
1414if (fd_write > 0) (void)close(fd_write);
059ec3d9
PH
1415
1416if (!yield)
1417 {
1418 if (filter_pid > 0) kill(filter_pid, SIGKILL);
1419 if (write_pid > 0) kill(write_pid, SIGKILL);
1420 }
1421
1422/* Wait for the filter process to complete. */
1423
1424DEBUG(D_transport) debug_printf("waiting for filter process\n");
1425if (filter_pid > 0 && (rc = child_close(filter_pid, 30)) != 0 && yield)
1426 {
1427 yield = FALSE;
1428 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1429 tctx->addr->more_errno = rc;
059ec3d9
PH
1430 DEBUG(D_transport) debug_printf("filter process returned %d\n", rc);
1431 }
1432
1433/* Wait for the writing process to complete. If it ends successfully,
8e669ac1 1434read the results from its pipe, provided we haven't already had a filter
35af9f61 1435process failure. */
059ec3d9
PH
1436
1437DEBUG(D_transport) debug_printf("waiting for writing process\n");
1438if (write_pid > 0)
1439 {
35af9f61
PH
1440 rc = child_close(write_pid, 30);
1441 if (yield)
8e669ac1 1442 if (rc == 0)
35af9f61
PH
1443 {
1444 BOOL ok;
4dc2379a
JH
1445 if (read(pfd[pipe_read], (void *)&ok, sizeof(BOOL)) != sizeof(BOOL))
1446 {
1447 DEBUG(D_transport)
1448 debug_printf("pipe read from writing process: %s\n", strerror(errno));
1449 save_errno = ERRNO_FILTER_FAIL;
1450 yield = FALSE;
1451 }
1452 else if (!ok)
35af9f61 1453 {
4dc2379a 1454 int dummy = read(pfd[pipe_read], (void *)&save_errno, sizeof(int));
6b46ecc6 1455 dummy = read(pfd[pipe_read], (void *)&(tctx->addr->more_errno), sizeof(int));
35af9f61
PH
1456 yield = FALSE;
1457 }
1458 }
1459 else
059ec3d9 1460 {
059ec3d9 1461 yield = FALSE;
35af9f61 1462 save_errno = ERRNO_FILTER_FAIL;
6b46ecc6 1463 tctx->addr->more_errno = rc;
35af9f61 1464 DEBUG(D_transport) debug_printf("writing process returned %d\n", rc);
059ec3d9 1465 }
059ec3d9 1466 }
f1e894f3 1467(void)close(pfd[pipe_read]);
059ec3d9
PH
1468
1469/* If there have been no problems we can now add the terminating "." if this is
1470SMTP output, turning off escaping beforehand. If the last character from the
1471filter was not NL, insert a NL to make the SMTP protocol work. */
1472
1473if (yield)
1474 {
1475 nl_check_length = nl_escape_length = 0;
6b46ecc6 1476 if ( tctx->options & topt_end_dot
ff5aac2b 1477 && ( last_filter_was_NL
6d5c916c
JH
1478 ? !write_chunk(fd, tctx, US".\n", 2)
1479 : !write_chunk(fd, tctx, US"\n.\n", 3)
ff5aac2b 1480 ) )
059ec3d9 1481 yield = FALSE;
059ec3d9
PH
1482
1483 /* Write out any remaining data in the buffer. */
1484
1485 else
ff5aac2b
JH
1486 yield = (len = chunk_ptr - deliver_out_buffer) <= 0
1487 || transport_write_block(fd, deliver_out_buffer, len);
059ec3d9 1488 }
ff5aac2b
JH
1489else
1490 errno = save_errno; /* From some earlier error */
059ec3d9
PH
1491
1492DEBUG(D_transport)
1493 {
1494 debug_printf("end of filtering transport writing: yield=%d\n", yield);
1495 if (!yield)
6b46ecc6 1496 debug_printf("errno=%d more_errno=%d\n", errno, tctx->addr->more_errno);
059ec3d9
PH
1497 }
1498
1499return yield;
1500}
1501
1502
1503
1504
1505
1506/*************************************************
1507* Update waiting database *
1508*************************************************/
1509
1510/* This is called when an address is deferred by remote transports that are
1511capable of sending more than one message over one connection. A database is
1512maintained for each transport, keeping track of which messages are waiting for
1513which hosts. The transport can then consult this when eventually a successful
1514delivery happens, and if it finds that another message is waiting for the same
1515host, it can fire up a new process to deal with it using the same connection.
1516
1517The database records are keyed by host name. They can get full if there are
1518lots of messages waiting, and so there is a continuation mechanism for them.
1519
1520Each record contains a list of message ids, packed end to end without any
1521zeros. Each one is MESSAGE_ID_LENGTH bytes long. The count field says how many
1522in this record, and the sequence field says if there are any other records for
1523this host. If the sequence field is 0, there are none. If it is 1, then another
1524record with the name <hostname>:0 exists; if it is 2, then two other records
1525with sequence numbers 0 and 1 exist, and so on.
1526
1527Currently, an exhaustive search of all continuation records has to be done to
1528determine whether to add a message id to a given record. This shouldn't be
1529too bad except in extreme cases. I can't figure out a *simple* way of doing
1530better.
1531
1532Old records should eventually get swept up by the exim_tidydb utility.
1533
1534Arguments:
f6c332bd 1535 hostlist list of hosts that this message could be sent to
059ec3d9
PH
1536 tpname name of the transport
1537
1538Returns: nothing
1539*/
1540
1541void
1542transport_update_waiting(host_item *hostlist, uschar *tpname)
1543{
1544uschar buffer[256];
55414b25 1545const uschar *prevname = US"";
059ec3d9
PH
1546host_item *host;
1547open_db dbblock;
1548open_db *dbm_file;
1549
7a0743eb
PH
1550DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
1551
059ec3d9
PH
1552/* Open the database for this transport */
1553
1554sprintf(CS buffer, "wait-%.200s", tpname);
1555dbm_file = dbfn_open(buffer, O_RDWR, &dbblock, TRUE);
1556if (dbm_file == NULL) return;
1557
1558/* Scan the list of hosts for which this message is waiting, and ensure
f6c332bd 1559that the message id is in each host record. */
059ec3d9
PH
1560
1561for (host = hostlist; host!= NULL; host = host->next)
1562 {
1563 BOOL already = FALSE;
1564 dbdata_wait *host_record;
1565 uschar *s;
1566 int i, host_length;
1567
059ec3d9
PH
1568 /* Skip if this is the same host as we just processed; otherwise remember
1569 the name for next time. */
1570
1571 if (Ustrcmp(prevname, host->name) == 0) continue;
1572 prevname = host->name;
1573
1574 /* Look up the host record; if there isn't one, make an empty one. */
1575
1576 host_record = dbfn_read(dbm_file, host->name);
1577 if (host_record == NULL)
1578 {
1579 host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH);
1580 host_record->count = host_record->sequence = 0;
1581 }
1582
1583 /* Compute the current length */
1584
1585 host_length = host_record->count * MESSAGE_ID_LENGTH;
1586
1587 /* Search the record to see if the current message is already in it. */
1588
1589 for (s = host_record->text; s < host_record->text + host_length;
1590 s += MESSAGE_ID_LENGTH)
1591 {
1592 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1593 { already = TRUE; break; }
1594 }
1595
1596 /* If we haven't found this message in the main record, search any
1597 continuation records that exist. */
1598
1599 for (i = host_record->sequence - 1; i >= 0 && !already; i--)
1600 {
1601 dbdata_wait *cont;
1602 sprintf(CS buffer, "%.200s:%d", host->name, i);
1603 cont = dbfn_read(dbm_file, buffer);
1604 if (cont != NULL)
1605 {
1606 int clen = cont->count * MESSAGE_ID_LENGTH;
1607 for (s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
1608 {
1609 if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
1610 { already = TRUE; break; }
1611 }
1612 }
1613 }
1614
1615 /* If this message is already in a record, no need to update. */
1616
7a0743eb
PH
1617 if (already)
1618 {
1619 DEBUG(D_transport) debug_printf("already listed for %s\n", host->name);
1620 continue;
1621 }
059ec3d9
PH
1622
1623
1624 /* If this record is full, write it out with a new name constructed
1625 from the sequence number, increase the sequence number, and empty
1626 the record. */
1627
1628 if (host_record->count >= WAIT_NAME_MAX)
1629 {
1630 sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
1631 dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
1632 host_record->sequence++;
1633 host_record->count = 0;
1634 host_length = 0;
1635 }
1636
1637 /* If this record is not full, increase the size of the record to
1638 allow for one new message id. */
1639
1640 else
1641 {
1642 dbdata_wait *newr =
1643 store_get(sizeof(dbdata_wait) + host_length + MESSAGE_ID_LENGTH);
1644 memcpy(newr, host_record, sizeof(dbdata_wait) + host_length);
1645 host_record = newr;
1646 }
1647
1648 /* Now add the new name on the end */
1649
1650 memcpy(host_record->text + host_length, message_id, MESSAGE_ID_LENGTH);
1651 host_record->count++;
1652 host_length += MESSAGE_ID_LENGTH;
1653
1654 /* Update the database */
1655
1656 dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
7a0743eb 1657 DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
059ec3d9
PH
1658 }
1659
1660/* All now done */
1661
1662dbfn_close(dbm_file);
1663}
1664
1665
1666
1667
1668/*************************************************
1669* Test for waiting messages *
1670*************************************************/
1671
1672/* This function is called by a remote transport which uses the previous
1673function to remember which messages are waiting for which remote hosts. It's
1674called after a successful delivery and its job is to check whether there is
1675another message waiting for the same host. However, it doesn't do this if the
1676current continue sequence is greater than the maximum supplied as an argument,
1677or greater than the global connection_max_messages, which, if set, overrides.
1678
1679Arguments:
1680 transport_name name of the transport
1681 hostname name of the host
1682 local_message_max maximum number of messages down one connection
1683 as set by the caller transport
1684 new_message_id set to the message id of a waiting message
1685 more set TRUE if there are yet more messages waiting
a39bd74d
JB
1686 oicf_func function to call to validate if it is ok to send
1687 to this message_id from the current instance.
1688 oicf_data opaque data for oicf_func
059ec3d9
PH
1689
1690Returns: TRUE if new_message_id set; FALSE otherwise
1691*/
1692
a39bd74d
JB
1693typedef struct msgq_s
1694{
1695 uschar message_id [MESSAGE_ID_LENGTH + 1];
1696 BOOL bKeep;
1697} msgq_t;
1698
059ec3d9 1699BOOL
55414b25 1700transport_check_waiting(const uschar *transport_name, const uschar *hostname,
a39bd74d 1701 int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data)
059ec3d9
PH
1702{
1703dbdata_wait *host_record;
0539a19d 1704int host_length;
059ec3d9
PH
1705open_db dbblock;
1706open_db *dbm_file;
1707uschar buffer[256];
1708
a39bd74d 1709int i;
a39bd74d 1710struct stat statbuf;
a39bd74d 1711
059ec3d9
PH
1712*more = FALSE;
1713
1714DEBUG(D_transport)
1715 {
1716 debug_printf("transport_check_waiting entered\n");
1717 debug_printf(" sequence=%d local_max=%d global_max=%d\n",
1718 continue_sequence, local_message_max, connection_max_messages);
1719 }
1720
1721/* Do nothing if we have hit the maximum number that can be send down one
1722connection. */
1723
1724if (connection_max_messages >= 0) local_message_max = connection_max_messages;
1725if (local_message_max > 0 && continue_sequence >= local_message_max)
1726 {
1727 DEBUG(D_transport)
1728 debug_printf("max messages for one connection reached: returning\n");
1729 return FALSE;
1730 }
1731
1732/* Open the waiting information database. */
1733
1734sprintf(CS buffer, "wait-%.200s", transport_name);
1735dbm_file = dbfn_open(buffer, O_RDWR, &dbblock, TRUE);
1736if (dbm_file == NULL) return FALSE;
1737
1738/* See if there is a record for this host; if not, there's nothing to do. */
1739
789f8a4f 1740if (!(host_record = dbfn_read(dbm_file, hostname)))
059ec3d9
PH
1741 {
1742 dbfn_close(dbm_file);
1743 DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname);
1744 return FALSE;
1745 }
1746
1747/* If the data in the record looks corrupt, just log something and
1748don't try to use it. */
1749
1750if (host_record->count > WAIT_NAME_MAX)
1751 {
1752 dbfn_close(dbm_file);
1753 log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
1754 "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
1755 return FALSE;
1756 }
1757
1758/* Scan the message ids in the record from the end towards the beginning,
1759until one is found for which a spool file actually exists. If the record gets
1760emptied, delete it and continue with any continuation records that may exist.
1761*/
1762
a39bd74d
JB
1763/* For Bug 1141, I refactored this major portion of the routine, it is risky
1764but the 1 off will remain without it. This code now allows me to SKIP over
1765a message I do not want to send out on this run. */
059ec3d9 1766
a39bd74d
JB
1767host_length = host_record->count * MESSAGE_ID_LENGTH;
1768
1769while (1)
059ec3d9 1770 {
789f8a4f
JH
1771 msgq_t *msgq;
1772 int msgq_count = 0;
1773 int msgq_actual = 0;
1774 BOOL bFound = FALSE;
1775 BOOL bContinuation = FALSE;
1776
a39bd74d 1777 /* create an array to read entire message queue into memory for processing */
059ec3d9 1778
40c90bca 1779 msgq = store_malloc(sizeof(msgq_t) * host_record->count);
a39bd74d
JB
1780 msgq_count = host_record->count;
1781 msgq_actual = msgq_count;
059ec3d9 1782
a39bd74d 1783 for (i = 0; i < host_record->count; ++i)
059ec3d9 1784 {
a39bd74d
JB
1785 msgq[i].bKeep = TRUE;
1786
1787 Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
059ec3d9 1788 MESSAGE_ID_LENGTH);
a39bd74d
JB
1789 msgq[i].message_id[MESSAGE_ID_LENGTH] = 0;
1790 }
1791
1792 /* first thing remove current message id if it exists */
059ec3d9 1793
a39bd74d
JB
1794 for (i = 0; i < msgq_count; ++i)
1795 if (Ustrcmp(msgq[i].message_id, message_id) == 0)
1796 {
1797 msgq[i].bKeep = FALSE;
1798 break;
1799 }
1800
1801 /* now find the next acceptable message_id */
1802
a39bd74d
JB
1803 for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
1804 {
41313d92
JH
1805 uschar subdir[2];
1806
1807 subdir[0] = split_spool_directory ? msgq[i].message_id[5] : 0;
1808 subdir[1] = 0;
059ec3d9 1809
41313d92
JH
1810 if (Ustat(spool_fname(US"input", subdir, msgq[i].message_id, US"-D"),
1811 &statbuf) != 0)
a39bd74d
JB
1812 msgq[i].bKeep = FALSE;
1813 else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data))
059ec3d9 1814 {
a39bd74d
JB
1815 Ustrcpy(new_message_id, msgq[i].message_id);
1816 msgq[i].bKeep = FALSE;
1817 bFound = TRUE;
059ec3d9
PH
1818 break;
1819 }
1820 }
1821
a39bd74d
JB
1822 /* re-count */
1823 for (msgq_actual = 0, i = 0; i < msgq_count; ++i)
1824 if (msgq[i].bKeep)
1825 msgq_actual++;
1826
1827 /* reassemble the host record, based on removed message ids, from in
789f8a4f 1828 memory queue */
a39bd74d
JB
1829
1830 if (msgq_actual <= 0)
1831 {
1832 host_length = 0;
1833 host_record->count = 0;
1834 }
1835 else
1836 {
1837 host_length = msgq_actual * MESSAGE_ID_LENGTH;
1838 host_record->count = msgq_actual;
1839
1840 if (msgq_actual < msgq_count)
1841 {
1842 int new_count;
1843 for (new_count = 0, i = 0; i < msgq_count; ++i)
1844 if (msgq[i].bKeep)
1845 Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH],
1846 msgq[i].message_id, MESSAGE_ID_LENGTH);
1847
1848 host_record->text[new_count * MESSAGE_ID_LENGTH] = 0;
1849 }
1850 }
1851
1852/* Jeremy: check for a continuation record, this code I do not know how to
1853test but the code should work */
1854
059ec3d9
PH
1855 while (host_length <= 0)
1856 {
1857 int i;
a39bd74d 1858 dbdata_wait * newr = NULL;
059ec3d9
PH
1859
1860 /* Search for a continuation */
1861
a39bd74d 1862 for (i = host_record->sequence - 1; i >= 0 && !newr; i--)
059ec3d9
PH
1863 {
1864 sprintf(CS buffer, "%.200s:%d", hostname, i);
1865 newr = dbfn_read(dbm_file, buffer);
1866 }
1867
1868 /* If no continuation, delete the current and break the loop */
1869
a39bd74d 1870 if (!newr)
059ec3d9
PH
1871 {
1872 dbfn_delete(dbm_file, hostname);
1873 break;
1874 }
1875
1876 /* Else replace the current with the continuation */
1877
1878 dbfn_delete(dbm_file, buffer);
1879 host_record = newr;
1880 host_length = host_record->count * MESSAGE_ID_LENGTH;
059ec3d9 1881
a39bd74d
JB
1882 bContinuation = TRUE;
1883 }
059ec3d9 1884
789f8a4f
JH
1885 if (bFound) /* Usual exit from main loop */
1886 {
40c90bca 1887 store_free (msgq);
a39bd74d 1888 break;
789f8a4f 1889 }
059ec3d9
PH
1890
1891 /* If host_length <= 0 we have emptied a record and not found a good message,
1892 and there are no continuation records. Otherwise there is a continuation
1893 record to process. */
1894
1895 if (host_length <= 0)
1896 {
1897 dbfn_close(dbm_file);
1898 DEBUG(D_transport) debug_printf("waiting messages already delivered\n");
1899 return FALSE;
1900 }
a39bd74d
JB
1901
1902 /* we were not able to find an acceptable message, nor was there a
1903 * continuation record. So bug out, outer logic will clean this up.
1904 */
1905
1906 if (!bContinuation)
1907 {
789f8a4f 1908 Ustrcpy(new_message_id, message_id);
a39bd74d
JB
1909 dbfn_close(dbm_file);
1910 return FALSE;
1911 }
a39bd74d 1912
40c90bca 1913 store_free(msgq);
789f8a4f 1914 } /* we need to process a continuation record */
059ec3d9
PH
1915
1916/* Control gets here when an existing message has been encountered; its
1917id is in new_message_id, and host_length is the revised length of the
1918host record. If it is zero, the record has been removed. Update the
1919record if required, close the database, and return TRUE. */
1920
1921if (host_length > 0)
1922 {
1923 host_record->count = host_length/MESSAGE_ID_LENGTH;
a39bd74d 1924
059ec3d9
PH
1925 dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
1926 *more = TRUE;
1927 }
1928
1929dbfn_close(dbm_file);
1930return TRUE;
1931}
1932
059ec3d9
PH
1933/*************************************************
1934* Deliver waiting message down same socket *
1935*************************************************/
1936
1937/* Fork a new exim process to deliver the message, and do a re-exec, both to
1938get a clean delivery process, and to regain root privilege in cases where it
1939has been given away.
1940
1941Arguments:
1942 transport_name to pass to the new process
1943 hostname ditto
1944 hostaddress ditto
1945 id the new message to process
1946 socket_fd the connected socket
1947
1948Returns: FALSE if fork fails; TRUE otherwise
1949*/
1950
1951BOOL
55414b25
JH
1952transport_pass_socket(const uschar *transport_name, const uschar *hostname,
1953 const uschar *hostaddress, uschar *id, int socket_fd)
059ec3d9
PH
1954{
1955pid_t pid;
1956int status;
1957
1958DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
1959
1960if ((pid = fork()) == 0)
1961 {
2d14f397 1962 int i = 17;
55414b25 1963 const uschar **argv;
059ec3d9
PH
1964
1965 /* Disconnect entirely from the parent process. If we are running in the
1966 test harness, wait for a bit to allow the previous process time to finish,
1967 write the log, etc., so that the output is always in the same order for
1968 automatic comparison. */
1969
1970 if ((pid = fork()) != 0) _exit(EXIT_SUCCESS);
ed0e9820 1971 if (running_in_test_harness) sleep(1);
059ec3d9
PH
1972
1973 /* Set up the calling arguments; use the standard function for the basics,
1974 but we have a number of extras that may be added. */
1975
55414b25 1976 argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0);
059ec3d9
PH
1977
1978 if (smtp_authenticated) argv[i++] = US"-MCA";
1979
2d14f397
JH
1980 if (smtp_peer_options & PEER_OFFERED_CHUNKING) argv[i++] = US"-MCK";
1981 if (smtp_peer_options & PEER_OFFERED_DSN) argv[i++] = US"-MCD";
1982 if (smtp_peer_options & PEER_OFFERED_PIPE) argv[i++] = US"-MCP";
1983 if (smtp_peer_options & PEER_OFFERED_SIZE) argv[i++] = US"-MCS";
1984#ifdef SUPPORT_TLS
1985 if (smtp_peer_options & PEER_OFFERED_TLS) argv[i++] = US"-MCT";
1986#endif
059ec3d9
PH
1987
1988 if (queue_run_pid != (pid_t)0)
1989 {
1990 argv[i++] = US"-MCQ";
1991 argv[i++] = string_sprintf("%d", queue_run_pid);
1992 argv[i++] = string_sprintf("%d", queue_run_pipe);
1993 }
1994
1995 argv[i++] = US"-MC";
55414b25
JH
1996 argv[i++] = US transport_name;
1997 argv[i++] = US hostname;
1998 argv[i++] = US hostaddress;
059ec3d9
PH
1999 argv[i++] = string_sprintf("%d", continue_sequence + 1);
2000 argv[i++] = id;
2001 argv[i++] = NULL;
2002
2003 /* Arrange for the channel to be on stdin. */
2004
2005 if (socket_fd != 0)
2006 {
f1e894f3
PH
2007 (void)dup2(socket_fd, 0);
2008 (void)close(socket_fd);
059ec3d9
PH
2009 }
2010
2011 DEBUG(D_exec) debug_print_argv(argv);
2012 exim_nullstd(); /* Ensure std{out,err} exist */
2013 execv(CS argv[0], (char *const *)argv);
2014
2015 DEBUG(D_any) debug_printf("execv failed: %s\n", strerror(errno));
2016 _exit(errno); /* Note: must be _exit(), NOT exit() */
2017 }
2018
2019/* If the process creation succeeded, wait for the first-level child, which
2020immediately exits, leaving the second level process entirely disconnected from
2021this one. */
2022
2023if (pid > 0)
2024 {
2025 int rc;
2026 while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
2027 DEBUG(D_transport) debug_printf("transport_pass_socket succeeded\n");
2028 return TRUE;
2029 }
2030else
2031 {
2032 DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
2033 strerror(errno));
2034 return FALSE;
2035 }
2036}
2037
2038
2039
2040/*************************************************
2041* Set up direct (non-shell) command *
2042*************************************************/
2043
2044/* This function is called when a command line is to be parsed and executed
2045directly, without the use of /bin/sh. It is called by the pipe transport,
2046the queryprogram router, and also from the main delivery code when setting up a
2047transport filter process. The code for ETRN also makes use of this; in that
2048case, no addresses are passed.
2049
2050Arguments:
2051 argvptr pointer to anchor for argv vector
55414b25 2052 cmd points to the command string (modified IN PLACE)
059ec3d9
PH
2053 expand_arguments true if expansion is to occur
2054 expand_failed error value to set if expansion fails; not relevant if
2055 addr == NULL
2056 addr chain of addresses, or NULL
2057 etext text for use in error messages
2058 errptr where to put error message if addr is NULL;
2059 otherwise it is put in the first address
2060
2061Returns: TRUE if all went well; otherwise an error will be
2062 set in the first address and FALSE returned
2063*/
2064
2065BOOL
55414b25
JH
2066transport_set_up_command(const uschar ***argvptr, uschar *cmd,
2067 BOOL expand_arguments, int expand_failed, address_item *addr,
2068 uschar *etext, uschar **errptr)
059ec3d9
PH
2069{
2070address_item *ad;
55414b25 2071const uschar **argv;
059ec3d9
PH
2072uschar *s, *ss;
2073int address_count = 0;
2074int argcount = 0;
2075int i, max_args;
2076
2077/* Get store in which to build an argument list. Count the number of addresses
2078supplied, and allow for that many arguments, plus an additional 60, which
2079should be enough for anybody. Multiple addresses happen only when the local
2080delivery batch option is set. */
2081
2082for (ad = addr; ad != NULL; ad = ad->next) address_count++;
2083max_args = address_count + 60;
2084*argvptr = argv = store_get((max_args+1)*sizeof(uschar *));
2085
2086/* Split the command up into arguments terminated by white space. Lose
2087trailing space at the start and end. Double-quoted arguments can contain \\ and
2088\" escapes and so can be handled by the standard function; single-quoted
2089arguments are verbatim. Copy each argument into a new string. */
2090
2091s = cmd;
2092while (isspace(*s)) s++;
2093
2094while (*s != 0 && argcount < max_args)
2095 {
2096 if (*s == '\'')
2097 {
2098 ss = s + 1;
2099 while (*ss != 0 && *ss != '\'') ss++;
2100 argv[argcount++] = ss = store_get(ss - s++);
2101 while (*s != 0 && *s != '\'') *ss++ = *s++;
2102 if (*s != 0) s++;
2103 *ss++ = 0;
2104 }
55414b25 2105 else argv[argcount++] = string_copy(string_dequote(CUSS &s));
059ec3d9
PH
2106 while (isspace(*s)) s++;
2107 }
2108
2109argv[argcount] = (uschar *)0;
2110
2111/* If *s != 0 we have run out of argument slots. */
2112
2113if (*s != 0)
2114 {
2115 uschar *msg = string_sprintf("Too many arguments in command \"%s\" in "
2116 "%s", cmd, etext);
2117 if (addr != NULL)
2118 {
2119 addr->transport_return = FAIL;
2120 addr->message = msg;
2121 }
2122 else *errptr = msg;
2123 return FALSE;
2124 }
2125
2126/* Expand each individual argument if required. Expansion happens for pipes set
2127up in filter files and with directly-supplied commands. It does not happen if
2128the pipe comes from a traditional .forward file. A failing expansion is a big
2129disaster if the command came from Exim's configuration; if it came from a user
2130it is just a normal failure. The expand_failed value is used as the error value
2131to cater for these two cases.
2132
2133An argument consisting just of the text "$pipe_addresses" is treated specially.
2134It is not passed to the general expansion function. Instead, it is replaced by
2135a number of arguments, one for each address. This avoids problems with shell
2136metacharacters and spaces in addresses.
2137
2138If the parent of the top address has an original part of "system-filter", this
2139pipe was set up by the system filter, and we can permit the expansion of
2140$recipients. */
2141
2142DEBUG(D_transport)
2143 {
2144 debug_printf("direct command:\n");
2145 for (i = 0; argv[i] != (uschar *)0; i++)
2146 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2147 }
2148
2149if (expand_arguments)
2150 {
2151 BOOL allow_dollar_recipients = addr != NULL &&
2152 addr->parent != NULL &&
2153 Ustrcmp(addr->parent->address, "system-filter") == 0;
2154
2155 for (i = 0; argv[i] != (uschar *)0; i++)
2156 {
2157
2158 /* Handle special fudge for passing an address list */
2159
2160 if (addr != NULL &&
2161 (Ustrcmp(argv[i], "$pipe_addresses") == 0 ||
2162 Ustrcmp(argv[i], "${pipe_addresses}") == 0))
2163 {
2164 int additional;
2165
2166 if (argcount + address_count - 1 > max_args)
2167 {
2168 addr->transport_return = FAIL;
2169 addr->message = string_sprintf("Too many arguments to command \"%s\" "
2170 "in %s", cmd, etext);
2171 return FALSE;
2172 }
2173
2174 additional = address_count - 1;
2175 if (additional > 0)
2176 memmove(argv + i + 1 + additional, argv + i + 1,
2177 (argcount - i)*sizeof(uschar *));
2178
09792322
NK
2179 for (ad = addr; ad != NULL; ad = ad->next) {
2180 argv[i++] = ad->address;
2181 argcount++;
2182 }
2183
700d22f3
PP
2184 /* Subtract one since we replace $pipe_addresses */
2185 argcount--;
2186 i--;
09792322
NK
2187 }
2188
2189 /* Handle special case of $address_pipe when af_force_command is set */
2190
2191 else if (addr != NULL && testflag(addr,af_force_command) &&
2192 (Ustrcmp(argv[i], "$address_pipe") == 0 ||
2193 Ustrcmp(argv[i], "${address_pipe}") == 0))
2194 {
2195 int address_pipe_i;
2196 int address_pipe_argcount = 0;
2197 int address_pipe_max_args;
2198 uschar **address_pipe_argv;
2199
2200 /* We can never have more then the argv we will be loading into */
2201 address_pipe_max_args = max_args - argcount + 1;
2202
2203 DEBUG(D_transport)
700d22f3 2204 debug_printf("address_pipe_max_args=%d\n", address_pipe_max_args);
09792322
NK
2205
2206 /* We allocate an additional for (uschar *)0 */
2207 address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *));
2208
2209 /* +1 because addr->local_part[0] == '|' since af_force_command is set */
2210 s = expand_string(addr->local_part + 1);
2211
700d22f3
PP
2212 if (s == NULL || *s == '\0')
2213 {
2214 addr->transport_return = FAIL;
2215 addr->message = string_sprintf("Expansion of \"%s\" "
2216 "from command \"%s\" in %s failed: %s",
2217 (addr->local_part + 1), cmd, etext, expand_string_message);
09792322 2218 return FALSE;
700d22f3 2219 }
09792322
NK
2220
2221 while (isspace(*s)) s++; /* strip leading space */
2222
2223 while (*s != 0 && address_pipe_argcount < address_pipe_max_args)
2224 {
2225 if (*s == '\'')
2226 {
2227 ss = s + 1;
2228 while (*ss != 0 && *ss != '\'') ss++;
2229 address_pipe_argv[address_pipe_argcount++] = ss = store_get(ss - s++);
2230 while (*s != 0 && *s != '\'') *ss++ = *s++;
2231 if (*s != 0) s++;
2232 *ss++ = 0;
2233 }
55414b25
JH
2234 else address_pipe_argv[address_pipe_argcount++] =
2235 string_copy(string_dequote(CUSS &s));
09792322
NK
2236 while (isspace(*s)) s++; /* strip space after arg */
2237 }
2238
2239 address_pipe_argv[address_pipe_argcount] = (uschar *)0;
2240
2241 /* If *s != 0 we have run out of argument slots. */
2242 if (*s != 0)
2243 {
2244 uschar *msg = string_sprintf("Too many arguments in $address_pipe "
2245 "\"%s\" in %s", addr->local_part + 1, etext);
2246 if (addr != NULL)
2247 {
2248 addr->transport_return = FAIL;
2249 addr->message = msg;
2250 }
2251 else *errptr = msg;
2252 return FALSE;
2253 }
2254
700d22f3
PP
2255 /* address_pipe_argcount - 1
2256 * because we are replacing $address_pipe in the argument list
2257 * with the first thing it expands to */
09792322
NK
2258 if (argcount + address_pipe_argcount - 1 > max_args)
2259 {
2260 addr->transport_return = FAIL;
2261 addr->message = string_sprintf("Too many arguments to command "
2262 "\"%s\" after expanding $address_pipe in %s", cmd, etext);
2263 return FALSE;
2264 }
2265
2266 /* If we are not just able to replace the slot that contained
2267 * $address_pipe (address_pipe_argcount == 1)
700d22f3 2268 * We have to move the existing argv by address_pipe_argcount - 1
09792322
NK
2269 * Visually if address_pipe_argcount == 2:
2270 * [argv 0][argv 1][argv 2($address_pipe)][argv 3][0]
700d22f3 2271 * [argv 0][argv 1][ap_arg0][ap_arg1][old argv 3][0]
09792322
NK
2272 */
2273 if (address_pipe_argcount > 1)
2274 memmove(
2275 /* current position + additonal args */
2276 argv + i + address_pipe_argcount,
2277 /* current position + 1 (for the (uschar *)0 at the end) */
2278 argv + i + 1,
2279 /* -1 for the (uschar *)0 at the end)*/
2280 (argcount - i)*sizeof(uschar *)
2281 );
2282
2283 /* Now we fill in the slots we just moved argv out of
2284 * [argv 0][argv 1][argv 2=pipeargv[0]][argv 3=pipeargv[1]][old argv 3][0]
2285 */
700d22f3
PP
2286 for (address_pipe_i = 0;
2287 address_pipe_argv[address_pipe_i] != (uschar *)0;
2288 address_pipe_i++)
2289 {
2290 argv[i++] = address_pipe_argv[address_pipe_i];
2291 argcount++;
09792322
NK
2292 }
2293
700d22f3
PP
2294 /* Subtract one since we replace $address_pipe */
2295 argcount--;
2296 i--;
059ec3d9
PH
2297 }
2298
2299 /* Handle normal expansion string */
2300
2301 else
2302 {
55414b25 2303 const uschar *expanded_arg;
059ec3d9 2304 enable_dollar_recipients = allow_dollar_recipients;
55414b25 2305 expanded_arg = expand_cstring(argv[i]);
059ec3d9
PH
2306 enable_dollar_recipients = FALSE;
2307
2308 if (expanded_arg == NULL)
2309 {
2310 uschar *msg = string_sprintf("Expansion of \"%s\" "
2311 "from command \"%s\" in %s failed: %s",
2312 argv[i], cmd, etext, expand_string_message);
2313 if (addr != NULL)
2314 {
2315 addr->transport_return = expand_failed;
2316 addr->message = msg;
2317 }
2318 else *errptr = msg;
2319 return FALSE;
2320 }
2321 argv[i] = expanded_arg;
2322 }
2323 }
2324
2325 DEBUG(D_transport)
2326 {
2327 debug_printf("direct command after expansion:\n");
2328 for (i = 0; argv[i] != (uschar *)0; i++)
2329 debug_printf(" argv[%d] = %s\n", i, string_printing(argv[i]));
2330 }
2331 }
2332
2333return TRUE;
2334}
2335
511a6c14
JH
2336/* vi: aw ai sw=2
2337*/
059ec3d9 2338/* End of transport.c */