Follow CNAME chains only one step. Bug 2264
[exim.git] / src / src / verify.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* Functions concerned with verifying things. The original code for callout
9caching was contributed by Kevin Fleming (but I hacked it around a bit). */
10
11
12#include "exim.h"
817d9f57 13#include "transports/smtp.h"
059ec3d9 14
e4bdf652
JH
15#define CUTTHROUGH_CMD_TIMEOUT 30 /* timeout for cutthrough-routing calls */
16#define CUTTHROUGH_DATA_TIMEOUT 60 /* timeout for cutthrough-routing calls */
817d9f57
JH
17static smtp_outblock ctblock;
18uschar ctbuffer[8192];
19
059ec3d9
PH
20
21/* Structure for caching DNSBL lookups */
22
23typedef struct dnsbl_cache_block {
14b3c5bc 24 time_t expiry;
059ec3d9
PH
25 dns_address *rhs;
26 uschar *text;
27 int rc;
28 BOOL text_set;
29} dnsbl_cache_block;
30
31
32/* Anchor for DNSBL cache */
33
34static tree_node *dnsbl_cache = NULL;
35
36
431b7361
PH
37/* Bits for match_type in one_check_dnsbl() */
38
39#define MT_NOT 1
40#define MT_ALL 2
41
57cc2785 42static uschar cutthrough_response(int, char, uschar **, int);
431b7361 43
8d330698 44
059ec3d9
PH
45
46/*************************************************
47* Retrieve a callout cache record *
48*************************************************/
49
50/* If a record exists, check whether it has expired.
51
52Arguments:
53 dbm_file an open hints file
54 key the record key
55 type "address" or "domain"
56 positive_expire expire time for positive records
57 negative_expire expire time for negative records
58
59Returns: the cache record if a non-expired one exists, else NULL
60*/
61
62static dbdata_callout_cache *
55414b25 63get_callout_cache_record(open_db *dbm_file, const uschar *key, uschar *type,
059ec3d9
PH
64 int positive_expire, int negative_expire)
65{
66BOOL negative;
67int length, expire;
68time_t now;
69dbdata_callout_cache *cache_record;
70
b6323c75 71if (!(cache_record = dbfn_read_with_length(dbm_file, key, &length)))
059ec3d9 72 {
6f4d5ad3 73 HDEBUG(D_verify) debug_printf("callout cache: no %s record found for %s\n", type, key);
059ec3d9
PH
74 return NULL;
75 }
76
77/* We treat a record as "negative" if its result field is not positive, or if
78it is a domain record and the postmaster field is negative. */
79
80negative = cache_record->result != ccache_accept ||
81 (type[0] == 'd' && cache_record->postmaster_result == ccache_reject);
82expire = negative? negative_expire : positive_expire;
83now = time(NULL);
84
85if (now - cache_record->time_stamp > expire)
86 {
6f4d5ad3 87 HDEBUG(D_verify) debug_printf("callout cache: %s record expired for %s\n", type, key);
059ec3d9
PH
88 return NULL;
89 }
90
91/* If this is a non-reject domain record, check for the obsolete format version
92that doesn't have the postmaster and random timestamps, by looking at the
93length. If so, copy it to a new-style block, replicating the record's
94timestamp. Then check the additional timestamps. (There's no point wasting
95effort if connections are rejected.) */
96
97if (type[0] == 'd' && cache_record->result != ccache_reject)
98 {
99 if (length == sizeof(dbdata_callout_cache_obs))
100 {
101 dbdata_callout_cache *new = store_get(sizeof(dbdata_callout_cache));
102 memcpy(new, cache_record, length);
103 new->postmaster_stamp = new->random_stamp = new->time_stamp;
104 cache_record = new;
105 }
106
107 if (now - cache_record->postmaster_stamp > expire)
108 cache_record->postmaster_result = ccache_unknown;
109
110 if (now - cache_record->random_stamp > expire)
111 cache_record->random_result = ccache_unknown;
112 }
113
6f4d5ad3 114HDEBUG(D_verify) debug_printf("callout cache: found %s record for %s\n", type, key);
059ec3d9
PH
115return cache_record;
116}
117
118
119
707ee5b1
JH
120/* Check the callout cache.
121Options * pm_mailfrom may be modified by cache partial results.
059ec3d9 122
707ee5b1 123Return: TRUE if result found
059ec3d9
PH
124*/
125
707ee5b1
JH
126static BOOL
127cached_callout_lookup(address_item * addr, uschar * address_key,
128 uschar * from_address, int * opt_ptr, uschar ** pm_ptr,
129 int * yield, uschar ** failure_ptr,
130 dbdata_callout_cache * new_domain_record, int * old_domain_res)
059ec3d9 131{
707ee5b1 132int options = *opt_ptr;
059ec3d9
PH
133open_db dbblock;
134open_db *dbm_file = NULL;
059ec3d9
PH
135
136/* Open the callout cache database, it it exists, for reading only at this
137stage, unless caching has been disabled. */
138
8b9476ba 139if (options & vopt_callout_no_cache)
059ec3d9
PH
140 {
141 HDEBUG(D_verify) debug_printf("callout cache: disabled by no_cache\n");
142 }
707ee5b1 143else if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE)))
059ec3d9
PH
144 {
145 HDEBUG(D_verify) debug_printf("callout cache: not available\n");
146 }
707ee5b1 147else
059ec3d9 148 {
707ee5b1
JH
149 /* If a cache database is available see if we can avoid the need to do an
150 actual callout by making use of previously-obtained data. */
151
152 dbdata_callout_cache_address * cache_address_record;
153 dbdata_callout_cache * cache_record = get_callout_cache_record(dbm_file,
154 addr->domain, US"domain",
155 callout_cache_domain_positive_expire, callout_cache_domain_negative_expire);
059ec3d9
PH
156
157 /* If an unexpired cache record was found for this domain, see if the callout
158 process can be short-circuited. */
159
ff5929e3 160 if (cache_record)
059ec3d9 161 {
2b1c6e3a
PH
162 /* In most cases, if an early command (up to and including MAIL FROM:<>)
163 was rejected, there is no point carrying on. The callout fails. However, if
164 we are doing a recipient verification with use_sender or use_postmaster
165 set, a previous failure of MAIL FROM:<> doesn't count, because this time we
166 will be using a non-empty sender. We have to remember this situation so as
167 not to disturb the cached domain value if this whole verification succeeds
168 (we don't want it turning into "accept"). */
169
707ee5b1 170 *old_domain_res = cache_record->result;
2b1c6e3a 171
707ee5b1
JH
172 if ( cache_record->result == ccache_reject
173 || *from_address == 0 && cache_record->result == ccache_reject_mfnull)
059ec3d9
PH
174 {
175 setflag(addr, af_verify_nsfail);
176 HDEBUG(D_verify)
707ee5b1
JH
177 debug_printf("callout cache: domain gave initial rejection, or "
178 "does not accept HELO or MAIL FROM:<>\n");
059ec3d9
PH
179 setflag(addr, af_verify_nsfail);
180 addr->user_message = US"(result of an earlier callout reused).";
707ee5b1 181 *yield = FAIL;
8e669ac1 182 *failure_ptr = US"mail";
707ee5b1
JH
183 dbfn_close(dbm_file);
184 return TRUE;
059ec3d9
PH
185 }
186
187 /* If a previous check on a "random" local part was accepted, we assume
188 that the server does not do any checking on local parts. There is therefore
189 no point in doing the callout, because it will always be successful. If a
190 random check previously failed, arrange not to do it again, but preserve
191 the data in the new record. If a random check is required but hasn't been
192 done, skip the remaining cache processing. */
193
8b9476ba 194 if (options & vopt_callout_random) switch(cache_record->random_result)
059ec3d9
PH
195 {
196 case ccache_accept:
8b9476ba
JH
197 HDEBUG(D_verify)
198 debug_printf("callout cache: domain accepts random addresses\n");
2ddb4094 199 *failure_ptr = US"random";
707ee5b1
JH
200 dbfn_close(dbm_file);
201 return TRUE; /* Default yield is OK */
059ec3d9
PH
202
203 case ccache_reject:
8b9476ba
JH
204 HDEBUG(D_verify)
205 debug_printf("callout cache: domain rejects random addresses\n");
707ee5b1
JH
206 *opt_ptr = options & ~vopt_callout_random;
207 new_domain_record->random_result = ccache_reject;
208 new_domain_record->random_stamp = cache_record->random_stamp;
8b9476ba 209 break;
059ec3d9
PH
210
211 default:
8b9476ba
JH
212 HDEBUG(D_verify)
213 debug_printf("callout cache: need to check random address handling "
214 "(not cached or cache expired)\n");
707ee5b1
JH
215 dbfn_close(dbm_file);
216 return FALSE;
059ec3d9
PH
217 }
218
219 /* If a postmaster check is requested, but there was a previous failure,
220 there is again no point in carrying on. If a postmaster check is required,
221 but has not been done before, we are going to have to do a callout, so skip
222 remaining cache processing. */
223
707ee5b1 224 if (*pm_ptr)
059ec3d9
PH
225 {
226 if (cache_record->postmaster_result == ccache_reject)
707ee5b1
JH
227 {
228 setflag(addr, af_verify_pmfail);
229 HDEBUG(D_verify)
230 debug_printf("callout cache: domain does not accept "
231 "RCPT TO:<postmaster@domain>\n");
232 *yield = FAIL;
233 *failure_ptr = US"postmaster";
234 setflag(addr, af_verify_pmfail);
235 addr->user_message = US"(result of earlier verification reused).";
236 dbfn_close(dbm_file);
237 return TRUE;
238 }
059ec3d9 239 if (cache_record->postmaster_result == ccache_unknown)
707ee5b1
JH
240 {
241 HDEBUG(D_verify)
242 debug_printf("callout cache: need to check RCPT "
243 "TO:<postmaster@domain> (not cached or cache expired)\n");
244 dbfn_close(dbm_file);
245 return FALSE;
246 }
059ec3d9
PH
247
248 /* If cache says OK, set pm_mailfrom NULL to prevent a redundant
249 postmaster check if the address itself has to be checked. Also ensure
250 that the value in the cache record is preserved (with its old timestamp).
251 */
252
253 HDEBUG(D_verify) debug_printf("callout cache: domain accepts RCPT "
707ee5b1
JH
254 "TO:<postmaster@domain>\n");
255 *pm_ptr = NULL;
256 new_domain_record->postmaster_result = ccache_accept;
257 new_domain_record->postmaster_stamp = cache_record->postmaster_stamp;
059ec3d9
PH
258 }
259 }
260
261 /* We can't give a result based on information about the domain. See if there
262 is an unexpired cache record for this specific address (combined with the
263 sender address if we are doing a recipient callout with a non-empty sender).
264 */
265
707ee5b1
JH
266 if (!(cache_address_record = (dbdata_callout_cache_address *)
267 get_callout_cache_record(dbm_file, address_key, US"address",
268 callout_cache_positive_expire, callout_cache_negative_expire)))
269 {
270 dbfn_close(dbm_file);
271 return FALSE;
272 }
059ec3d9 273
707ee5b1 274 if (cache_address_record->result == ccache_accept)
059ec3d9 275 {
707ee5b1
JH
276 HDEBUG(D_verify)
277 debug_printf("callout cache: address record is positive\n");
278 }
279 else
280 {
281 HDEBUG(D_verify)
282 debug_printf("callout cache: address record is negative\n");
283 addr->user_message = US"Previous (cached) callout verification failure";
284 *failure_ptr = US"recipient";
285 *yield = FAIL;
059ec3d9
PH
286 }
287
288 /* Close the cache database while we actually do the callout for real. */
289
059ec3d9 290 dbfn_close(dbm_file);
707ee5b1
JH
291 return TRUE;
292 }
293return FALSE;
294}
295
296
297/* Write results to callout cache
298*/
299static void
300cache_callout_write(dbdata_callout_cache * dom_rec, const uschar * domain,
301 int done, dbdata_callout_cache_address * addr_rec, uschar * address_key)
302{
303open_db dbblock;
304open_db *dbm_file = NULL;
305
306/* If we get here with done == TRUE, a successful callout happened, and yield
307will be set OK or FAIL according to the response to the RCPT command.
308Otherwise, we looped through the hosts but couldn't complete the business.
309However, there may be domain-specific information to cache in both cases.
310
311The value of the result field in the new_domain record is ccache_unknown if
312there was an error before or with MAIL FROM:, and errno was not zero,
313implying some kind of I/O error. We don't want to write the cache in that case.
314Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */
315
316if (dom_rec->result != ccache_unknown)
317 if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE)))
318 {
319 HDEBUG(D_verify) debug_printf("callout cache: not available\n");
320 }
321 else
322 {
323 (void)dbfn_write(dbm_file, domain, dom_rec,
324 (int)sizeof(dbdata_callout_cache));
325 HDEBUG(D_verify) debug_printf("wrote callout cache domain record for %s:\n"
326 " result=%d postmaster=%d random=%d\n",
327 domain,
328 dom_rec->result,
329 dom_rec->postmaster_result,
330 dom_rec->random_result);
331 }
332
333/* If a definite result was obtained for the callout, cache it unless caching
334is disabled. */
335
336if (done && addr_rec->result != ccache_unknown)
337 {
338 if (!dbm_file)
339 dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE);
340 if (!dbm_file)
341 {
342 HDEBUG(D_verify) debug_printf("no callout cache available\n");
343 }
344 else
345 {
346 (void)dbfn_write(dbm_file, address_key, addr_rec,
347 (int)sizeof(dbdata_callout_cache_address));
348 HDEBUG(D_verify) debug_printf("wrote %s callout cache address record for %s\n",
349 addr_rec->result == ccache_accept ? "positive" : "negative",
350 address_key);
351 }
352 }
353
354if (dbm_file) dbfn_close(dbm_file);
355}
356
357
c4c940fd
JH
358/* Cutthrough-multi. If the existing cached cutthrough connection matches
359the one we would make for a subsequent recipient, use it. Send the RCPT TO
360and check the result, nonpipelined as it may be wanted immediately for
361recipient-verification.
362
363It seems simpler to deal with this case separately from the main callout loop.
364We will need to remember it has sent, or not, so that rcpt-acl tail code
365can do it there for the non-rcpt-verify case. For this we keep an addresscount.
366
367Return: TRUE for a definitive result for the recipient
368*/
369static int
370cutthrough_multi(address_item * addr, host_item * host_list,
371 transport_feedback * tf, int * yield)
372{
373BOOL done = FALSE;
374host_item * host;
375
376if (addr->transport == cutthrough.addr.transport)
377 for (host = host_list; host; host = host->next)
378 if (Ustrcmp(host->address, cutthrough.host.address) == 0)
379 {
380 int host_af;
381 uschar *interface = NULL; /* Outgoing interface to use; NULL => any */
382 int port = 25;
383
384 deliver_host = host->name;
385 deliver_host_address = host->address;
386 deliver_host_port = host->port;
387 deliver_domain = addr->domain;
388 transport_name = addr->transport->name;
389
57cc2785 390 host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
c4c940fd 391
f83a760f
JH
392 if ( !smtp_get_interface(tf->interface, host_af, addr, &interface,
393 US"callout")
394 || !smtp_get_port(tf->port, addr, &port, US"callout")
395 )
c4c940fd
JH
396 log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
397 addr->message);
398
f83a760f
JH
399 smtp_port_for_connect(host, port);
400
c4c940fd
JH
401 if ( ( interface == cutthrough.interface
402 || ( interface
403 && cutthrough.interface
404 && Ustrcmp(interface, cutthrough.interface) == 0
405 ) )
f83a760f 406 && host->port == cutthrough.host.port
c4c940fd
JH
407 )
408 {
409 uschar * resp = NULL;
410
411 /* Match! Send the RCPT TO, set done from the response */
412 done =
4e910c01 413 smtp_write_command(&ctblock, SCMD_FLUSH, "RCPT TO:<%.1000s>\r\n",
c4c940fd
JH
414 transport_rcpt_address(addr,
415 addr->transport->rcpt_include_affixes)) >= 0 &&
57cc2785 416 cutthrough_response(cutthrough.fd, '2', &resp, CUTTHROUGH_DATA_TIMEOUT) == '2';
c4c940fd
JH
417
418 /* This would go horribly wrong if a callout fail was ignored by ACL.
419 We punt by abandoning cutthrough on a reject, like the
420 first-rcpt does. */
421
422 if (done)
423 {
424 address_item * na = store_get(sizeof(address_item));
425 *na = cutthrough.addr;
426 cutthrough.addr = *addr;
427 cutthrough.addr.host_used = &cutthrough.host;
428 cutthrough.addr.next = na;
429
430 cutthrough.nrcpt++;
431 }
432 else
433 {
57cc2785 434 cancel_cutthrough_connection(TRUE, US"recipient rejected");
c4c940fd
JH
435 if (!resp || errno == ETIMEDOUT)
436 {
437 HDEBUG(D_verify) debug_printf("SMTP timeout\n");
438 }
439 else if (errno == 0)
440 {
441 if (*resp == 0)
442 Ustrcpy(resp, US"connection dropped");
443
444 addr->message =
e9166683
JH
445 string_sprintf("response to \"%s\" was: %s",
446 big_buffer, string_printing(resp));
c4c940fd
JH
447
448 addr->user_message =
449 string_sprintf("Callout verification failed:\n%s", resp);
450
451 /* Hard rejection ends the process */
452
453 if (resp[0] == '5') /* Address rejected */
454 {
455 *yield = FAIL;
456 done = TRUE;
457 }
458 }
459 }
460 }
461 break; /* host_list */
462 }
463if (!done)
57cc2785 464 cancel_cutthrough_connection(TRUE, US"incompatible connection");
c4c940fd
JH
465return done;
466}
467
468
707ee5b1
JH
469/*************************************************
470* Do callout verification for an address *
471*************************************************/
472
473/* This function is called from verify_address() when the address has routed to
474a host list, and a callout has been requested. Callouts are expensive; that is
475why a cache is used to improve the efficiency.
476
477Arguments:
478 addr the address that's been routed
479 host_list the list of hosts to try
480 tf the transport feedback block
481
482 ifstring "interface" option from transport, or NULL
483 portstring "port" option from transport, or NULL
484 protocolstring "protocol" option from transport, or NULL
485 callout the per-command callout timeout
486 callout_overall the overall callout timeout (if < 0 use 4*callout)
487 callout_connect the callout connection timeout (if < 0 use callout)
488 options the verification options - these bits are used:
489 vopt_is_recipient => this is a recipient address
490 vopt_callout_no_cache => don't use callout cache
491 vopt_callout_fullpm => if postmaster check, do full one
492 vopt_callout_random => do the "random" thing
493 vopt_callout_recipsender => use real sender for recipient
494 vopt_callout_recippmaster => use postmaster for recipient
57cc2785 495 vopt_callout_hold => lazy close connection
707ee5b1
JH
496 se_mailfrom MAIL FROM address for sender verify; NULL => ""
497 pm_mailfrom if non-NULL, do the postmaster check with this sender
498
499Returns: OK/FAIL/DEFER
500*/
501
502static int
503do_callout(address_item *addr, host_item *host_list, transport_feedback *tf,
504 int callout, int callout_overall, int callout_connect, int options,
505 uschar *se_mailfrom, uschar *pm_mailfrom)
506{
507int yield = OK;
508int old_domain_cache_result = ccache_accept;
509BOOL done = FALSE;
510uschar *address_key;
511uschar *from_address;
512uschar *random_local_part = NULL;
513const uschar *save_deliver_domain = deliver_domain;
514uschar **failure_ptr = options & vopt_is_recipient
515 ? &recipient_verify_failure : &sender_verify_failure;
516dbdata_callout_cache new_domain_record;
517dbdata_callout_cache_address new_address_record;
707ee5b1
JH
518time_t callout_start_time;
519
520new_domain_record.result = ccache_unknown;
521new_domain_record.postmaster_result = ccache_unknown;
522new_domain_record.random_result = ccache_unknown;
523
524memset(&new_address_record, 0, sizeof(new_address_record));
525
526/* For a recipient callout, the key used for the address cache record must
527include the sender address if we are using the real sender in the callout,
528because that may influence the result of the callout. */
529
530if (options & vopt_is_recipient)
531 if (options & vopt_callout_recipsender)
532 {
533 from_address = sender_address;
534 address_key = string_sprintf("%s/<%s>", addr->address, sender_address);
535 if (cutthrough.delivery) options |= vopt_callout_no_cache;
536 }
537 else if (options & vopt_callout_recippmaster)
538 {
539 from_address = string_sprintf("postmaster@%s", qualify_domain_sender);
540 address_key = string_sprintf("%s/<postmaster@%s>", addr->address,
541 qualify_domain_sender);
542 }
543 else
544 {
545 from_address = US"";
546 address_key = addr->address;
547 }
548
549/* For a sender callout, we must adjust the key if the mailfrom address is not
550empty. */
551
552else
553 {
554 from_address = se_mailfrom ? se_mailfrom : US"";
555 address_key = *from_address
556 ? string_sprintf("%s/<%s>", addr->address, from_address) : addr->address;
059ec3d9
PH
557 }
558
707ee5b1
JH
559if (cached_callout_lookup(addr, address_key, from_address,
560 &options, &pm_mailfrom, &yield, failure_ptr,
561 &new_domain_record, &old_domain_cache_result))
57cc2785
JH
562 {
563 cancel_cutthrough_connection(TRUE, US"cache-hit");
707ee5b1 564 goto END_CALLOUT;
57cc2785 565 }
707ee5b1 566
193e3acd 567if (!addr->transport)
059ec3d9 568 {
193e3acd 569 HDEBUG(D_verify) debug_printf("cannot callout via null transport\n");
059ec3d9 570 }
6681531a
HSHR
571else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0)
572 log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp",
573 addr->transport->name, addr->transport->driver_name);
193e3acd
JH
574else
575 {
576 smtp_transport_options_block *ob =
9d9c3746 577 (smtp_transport_options_block *)addr->transport->options_block;
c4c940fd 578 host_item * host;
059ec3d9 579
193e3acd
JH
580 /* The information wasn't available in the cache, so we have to do a real
581 callout and save the result in the cache for next time, unless no_cache is set,
582 or unless we have a previously cached negative random result. If we are to test
583 with a random local part, ensure that such a local part is available. If not,
4c04137d 584 log the fact, but carry on without randomising. */
059ec3d9 585
707ee5b1 586 if (options & vopt_callout_random && callout_random_local_part)
65f1c92a 587 if (!(random_local_part = expand_string(callout_random_local_part)))
193e3acd
JH
588 log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand "
589 "callout_random_local_part: %s", expand_string_message);
059ec3d9 590
193e3acd
JH
591 /* Default the connect and overall callout timeouts if not set, and record the
592 time we are starting so that we can enforce it. */
4c590bd1 593
193e3acd
JH
594 if (callout_overall < 0) callout_overall = 4 * callout;
595 if (callout_connect < 0) callout_connect = callout;
596 callout_start_time = time(NULL);
4c590bd1 597
193e3acd
JH
598 /* Before doing a real callout, if this is an SMTP connection, flush the SMTP
599 output because a callout might take some time. When PIPELINING is active and
600 there are many recipients, the total time for doing lots of callouts can add up
601 and cause the client to time out. So in this case we forgo the PIPELINING
602 optimization. */
817d9f57 603
ff5929e3 604 if (smtp_out && !disable_callout_flush) mac_smtp_fflush();
059ec3d9 605
c4c940fd
JH
606 clearflag(addr, af_verify_pmfail); /* postmaster callout flag */
607 clearflag(addr, af_verify_nsfail); /* null sender callout flag */
608
5032d1cf
JH
609/* cutthrough-multi: if a nonfirst rcpt has the same routing as the first,
610and we are holding a cutthrough conn open, we can just append the rcpt to
611that conn for verification purposes (and later delivery also). Simplest
c4c940fd 612coding means skipping this whole loop and doing the append separately. */
5032d1cf
JH
613
614 /* Can we re-use an open cutthrough connection? */
615 if ( cutthrough.fd >= 0
616 && (options & (vopt_callout_recipsender | vopt_callout_recippmaster))
617 == vopt_callout_recipsender
618 && !random_local_part
619 && !pm_mailfrom
620 )
c4c940fd 621 done = cutthrough_multi(addr, host_list, tf, &yield);
5032d1cf 622
c4c940fd
JH
623 /* If we did not use a cached connection, make connections to the hosts
624 and do real callouts. The list of hosts is passed in as an argument. */
059ec3d9 625
ff5929e3 626 for (host = host_list; host && !done; host = host->next)
059ec3d9 627 {
193e3acd
JH
628 int host_af;
629 int port = 25;
193e3acd 630 uschar *interface = NULL; /* Outgoing interface to use; NULL => any */
02b41d71 631 smtp_context sx;
193e3acd 632
ff5929e3 633 if (!host->address)
193e3acd
JH
634 {
635 DEBUG(D_verify) debug_printf("no IP address for host name %s: skipping\n",
636 host->name);
637 continue;
638 }
059ec3d9 639
193e3acd 640 /* Check the overall callout timeout */
059ec3d9 641
193e3acd
JH
642 if (time(NULL) - callout_start_time >= callout_overall)
643 {
644 HDEBUG(D_verify) debug_printf("overall timeout for callout exceeded\n");
645 break;
646 }
059ec3d9 647
193e3acd 648 /* Set IPv4 or IPv6 */
059ec3d9 649
e9166683 650 host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
de3a88fb 651
193e3acd
JH
652 /* Expand and interpret the interface and port strings. The latter will not
653 be used if there is a host-specific port (e.g. from a manualroute router).
654 This has to be delayed till now, because they may expand differently for
655 different hosts. If there's a failure, log it, but carry on with the
656 defaults. */
de3a88fb 657
193e3acd
JH
658 deliver_host = host->name;
659 deliver_host_address = host->address;
a7538db1 660 deliver_host_port = host->port;
193e3acd 661 deliver_domain = addr->domain;
aec45841 662 transport_name = addr->transport->name;
059ec3d9 663
6f6dedcc 664 if ( !smtp_get_interface(tf->interface, host_af, addr, &interface,
bf7aabb4
JH
665 US"callout")
666 || !smtp_get_port(tf->port, addr, &port, US"callout")
667 )
193e3acd
JH
668 log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
669 addr->message);
059ec3d9 670
02b41d71
JH
671 sx.addrlist = addr;
672 sx.host = host;
673 sx.host_af = host_af,
674 sx.port = port;
675 sx.interface = interface;
676 sx.helo_data = tf->helo_data;
677 sx.tblock = addr->transport;
e9166683 678 sx.verify = TRUE;
02b41d71
JH
679
680tls_retry_connection:
681 /* Set the address state so that errors are recorded in it */
682
683 addr->transport_return = PENDING_DEFER;
684 ob->connect_timeout = callout_connect;
685 ob->command_timeout = callout;
686
687 /* Get the channel set up ready for a message (MAIL FROM being the next
688 SMTP command to send. If we tried TLS but it failed, try again without
689 if permitted */
690
f10e3ea3
JH
691 yield = smtp_setup_conn(&sx, FALSE);
692#ifdef SUPPORT_TLS
693 if ( yield == DEFER
02b41d71
JH
694 && addr->basic_errno == ERRNO_TLSFAILURE
695 && ob->tls_tempfail_tryclear
696 && verify_check_given_host(&ob->hosts_require_tls, host) != OK
697 )
193e3acd 698 {
cf0c6164
JH
699 log_write(0, LOG_MAIN,
700 "%s: callout unencrypted to %s [%s] (not in hosts_require_tls)",
701 addr->message, host->name, host->address);
e9166683
JH
702 addr->transport_return = PENDING_DEFER;
703 yield = smtp_setup_conn(&sx, TRUE);
02b41d71 704 }
f10e3ea3 705#endif
02b41d71
JH
706 if (yield != OK)
707 {
02b41d71 708 errno = addr->basic_errno;
aec45841 709 transport_name = NULL;
193e3acd
JH
710 deliver_host = deliver_host_address = NULL;
711 deliver_domain = save_deliver_domain;
bf7aabb4 712
02b41d71
JH
713 /* Failure to accept HELO is cached; this blocks the whole domain for all
714 senders. I/O errors and defer responses are not cached. */
770747fd 715
02b41d71 716 if (yield == FAIL && (errno == 0 || errno == ERRNO_SMTPCLOSED))
a7538db1 717 {
02b41d71
JH
718 setflag(addr, af_verify_nsfail);
719 new_domain_record.result = ccache_reject;
720 done = TRUE;
a7538db1 721 }
193e3acd 722 else
02b41d71
JH
723 done = FALSE;
724 goto no_conn;
3c8b3577 725 }
9bfc60eb 726
02b41d71
JH
727 /* If we needed to authenticate, smtp_setup_conn() did that. Copy
728 the AUTH info for logging */
fcc8e047 729
02b41d71
JH
730 addr->authenticator = client_authenticator;
731 addr->auth_id = client_authenticated_id;
b4a2b536 732
e9166683
JH
733 sx.from_addr = from_address;
734 sx.first_addr = sx.sync_addr = addr;
735 sx.ok = FALSE; /*XXX these 3 last might not be needed for verify? */
736 sx.send_rset = TRUE;
737 sx.completed_addr = FALSE;
b4a2b536 738
cf0c6164 739 new_domain_record.result = old_domain_cache_result == ccache_reject_mfnull
e9166683 740 ? ccache_reject_mfnull : ccache_accept;
c4c940fd 741
e9166683
JH
742 /* Do the random local part check first. Temporarily replace the recipient
743 with the "random" value */
02b41d71 744
e9166683 745 if (random_local_part)
059ec3d9 746 {
e9166683 747 uschar * main_address = addr->address;
921dfc11
JH
748 const uschar * rcpt_domain = addr->domain;
749
8c5d388a 750#ifdef SUPPORT_I18N
921dfc11
JH
751 uschar * errstr = NULL;
752 if ( testflag(addr, af_utf8_downcvt)
753 && (rcpt_domain = string_domain_utf8_to_alabel(rcpt_domain,
754 &errstr), errstr)
755 )
756 {
757 addr->message = errstr;
758 errno = ERRNO_EXPANDFAIL;
759 setflag(addr, af_verify_nsfail);
760 done = FALSE;
761 rcpt_domain = US""; /*XXX errorhandling! */
762 }
763#endif
764
57cc2785
JH
765 /* This would be ok for 1st rcpt of a cutthrough (the case handled here;
766 subsequents are done in cutthrough_multi()), but no way to
767 handle a subsequent because of the RSET vaporising the MAIL FROM.
768 So refuse to support any. Most cutthrough use will not involve
769 random_local_part, so no loss. */
770 cancel_cutthrough_connection(TRUE, US"random-recipient");
059ec3d9 771
e9166683
JH
772 addr->address = string_sprintf("%s@%.1000s",
773 random_local_part, rcpt_domain);
774 done = FALSE;
902fbd69
JH
775
776 /* If accepted, we aren't going to do any further tests below.
777 Otherwise, cache a real negative response, and get back to the right
778 state to send RCPT. Unless there's some problem such as a dropped
779 connection, we expect to succeed, because the commands succeeded above.
780 However, some servers drop the connection after responding to an
781 invalid recipient, so on (any) error we drop and remake the connection.
782 XXX We don't care about that for postmaster_full. Should we?
783
784 XXX could we add another flag to the context, and have the common
785 code emit the RSET too? Even pipelined after the RCPT...
786 Then the main-verify call could use it if there's to be a subsequent
787 postmaster-verify.
788 The sync_responses() would need to be taught about it and we'd
789 need another return code filtering out to here.
14de8063 790
d6e7df90 791 Avoid using a SIZE option on the MAIL for all random-rcpt checks.
902fbd69
JH
792 */
793
14de8063
JH
794 sx.avoid_option = OPTION_SIZE;
795
a65c4156
JH
796 /* Remember when we last did a random test */
797 new_domain_record.random_stamp = time(NULL);
798
e9166683
JH
799 if (smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0)
800 switch(addr->transport_return)
801 {
b6323c75 802 case PENDING_OK: /* random was accepted, unfortunately */
e9166683 803 new_domain_record.random_result = ccache_accept;
b6323c75 804 yield = OK; /* Only usable verify result we can return */
fae8970d 805 done = TRUE;
2ddb4094 806 *failure_ptr = US"random";
fae8970d 807 goto no_conn;
b6323c75 808 case FAIL: /* rejected: the preferred result */
e9166683 809 new_domain_record.random_result = ccache_reject;
14de8063 810 sx.avoid_option = 0;
193e3acd 811
e9166683 812 /* Between each check, issue RSET, because some servers accept only
902fbd69
JH
813 one recipient after MAIL FROM:<>.
814 XXX We don't care about that for postmaster_full. Should we? */
193e3acd 815
e9166683 816 if ((done =
4e910c01 817 smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0 &&
e9166683
JH
818 smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
819 '2', callout)))
820 break;
65f1c92a 821
65f1c92a 822 HDEBUG(D_acl|D_v)
e1d04f48 823 debug_printf_indent("problem after random/rset/mfrom; reopen conn\n");
65f1c92a
JH
824 random_local_part = NULL;
825#ifdef SUPPORT_TLS
dec766a1 826 tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
65f1c92a 827#endif
e1d04f48 828 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
02b41d71 829 (void)close(sx.inblock.sock);
e9166683 830 sx.inblock.sock = sx.outblock.sock = -1;
0cbf2b82 831#ifndef DISABLE_EVENT
65f1c92a
JH
832 (void) event_raise(addr->transport->event_action,
833 US"tcp:close", NULL);
834#endif
a65c4156
JH
835 addr->address = main_address;
836 addr->transport_return = PENDING_DEFER;
837 sx.first_addr = sx.sync_addr = addr;
838 sx.ok = FALSE;
839 sx.send_rset = TRUE;
840 sx.completed_addr = FALSE;
65f1c92a 841 goto tls_retry_connection;
b6323c75
JH
842 case DEFER: /* 4xx response to random */
843 break; /* Just to be clear. ccache_unknown, !done. */
e9166683 844 }
059ec3d9 845
e9166683
JH
846 /* Re-setup for main verify, or for the error message when failing */
847 addr->address = main_address;
848 addr->transport_return = PENDING_DEFER;
849 sx.first_addr = sx.sync_addr = addr;
850 sx.ok = FALSE;
851 sx.send_rset = TRUE;
852 sx.completed_addr = FALSE;
853 }
854 else
855 done = TRUE;
921dfc11 856
14de8063
JH
857 /* Main verify. For rcpt-verify use SIZE if we know it and we're not cacheing;
858 for sndr-verify never use it. */
e9166683
JH
859
860 if (done)
861 {
14de8063
JH
862 if (!(options & vopt_is_recipient && options & vopt_callout_no_cache))
863 sx.avoid_option = OPTION_SIZE;
864
e9166683
JH
865 done = FALSE;
866 switch(smtp_write_mail_and_rcpt_cmds(&sx, &yield))
867 {
868 case 0: switch(addr->transport_return) /* ok so far */
869 {
870 case PENDING_OK: done = TRUE;
871 new_address_record.result = ccache_accept;
872 break;
14de8063 873 case FAIL: done = TRUE;
e9166683
JH
874 yield = FAIL;
875 *failure_ptr = US"recipient";
876 new_address_record.result = ccache_reject;
877 break;
14de8063 878 default: break;
e9166683
JH
879 }
880 break;
881
882 case -1: /* MAIL response error */
883 *failure_ptr = US"mail";
884 if (errno == 0 && sx.buffer[0] == '5')
885 {
886 setflag(addr, af_verify_nsfail);
887 if (from_address[0] == 0)
888 new_domain_record.result = ccache_reject_mfnull;
889 }
890 break;
891 /* non-MAIL read i/o error */
892 /* non-MAIL response timeout */
893 /* internal error; channel still usable */
894 default: break; /* transmit failed */
895 }
896 }
897
898 addr->auth_sndr = client_authenticated_sender;
899
900 deliver_host = deliver_host_address = NULL;
901 deliver_domain = save_deliver_domain;
902
903 /* Do postmaster check if requested; if a full check is required, we
904 check for RCPT TO:<postmaster> (no domain) in accordance with RFC 821. */
905
906 if (done && pm_mailfrom)
907 {
908 /* Could possibly shift before main verify, just above, and be ok
909 for cutthrough. But no way to handle a subsequent rcpt, so just
910 refuse any */
57cc2785 911 cancel_cutthrough_connection(TRUE, US"postmaster verify");
e1d04f48 912 HDEBUG(D_acl|D_v) debug_printf_indent("Cutthrough cancelled by presence of postmaster verify\n");
e9166683 913
4e910c01 914 done = smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0
e9166683
JH
915 && smtp_read_response(&sx.inblock, sx.buffer,
916 sizeof(sx.buffer), '2', callout);
917
918 if (done)
919 {
920 uschar * main_address = addr->address;
921
922 /*XXX oops, affixes */
923 addr->address = string_sprintf("postmaster@%.1000s", addr->domain);
924 addr->transport_return = PENDING_DEFER;
925
926 sx.from_addr = pm_mailfrom;
927 sx.first_addr = sx.sync_addr = addr;
928 sx.ok = FALSE;
929 sx.send_rset = TRUE;
930 sx.completed_addr = FALSE;
14de8063 931 sx.avoid_option = OPTION_SIZE;
e9166683
JH
932
933 if( smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0
934 && addr->transport_return == PENDING_OK
935 )
936 done = TRUE;
921dfc11 937 else
e9166683 938 done = (options & vopt_callout_fullpm) != 0
4e910c01 939 && smtp_write_command(&sx.outblock, SCMD_FLUSH,
e9166683
JH
940 "RCPT TO:<postmaster>\r\n") >= 0
941 && smtp_read_response(&sx.inblock, sx.buffer,
942 sizeof(sx.buffer), '2', callout);
921dfc11 943
e9166683 944 /* Sort out the cache record */
2a4be8f9 945
e9166683
JH
946 new_domain_record.postmaster_stamp = time(NULL);
947
948 if (done)
949 new_domain_record.postmaster_result = ccache_accept;
950 else if (errno == 0 && sx.buffer[0] == '5')
951 {
952 *failure_ptr = US"postmaster";
953 setflag(addr, af_verify_pmfail);
954 new_domain_record.postmaster_result = ccache_reject;
955 }
956
957 addr->address = main_address;
958 }
959 }
193e3acd
JH
960 /* For any failure of the main check, other than a negative response, we just
961 close the connection and carry on. We can identify a negative response by the
962 fact that errno is zero. For I/O errors it will be non-zero
2a4be8f9 963
193e3acd
JH
964 Set up different error texts for logging and for sending back to the caller
965 as an SMTP response. Log in all cases, using a one-line format. For sender
966 callouts, give a full response to the caller, but for recipient callouts,
967 don't give the IP address because this may be an internal host whose identity
968 is not to be widely broadcast. */
2a4be8f9 969
02b41d71 970no_conn:
e9166683 971 switch(errno)
193e3acd 972 {
02b41d71
JH
973 case ETIMEDOUT:
974 HDEBUG(D_verify) debug_printf("SMTP timeout\n");
975 sx.send_quit = FALSE;
976 break;
977
8c5d388a 978#ifdef SUPPORT_I18N
02b41d71 979 case ERRNO_UTF8_FWD:
9bfc60eb
JH
980 {
981 extern int acl_where; /* src/acl.c */
982 errno = 0;
983 addr->message = string_sprintf(
e9166683 984 "response to \"EHLO\" did not include SMTPUTF8");
02b41d71
JH
985 addr->user_message = acl_where == ACL_WHERE_RCPT
986 ? US"533 no support for internationalised mailbox name"
9bfc60eb
JH
987 : US"550 mailbox unavailable";
988 yield = FAIL;
989 done = TRUE;
990 }
02b41d71 991 break;
9bfc60eb 992#endif
02b41d71
JH
993 case ECONNREFUSED:
994 sx.send_quit = FALSE;
995 break;
2a4be8f9 996
02b41d71 997 case 0:
e9166683 998 if (*sx.buffer == 0) Ustrcpy(sx.buffer, US"connection dropped");
059ec3d9 999
e9166683 1000 /*XXX test here is ugly; seem to have a split of responsibility for
b6323c75 1001 building this message. Need to rationalise. Where is it done
e9166683
JH
1002 before here, and when not?
1003 Not == 5xx resp to MAIL on main-verify
1004 */
1005 if (!addr->message) addr->message =
1006 string_sprintf("response to \"%s\" was: %s",
1007 big_buffer, string_printing(sx.buffer));
02b41d71
JH
1008
1009 addr->user_message = options & vopt_is_recipient
e9166683 1010 ? string_sprintf("Callout verification failed:\n%s", sx.buffer)
02b41d71 1011 : string_sprintf("Called: %s\nSent: %s\nResponse: %s",
e9166683 1012 host->address, big_buffer, sx.buffer);
059ec3d9 1013
02b41d71 1014 /* Hard rejection ends the process */
193e3acd 1015
e9166683 1016 if (sx.buffer[0] == '5') /* Address rejected */
02b41d71
JH
1017 {
1018 yield = FAIL;
1019 done = TRUE;
1020 }
1021 break;
193e3acd 1022 }
059ec3d9 1023
193e3acd
JH
1024 /* End the SMTP conversation and close the connection. */
1025
4c04137d 1026 /* Cutthrough - on a successful connect and recipient-verify with
5032d1cf 1027 use-sender and we are 1st rcpt and have no cutthrough conn so far
57cc2785
JH
1028 here is where we want to leave the conn open. Ditto for a lazy-close
1029 verify. */
1030
857eaf37
JH
1031 if (cutthrough.delivery)
1032 {
1033 if (addr->transport->filter_command)
1034 {
1035 cutthrough.delivery= FALSE;
1036 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n");
1037 }
72cb765f
JH
1038#ifndef DISABLE_DKIM
1039 if (ob->dkim.dkim_domain)
857eaf37
JH
1040 {
1041 cutthrough.delivery= FALSE;
72cb765f 1042 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n");
857eaf37 1043 }
72cb765f
JH
1044#endif
1045#ifdef EXPERIMENTAL_ARC
1046 if (ob->arc_sign)
1047 {
1048 cutthrough.delivery= FALSE;
1049 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of ARC signing\n");
1050 }
1051#endif
857eaf37
JH
1052 }
1053
57cc2785 1054 if ( (cutthrough.delivery || options & vopt_callout_hold)
5032d1cf 1055 && rcpt_count == 1
193e3acd
JH
1056 && done
1057 && yield == OK
98c82a3d
JH
1058 && (options & (vopt_callout_recipsender|vopt_callout_recippmaster|vopt_success_on_redirect))
1059 == vopt_callout_recipsender
193e3acd
JH
1060 && !random_local_part
1061 && !pm_mailfrom
5032d1cf 1062 && cutthrough.fd < 0
02b41d71 1063 && !sx.lmtp
193e3acd 1064 )
059ec3d9 1065 {
57cc2785
JH
1066 HDEBUG(D_acl|D_v) debug_printf_indent("holding verify callout open for %s\n",
1067 cutthrough.delivery
1068 ? "cutthrough delivery" : "potential further verifies and delivery");
1069
1070 cutthrough.callout_hold_only = !cutthrough.delivery;
1071 cutthrough.is_tls = tls_out.active >= 0;
1072 cutthrough.fd = sx.outblock.sock; /* We assume no buffer in use in the outblock */
1073 cutthrough.nrcpt = 1;
1074 cutthrough.transport = addr->transport->name;
1075 cutthrough.interface = interface;
1076 cutthrough.snd_port = sending_port;
1077 cutthrough.peer_options = smtp_peer_options;
1078 cutthrough.host = *host;
1079 {
1080 int oldpool = store_pool;
1081 store_pool = POOL_PERM;
1082 cutthrough.snd_ip = string_copy(sending_ip_address);
1083 cutthrough.host.name = string_copy(host->name);
1084 cutthrough.host.address = string_copy(host->address);
1085 store_pool = oldpool;
1086 }
1087 cutthrough.addr = *addr; /* Save the address_item for later logging */
1088 cutthrough.addr.next = NULL;
5032d1cf 1089 cutthrough.addr.host_used = &cutthrough.host;
193e3acd 1090 if (addr->parent)
5032d1cf
JH
1091 *(cutthrough.addr.parent = store_get(sizeof(address_item))) =
1092 *addr->parent;
193e3acd
JH
1093 ctblock.buffer = ctbuffer;
1094 ctblock.buffersize = sizeof(ctbuffer);
1095 ctblock.ptr = ctbuffer;
1096 /* ctblock.cmd_count = 0; ctblock.authenticating = FALSE; */
5032d1cf 1097 ctblock.sock = cutthrough.fd;
059ec3d9 1098 }
193e3acd 1099 else
059ec3d9 1100 {
57cc2785 1101 /* Ensure no cutthrough on multiple verifies that were incompatible */
193e3acd 1102 if (options & vopt_callout_recipsender)
57cc2785 1103 cancel_cutthrough_connection(TRUE, US"not usable for cutthrough");
02b41d71 1104 if (sx.send_quit)
2760b518 1105 {
4e910c01 1106 (void) smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n");
059ec3d9 1107
2760b518 1108 /* Wait a short time for response, and discard it */
e9166683 1109 smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
2760b518
JH
1110 '2', 1);
1111 }
1112
02b41d71
JH
1113 if (sx.inblock.sock >= 0)
1114 {
a7538db1 1115#ifdef SUPPORT_TLS
dec766a1 1116 tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
a7538db1 1117#endif
e1d04f48 1118 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
02b41d71
JH
1119 (void)close(sx.inblock.sock);
1120 sx.inblock.sock = sx.outblock.sock = -1;
0cbf2b82 1121#ifndef DISABLE_EVENT
02b41d71 1122 (void) event_raise(addr->transport->event_action, US"tcp:close", NULL);
a7538db1 1123#endif
02b41d71 1124 }
059ec3d9 1125 }
059ec3d9 1126
e9166683
JH
1127 if (!done || yield != OK)
1128 addr->message = string_sprintf("%s [%s] : %s", host->name, host->address,
1129 addr->message);
193e3acd
JH
1130 } /* Loop through all hosts, while !done */
1131 }
059ec3d9
PH
1132
1133/* If we get here with done == TRUE, a successful callout happened, and yield
1134will be set OK or FAIL according to the response to the RCPT command.
1135Otherwise, we looped through the hosts but couldn't complete the business.
707ee5b1 1136However, there may be domain-specific information to cache in both cases. */
059ec3d9 1137
707ee5b1
JH
1138if (!(options & vopt_callout_no_cache))
1139 cache_callout_write(&new_domain_record, addr->domain,
1140 done, &new_address_record, address_key);
059ec3d9
PH
1141
1142/* Failure to connect to any host, or any response other than 2xx or 5xx is a
1143temporary error. If there was only one host, and a response was received, leave
1144it alone if supplying details. Otherwise, give a generic response. */
1145
707ee5b1 1146if (!done)
059ec3d9 1147 {
ff5929e3 1148 uschar * dullmsg = string_sprintf("Could not complete %s verify callout",
8b9476ba 1149 options & vopt_is_recipient ? "recipient" : "sender");
059ec3d9
PH
1150 yield = DEFER;
1151
02b41d71
JH
1152 addr->message = host_list->next || !addr->message
1153 ? dullmsg : string_sprintf("%s: %s", dullmsg, addr->message);
059ec3d9 1154
ff5929e3
JH
1155 addr->user_message = smtp_return_error_details
1156 ? string_sprintf("%s for <%s>.\n"
059ec3d9
PH
1157 "The mail server(s) for the domain may be temporarily unreachable, or\n"
1158 "they may be permanently unreachable from this server. In the latter case,\n%s",
1159 dullmsg, addr->address,
8b9476ba 1160 options & vopt_is_recipient
ff5929e3 1161 ? "the address will never be accepted."
8b9476ba
JH
1162 : "you need to change the address or create an MX record for its domain\n"
1163 "if it is supposed to be generally accessible from the Internet.\n"
ff5929e3
JH
1164 "Talk to your mail administrator for details.")
1165 : dullmsg;
059ec3d9
PH
1166
1167 /* Force a specific error code */
1168
1169 addr->basic_errno = ERRNO_CALLOUTDEFER;
1170 }
1171
1172/* Come here from within the cache-reading code on fast-track exit. */
1173
1174END_CALLOUT:
02b41d71 1175tls_modify_variables(&tls_in);
059ec3d9
PH
1176return yield;
1177}
1178
1179
1180
817d9f57
JH
1181/* Called after recipient-acl to get a cutthrough connection open when
1182 one was requested and a recipient-verify wasn't subsequently done.
1183*/
f9334a28 1184int
e4bdf652
JH
1185open_cutthrough_connection( address_item * addr )
1186{
1187address_item addr2;
f9334a28 1188int rc;
e4bdf652
JH
1189
1190/* Use a recipient-verify-callout to set up the cutthrough connection. */
1191/* We must use a copy of the address for verification, because it might
1192get rewritten. */
1193
1194addr2 = *addr;
e1d04f48 1195HDEBUG(D_acl) debug_printf_indent("----------- %s cutthrough setup ------------\n",
5032d1cf 1196 rcpt_count > 1 ? "more" : "start");
ff5929e3 1197rc = verify_address(&addr2, NULL,
e4bdf652
JH
1198 vopt_is_recipient | vopt_callout_recipsender | vopt_callout_no_cache,
1199 CUTTHROUGH_CMD_TIMEOUT, -1, -1,
1200 NULL, NULL, NULL);
ff5929e3
JH
1201addr->message = addr2.message;
1202addr->user_message = addr2.user_message;
e1d04f48 1203HDEBUG(D_acl) debug_printf_indent("----------- end cutthrough setup ------------\n");
f9334a28 1204return rc;
e4bdf652
JH
1205}
1206
1207
e4bdf652 1208
817d9f57
JH
1209/* Send given number of bytes from the buffer */
1210static BOOL
1211cutthrough_send(int n)
e4bdf652 1212{
5032d1cf 1213if(cutthrough.fd < 0)
817d9f57 1214 return TRUE;
e4bdf652 1215
817d9f57
JH
1216if(
1217#ifdef SUPPORT_TLS
925ac8e4 1218 tls_out.active == cutthrough.fd ? tls_write(FALSE, ctblock.buffer, n, FALSE) :
817d9f57 1219#endif
5032d1cf 1220 send(cutthrough.fd, ctblock.buffer, n, 0) > 0
817d9f57
JH
1221 )
1222{
1223 transport_count += n;
1224 ctblock.ptr= ctblock.buffer;
1225 return TRUE;
1226}
e4bdf652 1227
e1d04f48 1228HDEBUG(D_transport|D_acl) debug_printf_indent("cutthrough_send failed: %s\n", strerror(errno));
817d9f57 1229return FALSE;
e4bdf652
JH
1230}
1231
1232
1233
817d9f57
JH
1234static BOOL
1235_cutthrough_puts(uschar * cp, int n)
1236{
1237while(n--)
1238 {
1239 if(ctblock.ptr >= ctblock.buffer+ctblock.buffersize)
1240 if(!cutthrough_send(ctblock.buffersize))
1241 return FALSE;
1242
1243 *ctblock.ptr++ = *cp++;
1244 }
1245return TRUE;
1246}
1247
1248/* Buffered output of counted data block. Return boolean success */
57cc2785 1249static BOOL
e4bdf652
JH
1250cutthrough_puts(uschar * cp, int n)
1251{
5032d1cf 1252if (cutthrough.fd < 0) return TRUE;
817d9f57 1253if (_cutthrough_puts(cp, n)) return TRUE;
57cc2785 1254cancel_cutthrough_connection(TRUE, US"transmit failed");
817d9f57
JH
1255return FALSE;
1256}
e4bdf652 1257
6851a9c5 1258void
57cc2785
JH
1259cutthrough_data_puts(uschar * cp, int n)
1260{
6851a9c5 1261if (cutthrough.delivery) (void) cutthrough_puts(cp, n);
78a3bbd5 1262return;
57cc2785
JH
1263}
1264
e4bdf652 1265
817d9f57 1266static BOOL
5032d1cf 1267_cutthrough_flush_send(void)
817d9f57 1268{
57cc2785 1269int n = ctblock.ptr - ctblock.buffer;
e4bdf652 1270
817d9f57
JH
1271if(n>0)
1272 if(!cutthrough_send(n))
1273 return FALSE;
1274return TRUE;
e4bdf652
JH
1275}
1276
817d9f57
JH
1277
1278/* Send out any bufferred output. Return boolean success. */
e4bdf652 1279BOOL
5032d1cf 1280cutthrough_flush_send(void)
e4bdf652 1281{
817d9f57 1282if (_cutthrough_flush_send()) return TRUE;
57cc2785 1283cancel_cutthrough_connection(TRUE, US"transmit failed");
e4bdf652
JH
1284return FALSE;
1285}
1286
1287
57cc2785 1288static BOOL
5032d1cf 1289cutthrough_put_nl(void)
e4bdf652
JH
1290{
1291return cutthrough_puts(US"\r\n", 2);
1292}
1293
1294
6851a9c5 1295void
57cc2785
JH
1296cutthrough_data_put_nl(void)
1297{
6851a9c5 1298cutthrough_data_puts(US"\r\n", 2);
57cc2785
JH
1299}
1300
1301
e4bdf652
JH
1302/* Get and check response from cutthrough target */
1303static uschar
57cc2785 1304cutthrough_response(int fd, char expect, uschar ** copy, int timeout)
e4bdf652
JH
1305{
1306smtp_inblock inblock;
1307uschar inbuffer[4096];
1308uschar responsebuffer[4096];
1309
1310inblock.buffer = inbuffer;
1311inblock.buffersize = sizeof(inbuffer);
1312inblock.ptr = inbuffer;
1313inblock.ptrend = inbuffer;
57cc2785 1314inblock.sock = fd;
817d9f57 1315/* this relies on (inblock.sock == tls_out.active) */
2760b518 1316if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, timeout))
57cc2785 1317 cancel_cutthrough_connection(TRUE, US"target timeout on read");
e4bdf652 1318
57cc2785 1319if(copy)
e4bdf652
JH
1320 {
1321 uschar * cp;
5032d1cf 1322 *copy = cp = string_copy(responsebuffer);
e4bdf652
JH
1323 /* Trim the trailing end of line */
1324 cp += Ustrlen(responsebuffer);
1325 if(cp > *copy && cp[-1] == '\n') *--cp = '\0';
1326 if(cp > *copy && cp[-1] == '\r') *--cp = '\0';
1327 }
1328
1329return responsebuffer[0];
1330}
1331
1332
1333/* Negotiate dataphase with the cutthrough target, returning success boolean */
1334BOOL
5032d1cf 1335cutthrough_predata(void)
e4bdf652 1336{
57cc2785 1337if(cutthrough.fd < 0 || cutthrough.callout_hold_only)
e4bdf652
JH
1338 return FALSE;
1339
e1d04f48 1340HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> DATA\n");
817d9f57
JH
1341cutthrough_puts(US"DATA\r\n", 6);
1342cutthrough_flush_send();
e4bdf652
JH
1343
1344/* Assume nothing buffered. If it was it gets ignored. */
57cc2785 1345return cutthrough_response(cutthrough.fd, '3', NULL, CUTTHROUGH_DATA_TIMEOUT) == '3';
e4bdf652
JH
1346}
1347
1348
42055a33 1349/* tctx arg only to match write_chunk() */
511a6c14 1350static BOOL
42055a33 1351cutthrough_write_chunk(transport_ctx * tctx, uschar * s, int len)
511a6c14
JH
1352{
1353uschar * s2;
1354while(s && (s2 = Ustrchr(s, '\n')))
1355 {
1356 if(!cutthrough_puts(s, s2-s) || !cutthrough_put_nl())
1357 return FALSE;
1358 s = s2+1;
1359 }
1360return TRUE;
1361}
1362
1363
e4bdf652 1364/* Buffered send of headers. Return success boolean. */
817d9f57 1365/* Expands newlines to wire format (CR,NL). */
e4bdf652 1366/* Also sends header-terminating blank line. */
e4bdf652 1367BOOL
5032d1cf 1368cutthrough_headers_send(void)
e4bdf652 1369{
6d5c916c
JH
1370transport_ctx tctx;
1371
57cc2785 1372if(cutthrough.fd < 0 || cutthrough.callout_hold_only)
e4bdf652
JH
1373 return FALSE;
1374
511a6c14
JH
1375/* We share a routine with the mainline transport to handle header add/remove/rewrites,
1376 but having a separate buffered-output function (for now)
1377*/
e1d04f48 1378HDEBUG(D_acl) debug_printf_indent("----------- start cutthrough headers send -----------\n");
e4bdf652 1379
42055a33 1380tctx.u.fd = cutthrough.fd;
6d5c916c
JH
1381tctx.tblock = cutthrough.addr.transport;
1382tctx.addr = &cutthrough.addr;
1383tctx.check_string = US".";
1384tctx.escape_string = US"..";
328c5688 1385/*XXX check under spool_files_wireformat. Might be irrelevant */
6d5c916c
JH
1386tctx.options = topt_use_crlf;
1387
42055a33 1388if (!transport_headers_send(&tctx, &cutthrough_write_chunk))
511a6c14
JH
1389 return FALSE;
1390
e1d04f48 1391HDEBUG(D_acl) debug_printf_indent("----------- done cutthrough headers send ------------\n");
511a6c14 1392return TRUE;
817d9f57
JH
1393}
1394
1395
1396static void
78a3bbd5 1397close_cutthrough_connection(const uschar * why)
817d9f57 1398{
57cc2785
JH
1399int fd = cutthrough.fd;
1400if(fd >= 0)
817d9f57
JH
1401 {
1402 /* We could be sending this after a bunch of data, but that is ok as
1403 the only way to cancel the transfer in dataphase is to drop the tcp
1404 conn before the final dot.
1405 */
1406 ctblock.ptr = ctbuffer;
e1d04f48 1407 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n");
817d9f57
JH
1408 _cutthrough_puts(US"QUIT\r\n", 6); /* avoid recursion */
1409 _cutthrough_flush_send();
57cc2785 1410 cutthrough.fd = -1; /* avoid recursion via read timeout */
06fdb9f7 1411 cutthrough.nrcpt = 0; /* permit re-cutthrough on subsequent message */
2760b518
JH
1412
1413 /* Wait a short time for response, and discard it */
57cc2785 1414 cutthrough_response(fd, '2', NULL, 1);
817d9f57 1415
57cc2785 1416#ifdef SUPPORT_TLS
dec766a1 1417 tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
57cc2785 1418#endif
e1d04f48 1419 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
57cc2785 1420 (void)close(fd);
e1d04f48 1421 HDEBUG(D_acl) debug_printf_indent("----------- cutthrough shutdown (%s) ------------\n", why);
817d9f57
JH
1422 }
1423ctblock.ptr = ctbuffer;
e4bdf652
JH
1424}
1425
817d9f57 1426void
57cc2785
JH
1427cancel_cutthrough_connection(BOOL close_noncutthrough_verifies, const uschar * why)
1428{
1429if (cutthrough.delivery || close_noncutthrough_verifies)
1430 close_cutthrough_connection(why);
1431cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
1432}
1433
1434
1435void
1436release_cutthrough_connection(const uschar * why)
817d9f57 1437{
5ea5d64c 1438if (cutthrough.fd < 0) return;
57cc2785
JH
1439HDEBUG(D_acl) debug_printf_indent("release cutthrough conn: %s\n", why);
1440cutthrough.fd = -1;
1441cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
817d9f57
JH
1442}
1443
1444
1445
e4bdf652
JH
1446
1447/* Have senders final-dot. Send one to cutthrough target, and grab the response.
1448 Log an OK response as a transmission.
817d9f57 1449 Close the connection.
e4bdf652 1450 Return smtp response-class digit.
e4bdf652
JH
1451*/
1452uschar *
5032d1cf 1453cutthrough_finaldot(void)
e4bdf652 1454{
5032d1cf
JH
1455uschar res;
1456address_item * addr;
e1d04f48 1457HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> .\n");
e4bdf652
JH
1458
1459/* Assume data finshed with new-line */
5032d1cf
JH
1460if( !cutthrough_puts(US".", 1)
1461 || !cutthrough_put_nl()
1462 || !cutthrough_flush_send()
1463 )
1464 return cutthrough.addr.message;
e4bdf652 1465
57cc2785 1466res = cutthrough_response(cutthrough.fd, '2', &cutthrough.addr.message, CUTTHROUGH_DATA_TIMEOUT);
5032d1cf 1467for (addr = &cutthrough.addr; addr; addr = addr->next)
817d9f57 1468 {
5032d1cf
JH
1469 addr->message = cutthrough.addr.message;
1470 switch(res)
1471 {
1472 case '2':
1473 delivery_log(LOG_MAIN, addr, (int)'>', NULL);
78a3bbd5 1474 close_cutthrough_connection(US"delivered");
5032d1cf 1475 break;
817d9f57 1476
5032d1cf
JH
1477 case '4':
1478 delivery_log(LOG_MAIN, addr, 0,
1479 US"tmp-reject from cutthrough after DATA:");
1480 break;
e4bdf652 1481
5032d1cf
JH
1482 case '5':
1483 delivery_log(LOG_MAIN|LOG_REJECT, addr, 0,
1484 US"rejected after DATA:");
1485 break;
e4bdf652 1486
5032d1cf
JH
1487 default:
1488 break;
1489 }
817d9f57 1490 }
5032d1cf 1491return cutthrough.addr.message;
e4bdf652
JH
1492}
1493
1494
817d9f57 1495
059ec3d9
PH
1496/*************************************************
1497* Copy error to toplevel address *
1498*************************************************/
1499
1500/* This function is used when a verify fails or defers, to ensure that the
1501failure or defer information is in the original toplevel address. This applies
1502when an address is redirected to a single new address, and the failure or
1503deferral happens to the child address.
1504
1505Arguments:
1506 vaddr the verify address item
1507 addr the final address item
1508 yield FAIL or DEFER
1509
1510Returns: the value of YIELD
1511*/
1512
1513static int
1514copy_error(address_item *vaddr, address_item *addr, int yield)
1515{
1516if (addr != vaddr)
1517 {
1518 vaddr->message = addr->message;
1519 vaddr->user_message = addr->user_message;
1520 vaddr->basic_errno = addr->basic_errno;
1521 vaddr->more_errno = addr->more_errno;
d43cbe25 1522 vaddr->prop.address_data = addr->prop.address_data;
42855d71 1523 copyflag(vaddr, addr, af_pass_message);
059ec3d9
PH
1524 }
1525return yield;
1526}
1527
1528
1529
1530
ce552449
NM
1531/**************************************************
1532* printf that automatically handles TLS if needed *
1533***************************************************/
1534
1535/* This function is used by verify_address() as a substitute for all fprintf()
1536calls; a direct fprintf() will not produce output in a TLS SMTP session, such
1537as a response to an EXPN command. smtp_in.c makes smtp_printf available but
1538that assumes that we always use the smtp_out FILE* when not using TLS or the
1539ssl buffer when we are. Instead we take a FILE* parameter and check to see if
1540that is smtp_out; if so, smtp_printf() with TLS support, otherwise regular
1541fprintf().
1542
1543Arguments:
1544 f the candidate FILE* to write to
1545 format format string
1546 ... optional arguments
1547
1548Returns:
1549 nothing
1550*/
1551
1552static void PRINTF_FUNCTION(2,3)
1ba28e2b 1553respond_printf(FILE *f, const char *format, ...)
ce552449
NM
1554{
1555va_list ap;
1556
1557va_start(ap, format);
1558if (smtp_out && (f == smtp_out))
925ac8e4 1559 smtp_vprintf(format, FALSE, ap);
ce552449 1560else
513afc6a 1561 vfprintf(f, format, ap);
ce552449
NM
1562va_end(ap);
1563}
1564
1565
1566
059ec3d9
PH
1567/*************************************************
1568* Verify an email address *
1569*************************************************/
1570
1571/* This function is used both for verification (-bv and at other times) and
1572address testing (-bt), which is indicated by address_test_mode being set.
1573
1574Arguments:
1575 vaddr contains the address to verify; the next field in this block
1576 must be NULL
1577 f if not NULL, write the result to this file
1578 options various option bits:
1579 vopt_fake_sender => this sender verify is not for the real
1580 sender (it was verify=sender=xxxx or an address from a
1581 header line) - rewriting must not change sender_address
1582 vopt_is_recipient => this is a recipient address, otherwise
1583 it's a sender address - this affects qualification and
1584 rewriting and messages from callouts
1585 vopt_qualify => qualify an unqualified address; else error
1586 vopt_expn => called from SMTP EXPN command
eafd343b
TK
1587 vopt_success_on_redirect => when a new address is generated
1588 the verification instantly succeeds
059ec3d9
PH
1589
1590 These ones are used by do_callout() -- the options variable
1591 is passed to it.
1592
2a4be8f9 1593 vopt_callout_fullpm => if postmaster check, do full one
059ec3d9
PH
1594 vopt_callout_no_cache => don't use callout cache
1595 vopt_callout_random => do the "random" thing
1596 vopt_callout_recipsender => use real sender for recipient
1597 vopt_callout_recippmaster => use postmaster for recipient
1598
1599 callout if > 0, specifies that callout is required, and gives timeout
4deaf07d 1600 for individual commands
059ec3d9
PH
1601 callout_overall if > 0, gives overall timeout for the callout function;
1602 if < 0, a default is used (see do_callout())
8e669ac1 1603 callout_connect the connection timeout for callouts
059ec3d9
PH
1604 se_mailfrom when callout is requested to verify a sender, use this
1605 in MAIL FROM; NULL => ""
1606 pm_mailfrom when callout is requested, if non-NULL, do the postmaster
1607 thing and use this as the sender address (may be "")
1608
1609 routed if not NULL, set TRUE if routing succeeded, so we can
1610 distinguish between routing failed and callout failed
1611
1612Returns: OK address verified
1613 FAIL address failed to verify
1614 DEFER can't tell at present
1615*/
1616
1617int
1618verify_address(address_item *vaddr, FILE *f, int options, int callout,
8e669ac1 1619 int callout_overall, int callout_connect, uschar *se_mailfrom,
4deaf07d 1620 uschar *pm_mailfrom, BOOL *routed)
059ec3d9
PH
1621{
1622BOOL allok = TRUE;
1623BOOL full_info = (f == NULL)? FALSE : (debug_selector != 0);
059ec3d9 1624BOOL expn = (options & vopt_expn) != 0;
eafd343b 1625BOOL success_on_redirect = (options & vopt_success_on_redirect) != 0;
059ec3d9
PH
1626int i;
1627int yield = OK;
1628int verify_type = expn? v_expn :
1629 address_test_mode? v_none :
8b9476ba 1630 options & vopt_is_recipient? v_recipient : v_sender;
059ec3d9
PH
1631address_item *addr_list;
1632address_item *addr_new = NULL;
1633address_item *addr_remote = NULL;
1634address_item *addr_local = NULL;
1635address_item *addr_succeed = NULL;
8b9476ba
JH
1636uschar **failure_ptr = options & vopt_is_recipient
1637 ? &recipient_verify_failure : &sender_verify_failure;
059ec3d9
PH
1638uschar *ko_prefix, *cr;
1639uschar *address = vaddr->address;
1640uschar *save_sender;
1641uschar null_sender[] = { 0 }; /* Ensure writeable memory */
1642
2c7db3f5
PH
1643/* Clear, just in case */
1644
1645*failure_ptr = NULL;
1646
059ec3d9
PH
1647/* Set up a prefix and suffix for error message which allow us to use the same
1648output statements both in EXPN mode (where an SMTP response is needed) and when
1649debugging with an output file. */
1650
1651if (expn)
1652 {
1653 ko_prefix = US"553 ";
1654 cr = US"\r";
1655 }
1656else ko_prefix = cr = US"";
1657
1658/* Add qualify domain if permitted; otherwise an unqualified address fails. */
1659
1660if (parse_find_at(address) == NULL)
1661 {
2ddb4094 1662 if (!(options & vopt_qualify))
059ec3d9 1663 {
2ddb4094 1664 if (f)
ce552449
NM
1665 respond_printf(f, "%sA domain is required for \"%s\"%s\n",
1666 ko_prefix, address, cr);
8e669ac1 1667 *failure_ptr = US"qualify";
059ec3d9
PH
1668 return FAIL;
1669 }
8b9476ba 1670 address = rewrite_address_qualify(address, options & vopt_is_recipient);
059ec3d9
PH
1671 }
1672
1673DEBUG(D_verify)
1674 {
1675 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1676 debug_printf("%s %s\n", address_test_mode? "Testing" : "Verifying", address);
1677 }
1678
1679/* Rewrite and report on it. Clear the domain and local part caches - these
1680may have been set by domains and local part tests during an ACL. */
1681
2ddb4094 1682if (global_rewrite_rules)
059ec3d9
PH
1683 {
1684 uschar *old = address;
8b9476ba 1685 address = rewrite_address(address, options & vopt_is_recipient, FALSE,
059ec3d9
PH
1686 global_rewrite_rules, rewrite_existflags);
1687 if (address != old)
1688 {
1689 for (i = 0; i < (MAX_NAMED_LIST * 2)/32; i++) vaddr->localpart_cache[i] = 0;
1690 for (i = 0; i < (MAX_NAMED_LIST * 2)/32; i++) vaddr->domain_cache[i] = 0;
2ddb4094 1691 if (f && !expn) fprintf(f, "Address rewritten as: %s\n", address);
059ec3d9
PH
1692 }
1693 }
1694
1695/* If this is the real sender address, we must update sender_address at
1696this point, because it may be referred to in the routers. */
1697
2ddb4094 1698if (!(options & (vopt_fake_sender|vopt_is_recipient)))
059ec3d9
PH
1699 sender_address = address;
1700
1701/* If the address was rewritten to <> no verification can be done, and we have
1702to return OK. This rewriting is permitted only for sender addresses; for other
1703addresses, such rewriting fails. */
1704
2ddb4094 1705if (!address[0]) return OK;
059ec3d9 1706
d9b2312b
JH
1707/* Flip the legacy TLS-related variables over to the outbound set in case
1708they're used in the context of a transport used by verification. Reset them
ea90b718 1709at exit from this routine (so no returns allowed from here on). */
d9b2312b 1710
35aba663 1711tls_modify_variables(&tls_out);
d9b2312b 1712
059ec3d9
PH
1713/* Save a copy of the sender address for re-instating if we change it to <>
1714while verifying a sender address (a nice bit of self-reference there). */
1715
1716save_sender = sender_address;
1717
ea90b718
JH
1718/* Observability variable for router/transport use */
1719
8b9476ba 1720verify_mode = options & vopt_is_recipient ? US"R" : US"S";
ea90b718 1721
059ec3d9
PH
1722/* Update the address structure with the possibly qualified and rewritten
1723address. Set it up as the starting address on the chain of new addresses. */
1724
1725vaddr->address = address;
1726addr_new = vaddr;
1727
1728/* We need a loop, because an address can generate new addresses. We must also
1729cope with generated pipes and files at the top level. (See also the code and
1730comment in deliver.c.) However, it is usually the case that the router for
1731user's .forward files has its verify flag turned off.
1732
1733If an address generates more than one child, the loop is used only when
1734full_info is set, and this can only be set locally. Remote enquiries just get
1735information about the top level address, not anything that it generated. */
1736
ea90b718 1737while (addr_new)
059ec3d9
PH
1738 {
1739 int rc;
1740 address_item *addr = addr_new;
1741
1742 addr_new = addr->next;
1743 addr->next = NULL;
1744
1745 DEBUG(D_verify)
1746 {
1747 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1748 debug_printf("Considering %s\n", addr->address);
1749 }
1750
1751 /* Handle generated pipe, file or reply addresses. We don't get these
1752 when handling EXPN, as it does only one level of expansion. */
1753
1754 if (testflag(addr, af_pfr))
1755 {
1756 allok = FALSE;
2ddb4094 1757 if (f)
059ec3d9
PH
1758 {
1759 BOOL allow;
1760
1761 if (addr->address[0] == '>')
1762 {
1763 allow = testflag(addr, af_allow_reply);
1764 fprintf(f, "%s -> mail %s", addr->parent->address, addr->address + 1);
1765 }
1766 else
1767 {
2ddb4094
JH
1768 allow = addr->address[0] == '|'
1769 ? testflag(addr, af_allow_pipe) : testflag(addr, af_allow_file);
059ec3d9
PH
1770 fprintf(f, "%s -> %s", addr->parent->address, addr->address);
1771 }
1772
1773 if (addr->basic_errno == ERRNO_BADTRANSPORT)
1774 fprintf(f, "\n*** Error in setting up pipe, file, or autoreply:\n"
1775 "%s\n", addr->message);
1776 else if (allow)
1777 fprintf(f, "\n transport = %s\n", addr->transport->name);
1778 else
1779 fprintf(f, " *** forbidden ***\n");
1780 }
1781 continue;
1782 }
1783
1784 /* Just in case some router parameter refers to it. */
1785
2f682e45
JH
1786 return_path = addr->prop.errors_address
1787 ? addr->prop.errors_address : sender_address;
059ec3d9
PH
1788
1789 /* Split the address into domain and local part, handling the %-hack if
1790 necessary, and then route it. While routing a sender address, set
1791 $sender_address to <> because that is what it will be if we were trying to
1792 send a bounce to the sender. */
1793
2f682e45 1794 if (routed) *routed = FALSE;
059ec3d9
PH
1795 if ((rc = deliver_split_address(addr)) == OK)
1796 {
8b9476ba 1797 if (!(options & vopt_is_recipient)) sender_address = null_sender;
059ec3d9
PH
1798 rc = route_address(addr, &addr_local, &addr_remote, &addr_new,
1799 &addr_succeed, verify_type);
1800 sender_address = save_sender; /* Put back the real sender */
1801 }
1802
1803 /* If routing an address succeeded, set the flag that remembers, for use when
1804 an ACL cached a sender verify (in case a callout fails). Then if routing set
1805 up a list of hosts or the transport has a host list, and the callout option
1806 is set, and we aren't in a host checking run, do the callout verification,
1807 and set another flag that notes that a callout happened. */
1808
1809 if (rc == OK)
1810 {
2f682e45 1811 if (routed) *routed = TRUE;
059ec3d9
PH
1812 if (callout > 0)
1813 {
08f3b11b 1814 transport_instance * tp;
2f682e45 1815 host_item * host_list = addr->host_list;
059ec3d9 1816
26da7e20
PH
1817 /* Make up some data for use in the case where there is no remote
1818 transport. */
1819
1820 transport_feedback tf = {
f2ed27cf
JH
1821 .interface = NULL, /* interface (=> any) */
1822 .port = US"smtp",
1823 .protocol = US"smtp",
1824 .hosts = NULL,
1825 .helo_data = US"$smtp_active_hostname",
1826 .hosts_override = FALSE,
1827 .hosts_randomize = FALSE,
1828 .gethostbyname = FALSE,
1829 .qualify_single = TRUE,
1830 .search_parents = FALSE
26da7e20 1831 };
059ec3d9
PH
1832
1833 /* If verification yielded a remote transport, we want to use that
1834 transport's options, so as to mimic what would happen if we were really
1835 sending a message to this address. */
1836
08f3b11b 1837 if ((tp = addr->transport) && !tp->info->local)
059ec3d9 1838 {
08f3b11b 1839 (void)(tp->setup)(tp, addr, &tf, 0, 0, NULL);
059ec3d9
PH
1840
1841 /* If the transport has hosts and the router does not, or if the
1842 transport is configured to override the router's hosts, we must build a
1843 host list of the transport's hosts, and find the IP addresses */
1844
2f682e45 1845 if (tf.hosts && (!host_list || tf.hosts_override))
059ec3d9
PH
1846 {
1847 uschar *s;
55414b25 1848 const uschar *save_deliver_domain = deliver_domain;
750af86e 1849 uschar *save_deliver_localpart = deliver_localpart;
059ec3d9
PH
1850
1851 host_list = NULL; /* Ignore the router's hosts */
1852
1853 deliver_domain = addr->domain;
1854 deliver_localpart = addr->local_part;
1855 s = expand_string(tf.hosts);
750af86e
PH
1856 deliver_domain = save_deliver_domain;
1857 deliver_localpart = save_deliver_localpart;
059ec3d9 1858
2f682e45 1859 if (!s)
059ec3d9
PH
1860 {
1861 log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand list of hosts "
1862 "\"%s\" in %s transport for callout: %s", tf.hosts,
08f3b11b 1863 tp->name, expand_string_message);
059ec3d9
PH
1864 }
1865 else
1866 {
322050c2 1867 int flags;
d8ef3577 1868 host_item *host, *nexthost;
059ec3d9
PH
1869 host_build_hostlist(&host_list, s, tf.hosts_randomize);
1870
1871 /* Just ignore failures to find a host address. If we don't manage
8e669ac1
PH
1872 to find any addresses, the callout will defer. Note that more than
1873 one address may be found for a single host, which will result in
1874 additional host items being inserted into the chain. Hence we must
d8ef3577 1875 save the next host first. */
059ec3d9 1876
66387a73 1877 flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA;
322050c2
PH
1878 if (tf.qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE;
1879 if (tf.search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
1880
2f682e45 1881 for (host = host_list; host; host = nexthost)
059ec3d9 1882 {
d8ef3577 1883 nexthost = host->next;
8e669ac1 1884 if (tf.gethostbyname ||
7e66e54d 1885 string_is_ip_address(host->name, NULL) != 0)
55414b25 1886 (void)host_find_byname(host, NULL, flags, NULL, TRUE);
059ec3d9 1887 else
9d9c3746 1888 {
7cd171b7 1889 dnssec_domains * dnssec_domains = NULL;
08f3b11b 1890 if (Ustrcmp(tp->driver_name, "smtp") == 0)
9d9c3746
JH
1891 {
1892 smtp_transport_options_block * ob =
08f3b11b 1893 (smtp_transport_options_block *) tp->options_block;
7cd171b7 1894 dnssec_domains = &ob->dnssec;
9d9c3746
JH
1895 }
1896
2546388c 1897 (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
7cd171b7 1898 dnssec_domains, NULL, NULL);
9d9c3746 1899 }
059ec3d9
PH
1900 }
1901 }
1902 }
1903 }
1904
8e669ac1 1905 /* Can only do a callout if we have at least one host! If the callout
2c7db3f5 1906 fails, it will have set ${sender,recipient}_verify_failure. */
059ec3d9 1907
2f682e45 1908 if (host_list)
059ec3d9
PH
1909 {
1910 HDEBUG(D_verify) debug_printf("Attempting full verification using callout\n");
1911 if (host_checking && !host_checking_callout)
1912 {
1913 HDEBUG(D_verify)
1914 debug_printf("... callout omitted by default when host testing\n"
1915 "(Use -bhc if you want the callouts to happen.)\n");
1916 }
1917 else
1918 {
4ed8d31a
JH
1919#ifdef SUPPORT_TLS
1920 deliver_set_expansions(addr);
1921#endif
059ec3d9 1922 rc = do_callout(addr, host_list, &tf, callout, callout_overall,
4deaf07d 1923 callout_connect, options, se_mailfrom, pm_mailfrom);
059ec3d9
PH
1924 }
1925 }
1926 else
1927 {
1928 HDEBUG(D_verify) debug_printf("Cannot do callout: neither router nor "
1929 "transport provided a host list\n");
1930 }
1931 }
1932 }
8e669ac1 1933
2c7db3f5 1934 /* Otherwise, any failure is a routing failure */
8e669ac1
PH
1935
1936 else *failure_ptr = US"route";
059ec3d9
PH
1937
1938 /* A router may return REROUTED if it has set up a child address as a result
1939 of a change of domain name (typically from widening). In this case we always
1940 want to continue to verify the new child. */
1941
1942 if (rc == REROUTED) continue;
8e669ac1 1943
059ec3d9
PH
1944 /* Handle hard failures */
1945
1946 if (rc == FAIL)
1947 {
1948 allok = FALSE;
2f682e45 1949 if (f)
059ec3d9 1950 {
e6f6568e
PH
1951 address_item *p = addr->parent;
1952
ce552449 1953 respond_printf(f, "%s%s %s", ko_prefix,
2f682e45
JH
1954 full_info ? addr->address : address,
1955 address_test_mode ? "is undeliverable" : "failed to verify");
059ec3d9
PH
1956 if (!expn && admin_user)
1957 {
1958 if (addr->basic_errno > 0)
ce552449 1959 respond_printf(f, ": %s", strerror(addr->basic_errno));
2f682e45 1960 if (addr->message)
ce552449 1961 respond_printf(f, ": %s", addr->message);
e6f6568e
PH
1962 }
1963
1964 /* Show parents iff doing full info */
1965
2f682e45 1966 if (full_info) while (p)
e6f6568e 1967 {
ce552449 1968 respond_printf(f, "%s\n <-- %s", cr, p->address);
e6f6568e 1969 p = p->parent;
059ec3d9 1970 }
ce552449 1971 respond_printf(f, "%s\n", cr);
059ec3d9 1972 }
57cc2785 1973 cancel_cutthrough_connection(TRUE, US"routing hard fail");
059ec3d9 1974
d9b2312b 1975 if (!full_info)
2f682e45 1976 {
d9b2312b
JH
1977 yield = copy_error(vaddr, addr, FAIL);
1978 goto out;
2f682e45
JH
1979 }
1980 yield = FAIL;
059ec3d9
PH
1981 }
1982
1983 /* Soft failure */
1984
1985 else if (rc == DEFER)
1986 {
1987 allok = FALSE;
2f682e45 1988 if (f)
059ec3d9 1989 {
e6f6568e 1990 address_item *p = addr->parent;
ce552449 1991 respond_printf(f, "%s%s cannot be resolved at this time", ko_prefix,
322050c2 1992 full_info? addr->address : address);
059ec3d9
PH
1993 if (!expn && admin_user)
1994 {
1995 if (addr->basic_errno > 0)
ce552449 1996 respond_printf(f, ": %s", strerror(addr->basic_errno));
2f682e45 1997 if (addr->message)
ce552449 1998 respond_printf(f, ": %s", addr->message);
059ec3d9 1999 else if (addr->basic_errno <= 0)
ce552449 2000 respond_printf(f, ": unknown error");
059ec3d9
PH
2001 }
2002
e6f6568e
PH
2003 /* Show parents iff doing full info */
2004
2f682e45 2005 if (full_info) while (p)
e6f6568e 2006 {
ce552449 2007 respond_printf(f, "%s\n <-- %s", cr, p->address);
e6f6568e
PH
2008 p = p->parent;
2009 }
ce552449 2010 respond_printf(f, "%s\n", cr);
059ec3d9 2011 }
57cc2785 2012 cancel_cutthrough_connection(TRUE, US"routing soft fail");
e4bdf652 2013
d9b2312b
JH
2014 if (!full_info)
2015 {
2016 yield = copy_error(vaddr, addr, DEFER);
2017 goto out;
2018 }
2f682e45 2019 if (yield == OK) yield = DEFER;
059ec3d9
PH
2020 }
2021
2022 /* If we are handling EXPN, we do not want to continue to route beyond
e6f6568e 2023 the top level (whose address is in "address"). */
059ec3d9
PH
2024
2025 else if (expn)
2026 {
2027 uschar *ok_prefix = US"250-";
2f682e45
JH
2028
2029 if (!addr_new)
2030 if (!addr_local && !addr_remote)
ce552449 2031 respond_printf(f, "250 mail to <%s> is discarded\r\n", address);
059ec3d9 2032 else
ce552449 2033 respond_printf(f, "250 <%s>\r\n", address);
2f682e45
JH
2034
2035 else do
059ec3d9
PH
2036 {
2037 address_item *addr2 = addr_new;
2038 addr_new = addr2->next;
2f682e45 2039 if (!addr_new) ok_prefix = US"250 ";
ce552449 2040 respond_printf(f, "%s<%s>\r\n", ok_prefix, addr2->address);
2f682e45 2041 } while (addr_new);
d9b2312b
JH
2042 yield = OK;
2043 goto out;
059ec3d9
PH
2044 }
2045
2046 /* Successful routing other than EXPN. */
2047
2048 else
2049 {
2050 /* Handle successful routing when short info wanted. Otherwise continue for
2051 other (generated) addresses. Short info is the operational case. Full info
2052 can be requested only when debug_selector != 0 and a file is supplied.
2053
2054 There is a conflict between the use of aliasing as an alternate email
2055 address, and as a sort of mailing list. If an alias turns the incoming
2056 address into just one address (e.g. J.Caesar->jc44) you may well want to
2057 carry on verifying the generated address to ensure it is valid when
2058 checking incoming mail. If aliasing generates multiple addresses, you
2059 probably don't want to do this. Exim therefore treats the generation of
2060 just a single new address as a special case, and continues on to verify the
2061 generated address. */
2062
2f682e45
JH
2063 if ( !full_info /* Stop if short info wanted AND */
2064 && ( ( !addr_new /* No new address OR */
2065 || addr_new->next /* More than one new address OR */
2066 || testflag(addr_new, af_pfr) /* New address is pfr */
2067 )
2068 || /* OR */
2069 ( addr_new /* At least one new address AND */
2070 && success_on_redirect /* success_on_redirect is set */
2071 ) )
2072 )
059ec3d9 2073 {
2f682e45
JH
2074 if (f) fprintf(f, "%s %s\n",
2075 address, address_test_mode ? "is deliverable" : "verified");
059ec3d9
PH
2076
2077 /* If we have carried on to verify a child address, we want the value
2078 of $address_data to be that of the child */
2079
d43cbe25 2080 vaddr->prop.address_data = addr->prop.address_data;
98c82a3d
JH
2081
2082 /* If stopped because more than one new address, cannot cutthrough */
2083
2084 if (addr_new && addr_new->next)
57cc2785 2085 cancel_cutthrough_connection(TRUE, US"multiple addresses from routing");
98c82a3d 2086
d9b2312b
JH
2087 yield = OK;
2088 goto out;
059ec3d9
PH
2089 }
2090 }
2091 } /* Loop for generated addresses */
2092
2093/* Display the full results of the successful routing, including any generated
2094addresses. Control gets here only when full_info is set, which requires f not
2095to be NULL, and this occurs only when a top-level verify is called with the
2096debugging switch on.
2097
2098If there are no local and no remote addresses, and there were no pipes, files,
2099or autoreplies, and there were no errors or deferments, the message is to be
2100discarded, usually because of the use of :blackhole: in an alias file. */
2101
2f682e45 2102if (allok && !addr_local && !addr_remote)
dbcef0ea 2103 {
059ec3d9 2104 fprintf(f, "mail to %s is discarded\n", address);
d9b2312b 2105 goto out;
dbcef0ea 2106 }
059ec3d9 2107
dbcef0ea 2108for (addr_list = addr_local, i = 0; i < 2; addr_list = addr_remote, i++)
08f3b11b 2109 while (addr_list)
059ec3d9
PH
2110 {
2111 address_item *addr = addr_list;
2112 address_item *p = addr->parent;
08f3b11b
JH
2113 transport_instance * tp = addr->transport;
2114
059ec3d9
PH
2115 addr_list = addr->next;
2116
2117 fprintf(f, "%s", CS addr->address);
384152a6 2118#ifdef EXPERIMENTAL_SRS
d43cbe25
JH
2119 if(addr->prop.srs_sender)
2120 fprintf(f, " [srs = %s]", addr->prop.srs_sender);
384152a6 2121#endif
dbcef0ea
PH
2122
2123 /* If the address is a duplicate, show something about it. */
2124
2125 if (!testflag(addr, af_pfr))
2126 {
2127 tree_node *tnode;
08f3b11b 2128 if ((tnode = tree_search(tree_duplicates, addr->unique)))
dbcef0ea
PH
2129 fprintf(f, " [duplicate, would not be delivered]");
2130 else tree_add_duplicate(addr->unique, addr);
2131 }
2132
2133 /* Now show its parents */
2134
08f3b11b 2135 for (p = addr->parent; p; p = p->parent)
059ec3d9 2136 fprintf(f, "\n <-- %s", p->address);
059ec3d9
PH
2137 fprintf(f, "\n ");
2138
2139 /* Show router, and transport */
2140
08f3b11b
JH
2141 fprintf(f, "router = %s, transport = %s\n",
2142 addr->router->name, tp ? tp->name : US"unset");
059ec3d9
PH
2143
2144 /* Show any hosts that are set up by a router unless the transport
2145 is going to override them; fiddle a bit to get a nice format. */
2146
08f3b11b 2147 if (addr->host_list && tp && !tp->overrides_hosts)
059ec3d9
PH
2148 {
2149 host_item *h;
2150 int maxlen = 0;
2151 int maxaddlen = 0;
08f3b11b
JH
2152 for (h = addr->host_list; h; h = h->next)
2153 { /* get max lengths of host names, addrs */
059ec3d9
PH
2154 int len = Ustrlen(h->name);
2155 if (len > maxlen) maxlen = len;
08f3b11b 2156 len = h->address ? Ustrlen(h->address) : 7;
059ec3d9
PH
2157 if (len > maxaddlen) maxaddlen = len;
2158 }
08f3b11b
JH
2159 for (h = addr->host_list; h; h = h->next)
2160 {
2161 fprintf(f, " host %-*s ", maxlen, h->name);
2162
2163 if (h->address)
2164 fprintf(f, "[%s%-*c", h->address, maxaddlen+1 - Ustrlen(h->address), ']');
2165 else if (tp->info->local)
2166 fprintf(f, " %-*s ", maxaddlen, ""); /* Omit [unknown] for local */
2167 else
2168 fprintf(f, "[%s%-*c", "unknown", maxaddlen+1 - 7, ']');
2169
2170 if (h->mx >= 0) fprintf(f, " MX=%d", h->mx);
059ec3d9 2171 if (h->port != PORT_NONE) fprintf(f, " port=%d", h->port);
08f3b11b
JH
2172 if (running_in_test_harness && h->dnssec == DS_YES) fputs(" AD", f);
2173 if (h->status == hstatus_unusable) fputs(" ** unusable **", f);
2174 fputc('\n', f);
059ec3d9
PH
2175 }
2176 }
2177 }
059ec3d9 2178
d9b2312b 2179/* Yield will be DEFER or FAIL if any one address has, only for full_info (which is
2c7db3f5
PH
2180the -bv or -bt case). */
2181
d9b2312b 2182out:
ea90b718 2183verify_mode = NULL;
35aba663 2184tls_modify_variables(&tls_in);
d9b2312b 2185
8e669ac1 2186return yield;
059ec3d9
PH
2187}
2188
2189
2190
2191
2192/*************************************************
2193* Check headers for syntax errors *
2194*************************************************/
2195
2196/* This function checks those header lines that contain addresses, and verifies
db57e575 2197that all the addresses therein are 5322-syntactially correct.
059ec3d9
PH
2198
2199Arguments:
2200 msgptr where to put an error message
2201
2202Returns: OK
2203 FAIL
2204*/
2205
2206int
2207verify_check_headers(uschar **msgptr)
2208{
2209header_line *h;
2210uschar *colon, *s;
1eccaa59 2211int yield = OK;
059ec3d9 2212
db57e575 2213for (h = header_list; h && yield == OK; h = h->next)
059ec3d9
PH
2214 {
2215 if (h->type != htype_from &&
2216 h->type != htype_reply_to &&
2217 h->type != htype_sender &&
2218 h->type != htype_to &&
2219 h->type != htype_cc &&
2220 h->type != htype_bcc)
2221 continue;
2222
2223 colon = Ustrchr(h->text, ':');
2224 s = colon + 1;
2225 while (isspace(*s)) s++;
2226
1eccaa59
PH
2227 /* Loop for multiple addresses in the header, enabling group syntax. Note
2228 that we have to reset this after the header has been scanned. */
059ec3d9 2229
1eccaa59 2230 parse_allow_group = TRUE;
059ec3d9 2231
db57e575 2232 while (*s)
059ec3d9
PH
2233 {
2234 uschar *ss = parse_find_address_end(s, FALSE);
2235 uschar *recipient, *errmess;
2236 int terminator = *ss;
2237 int start, end, domain;
2238
2239 /* Temporarily terminate the string at this point, and extract the
1eccaa59 2240 operative address within, allowing group syntax. */
059ec3d9
PH
2241
2242 *ss = 0;
2243 recipient = parse_extract_address(s,&errmess,&start,&end,&domain,FALSE);
2244 *ss = terminator;
2245
2246 /* Permit an unqualified address only if the message is local, or if the
2247 sending host is configured to be permitted to send them. */
2248
db57e575 2249 if (recipient && !domain)
059ec3d9
PH
2250 {
2251 if (h->type == htype_from || h->type == htype_sender)
2252 {
2253 if (!allow_unqualified_sender) recipient = NULL;
2254 }
2255 else
2256 {
2257 if (!allow_unqualified_recipient) recipient = NULL;
2258 }
2259 if (recipient == NULL) errmess = US"unqualified address not permitted";
2260 }
2261
2262 /* It's an error if no address could be extracted, except for the special
2263 case of an empty address. */
2264
db57e575 2265 if (!recipient && Ustrcmp(errmess, "empty address") != 0)
059ec3d9
PH
2266 {
2267 uschar *verb = US"is";
2268 uschar *t = ss;
1ab95fa6 2269 uschar *tt = colon;
059ec3d9
PH
2270 int len;
2271
2272 /* Arrange not to include any white space at the end in the
1ab95fa6 2273 error message or the header name. */
059ec3d9
PH
2274
2275 while (t > s && isspace(t[-1])) t--;
1ab95fa6 2276 while (tt > h->text && isspace(tt[-1])) tt--;
059ec3d9 2277
1ab95fa6 2278 /* Add the address that failed to the error message, since in a
059ec3d9
PH
2279 header with very many addresses it is sometimes hard to spot
2280 which one is at fault. However, limit the amount of address to
2281 quote - cases have been seen where, for example, a missing double
2282 quote in a humungous To: header creates an "address" that is longer
2283 than string_sprintf can handle. */
2284
2285 len = t - s;
2286 if (len > 1024)
2287 {
2288 len = 1024;
2289 verb = US"begins";
2290 }
2291
55414b25
JH
2292 /* deconst cast ok as we're passing a non-const to string_printing() */
2293 *msgptr = US string_printing(
1ab95fa6 2294 string_sprintf("%s: failing address in \"%.*s:\" header %s: %.*s",
bb07bcd3 2295 errmess, (int)(tt - h->text), h->text, verb, len, s));
059ec3d9 2296
1eccaa59
PH
2297 yield = FAIL;
2298 break; /* Out of address loop */
059ec3d9
PH
2299 }
2300
2301 /* Advance to the next address */
2302
db57e575 2303 s = ss + (terminator ? 1 : 0);
059ec3d9
PH
2304 while (isspace(*s)) s++;
2305 } /* Next address */
059ec3d9 2306
1eccaa59
PH
2307 parse_allow_group = FALSE;
2308 parse_found_group = FALSE;
2309 } /* Next header unless yield has been set FALSE */
2310
2311return yield;
059ec3d9
PH
2312}
2313
2314
770747fd
MFM
2315/*************************************************
2316* Check header names for 8-bit characters *
2317*************************************************/
2318
4c04137d 2319/* This function checks for invalid characters in header names. See
770747fd
MFM
2320RFC 5322, 2.2. and RFC 6532, 3.
2321
2322Arguments:
2323 msgptr where to put an error message
2324
2325Returns: OK
2326 FAIL
2327*/
2328
2329int
2330verify_check_header_names_ascii(uschar **msgptr)
2331{
2332header_line *h;
2333uschar *colon, *s;
2334
57cc2785 2335for (h = header_list; h; h = h->next)
770747fd 2336 {
57cc2785
JH
2337 colon = Ustrchr(h->text, ':');
2338 for(s = h->text; s < colon; s++)
2339 if ((*s < 33) || (*s > 126))
2340 {
2341 *msgptr = string_sprintf("Invalid character in header \"%.*s\" found",
2342 colon - h->text, h->text);
2343 return FAIL;
2344 }
770747fd
MFM
2345 }
2346return OK;
2347}
059ec3d9 2348
1c41c9cc
PH
2349/*************************************************
2350* Check for blind recipients *
2351*************************************************/
2352
2353/* This function checks that every (envelope) recipient is mentioned in either
2354the To: or Cc: header lines, thus detecting blind carbon copies.
2355
2356There are two ways of scanning that could be used: either scan the header lines
2357and tick off the recipients, or scan the recipients and check the header lines.
2358The original proposed patch did the former, but I have chosen to do the latter,
2359because (a) it requires no memory and (b) will use fewer resources when there
2360are many addresses in To: and/or Cc: and only one or two envelope recipients.
2361
2362Arguments: none
2363Returns: OK if there are no blind recipients
2364 FAIL if there is at least one blind recipient
2365*/
2366
2367int
2368verify_check_notblind(void)
2369{
2370int i;
2371for (i = 0; i < recipients_count; i++)
2372 {
2373 header_line *h;
2374 BOOL found = FALSE;
2375 uschar *address = recipients_list[i].address;
2376
2377 for (h = header_list; !found && h != NULL; h = h->next)
2378 {
2379 uschar *colon, *s;
2380
2381 if (h->type != htype_to && h->type != htype_cc) continue;
2382
2383 colon = Ustrchr(h->text, ':');
2384 s = colon + 1;
2385 while (isspace(*s)) s++;
2386
1eccaa59
PH
2387 /* Loop for multiple addresses in the header, enabling group syntax. Note
2388 that we have to reset this after the header has been scanned. */
1c41c9cc 2389
1eccaa59 2390 parse_allow_group = TRUE;
1c41c9cc
PH
2391
2392 while (*s != 0)
2393 {
2394 uschar *ss = parse_find_address_end(s, FALSE);
2395 uschar *recipient,*errmess;
2396 int terminator = *ss;
2397 int start, end, domain;
2398
2399 /* Temporarily terminate the string at this point, and extract the
1eccaa59 2400 operative address within, allowing group syntax. */
1c41c9cc
PH
2401
2402 *ss = 0;
2403 recipient = parse_extract_address(s,&errmess,&start,&end,&domain,FALSE);
2404 *ss = terminator;
2405
2406 /* If we found a valid recipient that has a domain, compare it with the
2407 envelope recipient. Local parts are compared case-sensitively, domains
2408 case-insensitively. By comparing from the start with length "domain", we
2409 include the "@" at the end, which ensures that we are comparing the whole
2410 local part of each address. */
2411
2412 if (recipient != NULL && domain != 0)
2413 {
2414 found = Ustrncmp(recipient, address, domain) == 0 &&
2415 strcmpic(recipient + domain, address + domain) == 0;
2416 if (found) break;
2417 }
2418
2419 /* Advance to the next address */
2420
2421 s = ss + (terminator? 1:0);
2422 while (isspace(*s)) s++;
2423 } /* Next address */
1eccaa59
PH
2424
2425 parse_allow_group = FALSE;
2426 parse_found_group = FALSE;
1c41c9cc
PH
2427 } /* Next header (if found is false) */
2428
2429 if (!found) return FAIL;
2430 } /* Next recipient */
2431
2432return OK;
2433}
2434
2435
059ec3d9
PH
2436
2437/*************************************************
2438* Find if verified sender *
2439*************************************************/
2440
2441/* Usually, just a single address is verified as the sender of the message.
2442However, Exim can be made to verify other addresses as well (often related in
2443some way), and this is useful in some environments. There may therefore be a
2444chain of such addresses that have previously been tested. This function finds
2445whether a given address is on the chain.
2446
2447Arguments: the address to be verified
2448Returns: pointer to an address item, or NULL
2449*/
2450
2451address_item *
2452verify_checked_sender(uschar *sender)
2453{
2454address_item *addr;
2455for (addr = sender_verified_list; addr != NULL; addr = addr->next)
2456 if (Ustrcmp(sender, addr->address) == 0) break;
2457return addr;
2458}
2459
2460
2461
2462
2463
2464/*************************************************
2465* Get valid header address *
2466*************************************************/
2467
2468/* Scan the originator headers of the message, looking for an address that
2469verifies successfully. RFC 822 says:
2470
2471 o The "Sender" field mailbox should be sent notices of
2472 any problems in transport or delivery of the original
2473 messages. If there is no "Sender" field, then the
2474 "From" field mailbox should be used.
2475
2476 o If the "Reply-To" field exists, then the reply should
2477 go to the addresses indicated in that field and not to
2478 the address(es) indicated in the "From" field.
2479
2480So we check a Sender field if there is one, else a Reply_to field, else a From
2481field. As some strange messages may have more than one of these fields,
2482especially if they are resent- fields, check all of them if there is more than
2483one.
2484
2485Arguments:
2486 user_msgptr points to where to put a user error message
2487 log_msgptr points to where to put a log error message
2488 callout timeout for callout check (passed to verify_address())
2489 callout_overall overall callout timeout (ditto)
8e669ac1 2490 callout_connect connect callout timeout (ditto)
059ec3d9
PH
2491 se_mailfrom mailfrom for verify; NULL => ""
2492 pm_mailfrom sender for pm callout check (passed to verify_address())
2493 options callout options (passed to verify_address())
8e669ac1 2494 verrno where to put the address basic_errno
059ec3d9
PH
2495
2496If log_msgptr is set to something without setting user_msgptr, the caller
2497normally uses log_msgptr for both things.
2498
2499Returns: result of the verification attempt: OK, FAIL, or DEFER;
2500 FAIL is given if no appropriate headers are found
2501*/
2502
2503int
2504verify_check_header_address(uschar **user_msgptr, uschar **log_msgptr,
8e669ac1 2505 int callout, int callout_overall, int callout_connect, uschar *se_mailfrom,
fe5b5d0b 2506 uschar *pm_mailfrom, int options, int *verrno)
059ec3d9
PH
2507{
2508static int header_types[] = { htype_sender, htype_reply_to, htype_from };
1eccaa59 2509BOOL done = FALSE;
059ec3d9
PH
2510int yield = FAIL;
2511int i;
2512
1eccaa59 2513for (i = 0; i < 3 && !done; i++)
059ec3d9
PH
2514 {
2515 header_line *h;
1eccaa59 2516 for (h = header_list; h != NULL && !done; h = h->next)
059ec3d9
PH
2517 {
2518 int terminator, new_ok;
2519 uschar *s, *ss, *endname;
2520
2521 if (h->type != header_types[i]) continue;
2522 s = endname = Ustrchr(h->text, ':') + 1;
2523
1eccaa59
PH
2524 /* Scan the addresses in the header, enabling group syntax. Note that we
2525 have to reset this after the header has been scanned. */
2526
2527 parse_allow_group = TRUE;
2528
059ec3d9
PH
2529 while (*s != 0)
2530 {
2531 address_item *vaddr;
2532
2533 while (isspace(*s) || *s == ',') s++;
2534 if (*s == 0) break; /* End of header */
2535
2536 ss = parse_find_address_end(s, FALSE);
2537
2538 /* The terminator is a comma or end of header, but there may be white
2539 space preceding it (including newline for the last address). Move back
2540 past any white space so we can check against any cached envelope sender
2541 address verifications. */
2542
2543 while (isspace(ss[-1])) ss--;
2544 terminator = *ss;
2545 *ss = 0;
2546
2547 HDEBUG(D_verify) debug_printf("verifying %.*s header address %s\n",
2548 (int)(endname - h->text), h->text, s);
2549
2550 /* See if we have already verified this address as an envelope sender,
2551 and if so, use the previous answer. */
2552
2553 vaddr = verify_checked_sender(s);
2554
2555 if (vaddr != NULL && /* Previously checked */
2556 (callout <= 0 || /* No callout needed; OR */
2557 vaddr->special_action > 256)) /* Callout was done */
2558 {
2559 new_ok = vaddr->special_action & 255;
2560 HDEBUG(D_verify) debug_printf("previously checked as envelope sender\n");
2561 *ss = terminator; /* Restore shortened string */
2562 }
2563
2564 /* Otherwise we run the verification now. We must restore the shortened
2565 string before running the verification, so the headers are correct, in
2566 case there is any rewriting. */
2567
2568 else
2569 {
2570 int start, end, domain;
1eccaa59
PH
2571 uschar *address = parse_extract_address(s, log_msgptr, &start, &end,
2572 &domain, FALSE);
059ec3d9
PH
2573
2574 *ss = terminator;
2575
1eccaa59
PH
2576 /* If we found an empty address, just carry on with the next one, but
2577 kill the message. */
2578
2579 if (address == NULL && Ustrcmp(*log_msgptr, "empty address") == 0)
2580 {
2581 *log_msgptr = NULL;
2582 s = ss;
2583 continue;
2584 }
2585
059ec3d9
PH
2586 /* If verification failed because of a syntax error, fail this
2587 function, and ensure that the failing address gets added to the error
2588 message. */
2589
2590 if (address == NULL)
2591 {
2592 new_ok = FAIL;
1eccaa59
PH
2593 while (ss > s && isspace(ss[-1])) ss--;
2594 *log_msgptr = string_sprintf("syntax error in '%.*s' header when "
2595 "scanning for sender: %s in \"%.*s\"",
bb07bcd3 2596 (int)(endname - h->text), h->text, *log_msgptr, (int)(ss - s), s);
1eccaa59
PH
2597 yield = FAIL;
2598 done = TRUE;
2599 break;
059ec3d9
PH
2600 }
2601
2f6603e1 2602 /* Else go ahead with the sender verification. But it isn't *the*
059ec3d9
PH
2603 sender of the message, so set vopt_fake_sender to stop sender_address
2604 being replaced after rewriting or qualification. */
2605
2606 else
2607 {
2608 vaddr = deliver_make_addr(address, FALSE);
2609 new_ok = verify_address(vaddr, NULL, options | vopt_fake_sender,
8e669ac1 2610 callout, callout_overall, callout_connect, se_mailfrom,
4deaf07d 2611 pm_mailfrom, NULL);
059ec3d9
PH
2612 }
2613 }
2614
2615 /* We now have the result, either newly found, or cached. If we are
2616 giving out error details, set a specific user error. This means that the
2617 last of these will be returned to the user if all three fail. We do not
2618 set a log message - the generic one below will be used. */
2619
fe5b5d0b 2620 if (new_ok != OK)
059ec3d9 2621 {
8e669ac1 2622 *verrno = vaddr->basic_errno;
fe5b5d0b 2623 if (smtp_return_error_details)
fe5b5d0b
PH
2624 *user_msgptr = string_sprintf("Rejected after DATA: "
2625 "could not verify \"%.*s\" header address\n%s: %s",
bb07bcd3 2626 (int)(endname - h->text), h->text, vaddr->address, vaddr->message);
8e669ac1 2627 }
059ec3d9
PH
2628
2629 /* Success or defer */
2630
1eccaa59
PH
2631 if (new_ok == OK)
2632 {
2633 yield = OK;
2634 done = TRUE;
2635 break;
2636 }
2637
059ec3d9
PH
2638 if (new_ok == DEFER) yield = DEFER;
2639
2640 /* Move on to any more addresses in the header */
2641
2642 s = ss;
1eccaa59
PH
2643 } /* Next address */
2644
2645 parse_allow_group = FALSE;
2646 parse_found_group = FALSE;
2647 } /* Next header, unless done */
2648 } /* Next header type unless done */
059ec3d9
PH
2649
2650if (yield == FAIL && *log_msgptr == NULL)
2651 *log_msgptr = US"there is no valid sender in any header line";
2652
2653if (yield == DEFER && *log_msgptr == NULL)
2654 *log_msgptr = US"all attempts to verify a sender in a header line deferred";
2655
2656return yield;
2657}
2658
2659
2660
2661
2662/*************************************************
2663* Get RFC 1413 identification *
2664*************************************************/
2665
2666/* Attempt to get an id from the sending machine via the RFC 1413 protocol. If
2667the timeout is set to zero, then the query is not done. There may also be lists
2668of hosts and nets which are exempt. To guard against malefactors sending
2669non-printing characters which could, for example, disrupt a message's headers,
2670make sure the string consists of printing characters only.
2671
2672Argument:
2673 port the port to connect to; usually this is IDENT_PORT (113), but when
2674 running in the test harness with -bh a different value is used.
2675
2676Returns: nothing
2677
2678Side effect: any received ident value is put in sender_ident (NULL otherwise)
2679*/
2680
2681void
2682verify_get_ident(int port)
2683{
2684int sock, host_af, qlen;
2685int received_sender_port, received_interface_port, n;
2686uschar *p;
44e6c20c 2687blob early_data;
059ec3d9
PH
2688uschar buffer[2048];
2689
2690/* Default is no ident. Check whether we want to do an ident check for this
2691host. */
2692
2693sender_ident = NULL;
2694if (rfc1413_query_timeout <= 0 || verify_check_host(&rfc1413_hosts) != OK)
2695 return;
2696
2697DEBUG(D_ident) debug_printf("doing ident callback\n");
2698
2699/* Set up a connection to the ident port of the remote host. Bind the local end
2700to the incoming interface address. If the sender host address is an IPv6
2701address, the incoming interface address will also be IPv6. */
2702
fb05276a
JH
2703host_af = Ustrchr(sender_host_address, ':') == NULL ? AF_INET : AF_INET6;
2704if ((sock = ip_socket(SOCK_STREAM, host_af)) < 0) return;
059ec3d9
PH
2705
2706if (ip_bind(sock, host_af, interface_address, 0) < 0)
2707 {
2708 DEBUG(D_ident) debug_printf("bind socket for ident failed: %s\n",
2709 strerror(errno));
2710 goto END_OFF;
2711 }
2712
44e6c20c
JH
2713/* Construct and send the query. */
2714
2715qlen = snprintf(CS buffer, sizeof(buffer), "%d , %d\r\n",
2716 sender_host_port, interface_port);
2717early_data.data = buffer;
2718early_data.len = qlen;
2719
fb05276a 2720if (ip_connect(sock, host_af, sender_host_address, port,
44e6c20c 2721 rfc1413_query_timeout, &early_data) < 0)
059ec3d9 2722 {
6c6d6e48 2723 if (errno == ETIMEDOUT && LOGGING(ident_timeout))
059ec3d9
PH
2724 log_write(0, LOG_MAIN, "ident connection to %s timed out",
2725 sender_host_address);
059ec3d9 2726 else
059ec3d9
PH
2727 DEBUG(D_ident) debug_printf("ident connection to %s failed: %s\n",
2728 sender_host_address, strerror(errno));
059ec3d9
PH
2729 goto END_OFF;
2730 }
2731
059ec3d9
PH
2732/* Read a response line. We put it into the rest of the buffer, using several
2733recv() calls if necessary. */
2734
2735p = buffer + qlen;
2736
2737for (;;)
2738 {
2739 uschar *pp;
2740 int count;
2741 int size = sizeof(buffer) - (p - buffer);
2742
2743 if (size <= 0) goto END_OFF; /* Buffer filled without seeing \n. */
2744 count = ip_recv(sock, p, size, rfc1413_query_timeout);
2745 if (count <= 0) goto END_OFF; /* Read error or EOF */
2746
2747 /* Scan what we just read, to see if we have reached the terminating \r\n. Be
2748 generous, and accept a plain \n terminator as well. The only illegal
2749 character is 0. */
2750
2751 for (pp = p; pp < p + count; pp++)
2752 {
2753 if (*pp == 0) goto END_OFF; /* Zero octet not allowed */
2754 if (*pp == '\n')
2755 {
2756 if (pp[-1] == '\r') pp--;
2757 *pp = 0;
2758 goto GOT_DATA; /* Break out of both loops */
2759 }
2760 }
2761
2762 /* Reached the end of the data without finding \n. Let the loop continue to
2763 read some more, if there is room. */
2764
2765 p = pp;
2766 }
2767
2768GOT_DATA:
2769
2770/* We have received a line of data. Check it carefully. It must start with the
2771same two port numbers that we sent, followed by data as defined by the RFC. For
2772example,
2773
2774 12345 , 25 : USERID : UNIX :root
2775
2776However, the amount of white space may be different to what we sent. In the
2777"osname" field there may be several sub-fields, comma separated. The data we
2778actually want to save follows the third colon. Some systems put leading spaces
2779in it - we discard those. */
2780
2781if (sscanf(CS buffer + qlen, "%d , %d%n", &received_sender_port,
2782 &received_interface_port, &n) != 2 ||
2783 received_sender_port != sender_host_port ||
2784 received_interface_port != interface_port)
2785 goto END_OFF;
2786
2787p = buffer + qlen + n;
2788while(isspace(*p)) p++;
2789if (*p++ != ':') goto END_OFF;
2790while(isspace(*p)) p++;
2791if (Ustrncmp(p, "USERID", 6) != 0) goto END_OFF;
2792p += 6;
2793while(isspace(*p)) p++;
2794if (*p++ != ':') goto END_OFF;
2795while (*p != 0 && *p != ':') p++;
2796if (*p++ == 0) goto END_OFF;
2797while(isspace(*p)) p++;
2798if (*p == 0) goto END_OFF;
2799
2800/* The rest of the line is the data we want. We turn it into printing
2801characters when we save it, so that it cannot mess up the format of any logging
2802or Received: lines into which it gets inserted. We keep a maximum of 127
55414b25 2803characters. The deconst cast is ok as we fed a nonconst to string_printing() */
059ec3d9 2804
55414b25 2805sender_ident = US string_printing(string_copyn(p, 127));
059ec3d9
PH
2806DEBUG(D_ident) debug_printf("sender_ident = %s\n", sender_ident);
2807
2808END_OFF:
f1e894f3 2809(void)close(sock);
059ec3d9
PH
2810return;
2811}
2812
2813
2814
2815
2816/*************************************************
2817* Match host to a single host-list item *
2818*************************************************/
2819
2820/* This function compares a host (name or address) against a single item
2821from a host list. The host name gets looked up if it is needed and is not
2822already known. The function is called from verify_check_this_host() via
2823match_check_list(), which is why most of its arguments are in a single block.
2824
2825Arguments:
2826 arg the argument block (see below)
2827 ss the host-list item
2828 valueptr where to pass back looked up data, or NULL
2829 error for error message when returning ERROR
2830
2831The block contains:
32d668a5
PH
2832 host_name (a) the host name, or
2833 (b) NULL, implying use sender_host_name and
2834 sender_host_aliases, looking them up if required, or
2835 (c) the empty string, meaning that only IP address matches
2836 are permitted
059ec3d9
PH
2837 host_address the host address
2838 host_ipv4 the IPv4 address taken from an IPv6 one
2839
2840Returns: OK matched
2841 FAIL did not match
2842 DEFER lookup deferred
32d668a5
PH
2843 ERROR (a) failed to find the host name or IP address, or
2844 (b) unknown lookup type specified, or
2845 (c) host name encountered when only IP addresses are
2846 being matched
059ec3d9
PH
2847*/
2848
32d668a5 2849int
55414b25 2850check_host(void *arg, const uschar *ss, const uschar **valueptr, uschar **error)
059ec3d9
PH
2851{
2852check_host_block *cb = (check_host_block *)arg;
32d668a5 2853int mlen = -1;
059ec3d9 2854int maskoffset;
32d668a5 2855BOOL iplookup = FALSE;
059ec3d9 2856BOOL isquery = FALSE;
32d668a5 2857BOOL isiponly = cb->host_name != NULL && cb->host_name[0] == 0;
55414b25 2858const uschar *t;
32d668a5 2859uschar *semicolon;
059ec3d9
PH
2860uschar **aliases;
2861
2862/* Optimize for the special case when the pattern is "*". */
2863
2864if (*ss == '*' && ss[1] == 0) return OK;
2865
2866/* If the pattern is empty, it matches only in the case when there is no host -
2867this can occur in ACL checking for SMTP input using the -bs option. In this
2868situation, the host address is the empty string. */
2869
2870if (cb->host_address[0] == 0) return (*ss == 0)? OK : FAIL;
2871if (*ss == 0) return FAIL;
2872
32d668a5
PH
2873/* If the pattern is precisely "@" then match against the primary host name,
2874provided that host name matching is permitted; if it's "@[]" match against the
2875local host's IP addresses. */
059ec3d9
PH
2876
2877if (*ss == '@')
2878 {
32d668a5
PH
2879 if (ss[1] == 0)
2880 {
2881 if (isiponly) return ERROR;
2882 ss = primary_hostname;
2883 }
059ec3d9
PH
2884 else if (Ustrcmp(ss, "@[]") == 0)
2885 {
2886 ip_address_item *ip;
2887 for (ip = host_find_interfaces(); ip != NULL; ip = ip->next)
2888 if (Ustrcmp(ip->address, cb->host_address) == 0) return OK;
2889 return FAIL;
2890 }
2891 }
2892
2893/* If the pattern is an IP address, optionally followed by a bitmask count, do
4c04137d 2894a (possibly masked) comparison with the current IP address. */
059ec3d9 2895
7e66e54d 2896if (string_is_ip_address(ss, &maskoffset) != 0)
059ec3d9
PH
2897 return (host_is_in_net(cb->host_address, ss, maskoffset)? OK : FAIL);
2898
1688f43b
PH
2899/* The pattern is not an IP address. A common error that people make is to omit
2900one component of an IPv4 address, either by accident, or believing that, for
2901example, 1.2.3/24 is the same as 1.2.3.0/24, or 1.2.3 is the same as 1.2.3.0,
2902which it isn't. (Those applications that do accept 1.2.3 as an IP address
2903interpret it as 1.2.0.3 because the final component becomes 16-bit - this is an
2904ancient specification.) To aid in debugging these cases, we give a specific
2905error if the pattern contains only digits and dots or contains a slash preceded
2906only by digits and dots (a slash at the start indicates a file name and of
2907course slashes may be present in lookups, but not preceded only by digits and
2908dots). */
2909
2910for (t = ss; isdigit(*t) || *t == '.'; t++);
2911if (*t == 0 || (*t == '/' && t != ss))
2912 {
2913 *error = US"malformed IPv4 address or address mask";
2914 return ERROR;
2915 }
2916
32d668a5 2917/* See if there is a semicolon in the pattern */
059ec3d9 2918
32d668a5
PH
2919semicolon = Ustrchr(ss, ';');
2920
2921/* If we are doing an IP address only match, then all lookups must be IP
df199fec 2922address lookups, even if there is no "net-". */
32d668a5
PH
2923
2924if (isiponly)
059ec3d9 2925 {
32d668a5
PH
2926 iplookup = semicolon != NULL;
2927 }
059ec3d9 2928
32d668a5 2929/* Otherwise, if the item is of the form net[n]-lookup;<file|query> then it is
df199fec
PH
2930a lookup on a masked IP network, in textual form. We obey this code even if we
2931have already set iplookup, so as to skip over the "net-" prefix and to set the
2932mask length. The net- stuff really only applies to single-key lookups where the
2933key is implicit. For query-style lookups the key is specified in the query.
2934From release 4.30, the use of net- for query style is no longer needed, but we
2935retain it for backward compatibility. */
2936
2937if (Ustrncmp(ss, "net", 3) == 0 && semicolon != NULL)
32d668a5
PH
2938 {
2939 mlen = 0;
2940 for (t = ss + 3; isdigit(*t); t++) mlen = mlen * 10 + *t - '0';
2941 if (mlen == 0 && t == ss+3) mlen = -1; /* No mask supplied */
2942 iplookup = (*t++ == '-');
2943 }
1688f43b 2944else t = ss;
059ec3d9 2945
32d668a5 2946/* Do the IP address lookup if that is indeed what we have */
059ec3d9 2947
32d668a5
PH
2948if (iplookup)
2949 {
2950 int insize;
2951 int search_type;
2952 int incoming[4];
2953 void *handle;
2954 uschar *filename, *key, *result;
2955 uschar buffer[64];
059ec3d9 2956
32d668a5 2957 /* Find the search type */
059ec3d9 2958
32d668a5 2959 search_type = search_findtype(t, semicolon - t);
059ec3d9 2960
32d668a5
PH
2961 if (search_type < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s",
2962 search_error_message);
059ec3d9 2963
13b685f9
PH
2964 /* Adjust parameters for the type of lookup. For a query-style lookup, there
2965 is no file name, and the "key" is just the query. For query-style with a file
2966 name, we have to fish the file off the start of the query. For a single-key
2967 lookup, the key is the current IP address, masked appropriately, and
2968 reconverted to text form, with the mask appended. For IPv6 addresses, specify
6a3bceb1
PH
2969 dot separators instead of colons, except when the lookup type is "iplsearch".
2970 */
059ec3d9 2971
13b685f9
PH
2972 if (mac_islookup(search_type, lookup_absfilequery))
2973 {
2974 filename = semicolon + 1;
2975 key = filename;
2976 while (*key != 0 && !isspace(*key)) key++;
2977 filename = string_copyn(filename, key - filename);
2978 while (isspace(*key)) key++;
2979 }
2980 else if (mac_islookup(search_type, lookup_querystyle))
32d668a5
PH
2981 {
2982 filename = NULL;
2983 key = semicolon + 1;
2984 }
6a3bceb1 2985 else /* Single-key style */
32d668a5 2986 {
e6d225ae 2987 int sep = (Ustrcmp(lookup_list[search_type]->name, "iplsearch") == 0)?
6a3bceb1 2988 ':' : '.';
32d668a5
PH
2989 insize = host_aton(cb->host_address, incoming);
2990 host_mask(insize, incoming, mlen);
6a3bceb1 2991 (void)host_nmtoa(insize, incoming, mlen, buffer, sep);
32d668a5
PH
2992 key = buffer;
2993 filename = semicolon + 1;
059ec3d9 2994 }
32d668a5
PH
2995
2996 /* Now do the actual lookup; note that there is no search_close() because
2997 of the caching arrangements. */
2998
d4ff61d1
JH
2999 if (!(handle = search_open(filename, search_type, 0, NULL, NULL)))
3000 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);
3001
32d668a5
PH
3002 result = search_find(handle, filename, key, -1, NULL, 0, 0, NULL);
3003 if (valueptr != NULL) *valueptr = result;
3004 return (result != NULL)? OK : search_find_defer? DEFER: FAIL;
059ec3d9
PH
3005 }
3006
3007/* The pattern is not an IP address or network reference of any kind. That is,
32d668a5
PH
3008it is a host name pattern. If this is an IP only match, there's an error in the
3009host list. */
3010
3011if (isiponly)
3012 {
3013 *error = US"cannot match host name in match_ip list";
3014 return ERROR;
3015 }
3016
3017/* Check the characters of the pattern to see if they comprise only letters,
3018digits, full stops, and hyphens (the constituents of domain names). Allow
3019underscores, as they are all too commonly found. Sigh. Also, if
3020allow_utf8_domains is set, allow top-bit characters. */
059ec3d9
PH
3021
3022for (t = ss; *t != 0; t++)
3023 if (!isalnum(*t) && *t != '.' && *t != '-' && *t != '_' &&
3024 (!allow_utf8_domains || *t < 128)) break;
3025
3026/* If the pattern is a complete domain name, with no fancy characters, look up
3027its IP address and match against that. Note that a multi-homed host will add
3028items to the chain. */
3029
3030if (*t == 0)
3031 {
3032 int rc;
3033 host_item h;
3034 h.next = NULL;
3035 h.name = ss;
3036 h.address = NULL;
3037 h.mx = MX_NONE;
9b8fadde 3038
1f155f8e
JH
3039 /* Using byname rather than bydns here means we cannot determine dnssec
3040 status. On the other hand it is unclear how that could be either
3041 propagated up or enforced. */
3042
322050c2 3043 rc = host_find_byname(&h, NULL, HOST_FIND_QUALIFY_SINGLE, NULL, FALSE);
059ec3d9
PH
3044 if (rc == HOST_FOUND || rc == HOST_FOUND_LOCAL)
3045 {
3046 host_item *hh;
3047 for (hh = &h; hh != NULL; hh = hh->next)
3048 {
96776534 3049 if (host_is_in_net(hh->address, cb->host_address, 0)) return OK;
059ec3d9
PH
3050 }
3051 return FAIL;
3052 }
3053 if (rc == HOST_FIND_AGAIN) return DEFER;
3054 *error = string_sprintf("failed to find IP address for %s", ss);
3055 return ERROR;
3056 }
3057
3058/* Almost all subsequent comparisons require the host name, and can be done
3059using the general string matching function. When this function is called for
3060outgoing hosts, the name is always given explicitly. If it is NULL, it means we
3061must use sender_host_name and its aliases, looking them up if necessary. */
3062
3063if (cb->host_name != NULL) /* Explicit host name given */
3064 return match_check_string(cb->host_name, ss, -1, TRUE, TRUE, TRUE,
3065 valueptr);
3066
3067/* Host name not given; in principle we need the sender host name and its
3068aliases. However, for query-style lookups, we do not need the name if the
3069query does not contain $sender_host_name. From release 4.23, a reference to
3070$sender_host_name causes it to be looked up, so we don't need to do the lookup
3071on spec. */
3072
3073if ((semicolon = Ustrchr(ss, ';')) != NULL)
3074 {
55414b25 3075 const uschar *affix;
059ec3d9
PH
3076 int partial, affixlen, starflags, id;
3077
3078 *semicolon = 0;
3079 id = search_findtype_partial(ss, &partial, &affix, &affixlen, &starflags);
3080 *semicolon=';';
3081
3082 if (id < 0) /* Unknown lookup type */
3083 {
3084 log_write(0, LOG_MAIN|LOG_PANIC, "%s in host list item \"%s\"",
3085 search_error_message, ss);
3086 return DEFER;
3087 }
13b685f9 3088 isquery = mac_islookup(id, lookup_querystyle|lookup_absfilequery);
059ec3d9
PH
3089 }
3090
3091if (isquery)
3092 {
3093 switch(match_check_string(US"", ss, -1, TRUE, TRUE, TRUE, valueptr))
3094 {
3095 case OK: return OK;
3096 case DEFER: return DEFER;
3097 default: return FAIL;
3098 }
3099 }
3100
3101/* Not a query-style lookup; must ensure the host name is present, and then we
3102do a check on the name and all its aliases. */
3103
2d0dc929 3104if (!sender_host_name)
059ec3d9
PH
3105 {
3106 HDEBUG(D_host_lookup)
3107 debug_printf("sender host name required, to match against %s\n", ss);
3108 if (host_lookup_failed || host_name_lookup() != OK)
3109 {
3110 *error = string_sprintf("failed to find host name for %s",
3111 sender_host_address);;
3112 return ERROR;
3113 }
3114 host_build_sender_fullhost();
3115 }
3116
3117/* Match on the sender host name, using the general matching function */
3118
2d0dc929 3119switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, valueptr))
059ec3d9
PH
3120 {
3121 case OK: return OK;
3122 case DEFER: return DEFER;
3123 }
3124
3125/* If there are aliases, try matching on them. */
3126
3127aliases = sender_host_aliases;
2d0dc929 3128while (*aliases)
059ec3d9
PH
3129 switch(match_check_string(*aliases++, ss, -1, TRUE, TRUE, TRUE, valueptr))
3130 {
3131 case OK: return OK;
3132 case DEFER: return DEFER;
3133 }
059ec3d9
PH
3134return FAIL;
3135}
3136
3137
3138
3139
3140/*************************************************
3141* Check a specific host matches a host list *
3142*************************************************/
3143
3144/* This function is passed a host list containing items in a number of
3145different formats and the identity of a host. Its job is to determine whether
3146the given host is in the set of hosts defined by the list. The host name is
3147passed as a pointer so that it can be looked up if needed and not already
3148known. This is commonly the case when called from verify_check_host() to check
3149an incoming connection. When called from elsewhere the host name should usually
3150be set.
3151
3152This function is now just a front end to match_check_list(), which runs common
3153code for scanning a list. We pass it the check_host() function to perform a
3154single test.
3155
3156Arguments:
3157 listptr pointer to the host list
3158 cache_bits pointer to cache for named lists, or NULL
3159 host_name the host name or NULL, implying use sender_host_name and
3160 sender_host_aliases, looking them up if required
3161 host_address the IP address
3162 valueptr if not NULL, data from a lookup is passed back here
3163
3164Returns: OK if the host is in the defined set
3165 FAIL if the host is not in the defined set,
3166 DEFER if a data lookup deferred (not a host lookup)
3167
3168If the host name was needed in order to make a comparison, and could not be
3169determined from the IP address, the result is FAIL unless the item
3170"+allow_unknown" was met earlier in the list, in which case OK is returned. */
3171
3172int
55414b25
JH
3173verify_check_this_host(const uschar **listptr, unsigned int *cache_bits,
3174 const uschar *host_name, const uschar *host_address, const uschar **valueptr)
059ec3d9 3175{
d4eb88df 3176int rc;
059ec3d9 3177unsigned int *local_cache_bits = cache_bits;
55414b25 3178const uschar *save_host_address = deliver_host_address;
0ab63f3d 3179check_host_block cb = { .host_name = host_name, .host_address = host_address };
059ec3d9 3180
0ab63f3d 3181if (valueptr) *valueptr = NULL;
059ec3d9
PH
3182
3183/* If the host address starts off ::ffff: it is an IPv6 address in
3184IPv4-compatible mode. Find the IPv4 part for checking against IPv4
3185addresses. */
3186
0ab63f3d
JH
3187cb.host_ipv4 = Ustrncmp(host_address, "::ffff:", 7) == 0
3188 ? host_address + 7 : host_address;
059ec3d9 3189
8e669ac1
PH
3190/* During the running of the check, put the IP address into $host_address. In
3191the case of calls from the smtp transport, it will already be there. However,
3192in other calls (e.g. when testing ignore_target_hosts), it won't. Just to be on
d4eb88df
PH
3193the safe side, any existing setting is preserved, though as I write this
3194(November 2004) I can't see any cases where it is actually needed. */
3195
3196deliver_host_address = host_address;
3197rc = match_check_list(
3198 listptr, /* the list */
3199 0, /* separator character */
3200 &hostlist_anchor, /* anchor pointer */
3201 &local_cache_bits, /* cache pointer */
3202 check_host, /* function for testing */
3203 &cb, /* argument for function */
3204 MCL_HOST, /* type of check */
8e669ac1 3205 (host_address == sender_host_address)?
d4eb88df
PH
3206 US"host" : host_address, /* text for debugging */
3207 valueptr); /* where to pass back data */
3208deliver_host_address = save_host_address;
8e669ac1 3209return rc;
059ec3d9
PH
3210}
3211
3212
3213
3214
3215/*************************************************
5130845b
JH
3216* Check the given host item matches a list *
3217*************************************************/
3218int
3219verify_check_given_host(uschar **listptr, host_item *host)
3220{
55414b25 3221return verify_check_this_host(CUSS listptr, NULL, host->name, host->address, NULL);
5130845b
JH
3222}
3223
3224/*************************************************
059ec3d9
PH
3225* Check the remote host matches a list *
3226*************************************************/
3227
3228/* This is a front end to verify_check_this_host(), created because checking
3229the remote host is a common occurrence. With luck, a good compiler will spot
3230the tail recursion and optimize it. If there's no host address, this is
3231command-line SMTP input - check against an empty string for the address.
3232
3233Arguments:
3234 listptr pointer to the host list
3235
3236Returns: the yield of verify_check_this_host(),
3237 i.e. OK, FAIL, or DEFER
3238*/
3239
3240int
3241verify_check_host(uschar **listptr)
3242{
55414b25 3243return verify_check_this_host(CUSS listptr, sender_host_cache, NULL,
059ec3d9
PH
3244 (sender_host_address == NULL)? US"" : sender_host_address, NULL);
3245}
3246
3247
3248
3249
3250
3251/*************************************************
83e029d5 3252* Invert an IP address *
059ec3d9
PH
3253*************************************************/
3254
83e029d5
PP
3255/* Originally just used for DNS xBL lists, now also used for the
3256reverse_ip expansion operator.
3257
059ec3d9
PH
3258Arguments:
3259 buffer where to put the answer
3260 address the address to invert
3261*/
3262
83e029d5 3263void
059ec3d9
PH
3264invert_address(uschar *buffer, uschar *address)
3265{
3266int bin[4];
3267uschar *bptr = buffer;
3268
3269/* If this is an IPv4 address mapped into IPv6 format, adjust the pointer
3270to the IPv4 part only. */
3271
3272if (Ustrncmp(address, "::ffff:", 7) == 0) address += 7;
3273
3274/* Handle IPv4 address: when HAVE_IPV6 is false, the result of host_aton() is
3275always 1. */
3276
3277if (host_aton(address, bin) == 1)
3278 {
3279 int i;
3280 int x = bin[0];
3281 for (i = 0; i < 4; i++)
3282 {
3283 sprintf(CS bptr, "%d.", x & 255);
3284 while (*bptr) bptr++;
3285 x >>= 8;
3286 }
3287 }
3288
3289/* Handle IPv6 address. Actually, as far as I know, there are no IPv6 addresses
3290in any DNS black lists, and the format in which they will be looked up is
3291unknown. This is just a guess. */
3292
3293#if HAVE_IPV6
3294else
3295 {
3296 int i, j;
3297 for (j = 3; j >= 0; j--)
3298 {
3299 int x = bin[j];
3300 for (i = 0; i < 8; i++)
3301 {
3302 sprintf(CS bptr, "%x.", x & 15);
3303 while (*bptr) bptr++;
3304 x >>= 4;
3305 }
3306 }
3307 }
3308#endif
d6f6e0dc
PH
3309
3310/* Remove trailing period -- this is needed so that both arbitrary
3311dnsbl keydomains and inverted addresses may be combined with the
3312same format string, "%s.%s" */
3313
3314*(--bptr) = 0;
059ec3d9
PH
3315}
3316
3317
3318
3319/*************************************************
0bcb2a0e
PH
3320* Perform a single dnsbl lookup *
3321*************************************************/
3322
d6f6e0dc
PH
3323/* This function is called from verify_check_dnsbl() below. It is also called
3324recursively from within itself when domain and domain_txt are different
3325pointers, in order to get the TXT record from the alternate domain.
0bcb2a0e
PH
3326
3327Arguments:
d6f6e0dc
PH
3328 domain the outer dnsbl domain
3329 domain_txt alternate domain to lookup TXT record on success; when the
3330 same domain is to be used, domain_txt == domain (that is,
3331 the pointers must be identical, not just the text)
8e669ac1 3332 keydomain the current keydomain (for debug message)
d6f6e0dc
PH
3333 prepend subdomain to lookup (like keydomain, but
3334 reversed if IP address)
3335 iplist the list of matching IP addresses, or NULL for "any"
8e669ac1 3336 bitmask true if bitmask matching is wanted
431b7361
PH
3337 match_type condition for 'succeed' result
3338 0 => Any RR in iplist (=)
3339 1 => No RR in iplist (!=)
3340 2 => All RRs in iplist (==)
3341 3 => Some RRs not in iplist (!==)
3342 the two bits are defined as MT_NOT and MT_ALL
8e669ac1 3343 defer_return what to return for a defer
0bcb2a0e
PH
3344
3345Returns: OK if lookup succeeded
3346 FAIL if not
3347*/
3348
3349static int
d6f6e0dc 3350one_check_dnsbl(uschar *domain, uschar *domain_txt, uschar *keydomain,
431b7361 3351 uschar *prepend, uschar *iplist, BOOL bitmask, int match_type,
d6f6e0dc 3352 int defer_return)
8e669ac1 3353{
0bcb2a0e
PH
3354dns_answer dnsa;
3355dns_scan dnss;
3356tree_node *t;
3357dnsbl_cache_block *cb;
3358int old_pool = store_pool;
d6f6e0dc
PH
3359uschar query[256]; /* DNS domain max length */
3360
3361/* Construct the specific query domainname */
3362
3363if (!string_format(query, sizeof(query), "%s.%s", prepend, domain))
3364 {
3365 log_write(0, LOG_MAIN|LOG_PANIC, "dnslist query is too long "
3366 "(ignored): %s...", query);
3367 return FAIL;
3368 }
0bcb2a0e
PH
3369
3370/* Look for this query in the cache. */
3371
14b3c5bc
JH
3372if ( (t = tree_search(dnsbl_cache, query))
3373 && (cb = t->data.ptr)->expiry > time(NULL)
3374 )
3375
3376/* Previous lookup was cached */
3377
3378 {
3379 HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
3380 }
0bcb2a0e
PH
3381
3382/* If not cached from a previous lookup, we must do a DNS lookup, and
3383cache the result in permanent memory. */
3384
14b3c5bc 3385else
0bcb2a0e 3386 {
e162fc97 3387 uint ttl = 3600;
14b3c5bc 3388
0bcb2a0e
PH
3389 store_pool = POOL_PERM;
3390
14b3c5bc
JH
3391 if (t)
3392 {
3393 HDEBUG(D_dnsbl) debug_printf("cached data found but past valid time; ");
3394 }
0bcb2a0e 3395
14b3c5bc
JH
3396 else
3397 { /* Set up a tree entry to cache the lookup */
3398 t = store_get(sizeof(tree_node) + Ustrlen(query));
3399 Ustrcpy(t->name, query);
3400 t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
3401 (void)tree_insertnode(&dnsbl_cache, t);
3402 }
0bcb2a0e 3403
4c04137d 3404 /* Do the DNS lookup . */
0bcb2a0e
PH
3405
3406 HDEBUG(D_dnsbl) debug_printf("new DNS lookup for %s\n", query);
3407 cb->rc = dns_basic_lookup(&dnsa, query, T_A);
3408 cb->text_set = FALSE;
3409 cb->text = NULL;
3410 cb->rhs = NULL;
3411
3412 /* If the lookup succeeded, cache the RHS address. The code allows for
3413 more than one address - this was for complete generality and the possible
d8d9f930
JH
3414 use of A6 records. However, A6 records are no longer supported. Leave the code
3415 here, just in case.
0bcb2a0e
PH
3416
3417 Quite apart from one A6 RR generating multiple addresses, there are DNS
3418 lists that return more than one A record, so we must handle multiple
e162fc97
JH
3419 addresses generated in that way as well.
3420
3421 Mark the cache entry with the "now" plus the minimum of the address TTLs,
3422 or some suitably far-future time if none were found. */
0bcb2a0e
PH
3423
3424 if (cb->rc == DNS_SUCCEED)
3425 {
3426 dns_record *rr;
3427 dns_address **addrp = &(cb->rhs);
3428 for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS);
e1a3f32f 3429 rr;
0bcb2a0e 3430 rr = dns_next_rr(&dnsa, &dnss, RESET_NEXT))
0bcb2a0e
PH
3431 if (rr->type == T_A)
3432 {
3433 dns_address *da = dns_address_from_rr(&dnsa, rr);
e1a3f32f 3434 if (da)
0bcb2a0e
PH
3435 {
3436 *addrp = da;
2d0dc929
JH
3437 while (da->next) da = da->next;
3438 addrp = &da->next;
14b3c5bc 3439 if (ttl > rr->ttl) ttl = rr->ttl;
0bcb2a0e
PH
3440 }
3441 }
0bcb2a0e
PH
3442
3443 /* If we didn't find any A records, change the return code. This can
3444 happen when there is a CNAME record but there are no A records for what
3445 it points to. */
3446
2d0dc929 3447 if (!cb->rhs) cb->rc = DNS_NODATA;
0bcb2a0e
PH
3448 }
3449
14b3c5bc 3450 cb->expiry = time(NULL)+ttl;
0bcb2a0e
PH
3451 store_pool = old_pool;
3452 }
3453
0bcb2a0e
PH
3454/* We now have the result of the DNS lookup, either newly done, or cached
3455from a previous call. If the lookup succeeded, check against the address
3456list if there is one. This may be a positive equality list (introduced by
3457"="), a negative equality list (introduced by "!="), a positive bitmask
3458list (introduced by "&"), or a negative bitmask list (introduced by "!&").*/
3459
3460if (cb->rc == DNS_SUCCEED)
3461 {
3462 dns_address *da = NULL;
3463 uschar *addlist = cb->rhs->address;
3464
3465 /* For A and AAAA records, there may be multiple addresses from multiple
3466 records. For A6 records (currently not expected to be used) there may be
3467 multiple addresses from a single record. */
3468
2d0dc929 3469 for (da = cb->rhs->next; da; da = da->next)
0bcb2a0e
PH
3470 addlist = string_sprintf("%s, %s", addlist, da->address);
3471
3472 HDEBUG(D_dnsbl) debug_printf("DNS lookup for %s succeeded (yielding %s)\n",
3473 query, addlist);
3474
3475 /* Address list check; this can be either for equality, or via a bitmask.
3476 In the latter case, all the bits must match. */
3477
2d0dc929 3478 if (iplist)
0bcb2a0e 3479 {
2d0dc929 3480 for (da = cb->rhs; da; da = da->next)
0bcb2a0e 3481 {
431b7361
PH
3482 int ipsep = ',';
3483 uschar ip[46];
55414b25 3484 const uschar *ptr = iplist;
431b7361
PH
3485 uschar *res;
3486
0bcb2a0e 3487 /* Handle exact matching */
431b7361 3488
0bcb2a0e 3489 if (!bitmask)
2d0dc929