Fix cutthrough delivery for more than one iteration of address redirection. Bug...
[exim.git] / src / src / verify.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* Functions concerned with verifying things. The original code for callout
9caching was contributed by Kevin Fleming (but I hacked it around a bit). */
10
11
12#include "exim.h"
817d9f57 13#include "transports/smtp.h"
059ec3d9 14
e4bdf652
JH
15#define CUTTHROUGH_CMD_TIMEOUT 30 /* timeout for cutthrough-routing calls */
16#define CUTTHROUGH_DATA_TIMEOUT 60 /* timeout for cutthrough-routing calls */
817d9f57
JH
17static smtp_outblock ctblock;
18uschar ctbuffer[8192];
19
059ec3d9
PH
20
21/* Structure for caching DNSBL lookups */
22
23typedef struct dnsbl_cache_block {
14b3c5bc 24 time_t expiry;
059ec3d9
PH
25 dns_address *rhs;
26 uschar *text;
27 int rc;
28 BOOL text_set;
29} dnsbl_cache_block;
30
31
32/* Anchor for DNSBL cache */
33
34static tree_node *dnsbl_cache = NULL;
35
36
431b7361
PH
37/* Bits for match_type in one_check_dnsbl() */
38
39#define MT_NOT 1
40#define MT_ALL 2
41
74f1a423 42static uschar cutthrough_response(client_conn_ctx *, char, uschar **, int);
431b7361 43
8d330698 44
059ec3d9
PH
45
46/*************************************************
47* Retrieve a callout cache record *
48*************************************************/
49
50/* If a record exists, check whether it has expired.
51
52Arguments:
53 dbm_file an open hints file
54 key the record key
55 type "address" or "domain"
56 positive_expire expire time for positive records
57 negative_expire expire time for negative records
58
59Returns: the cache record if a non-expired one exists, else NULL
60*/
61
62static dbdata_callout_cache *
55414b25 63get_callout_cache_record(open_db *dbm_file, const uschar *key, uschar *type,
059ec3d9
PH
64 int positive_expire, int negative_expire)
65{
66BOOL negative;
67int length, expire;
68time_t now;
69dbdata_callout_cache *cache_record;
70
b6323c75 71if (!(cache_record = dbfn_read_with_length(dbm_file, key, &length)))
059ec3d9 72 {
6f4d5ad3 73 HDEBUG(D_verify) debug_printf("callout cache: no %s record found for %s\n", type, key);
059ec3d9
PH
74 return NULL;
75 }
76
77/* We treat a record as "negative" if its result field is not positive, or if
78it is a domain record and the postmaster field is negative. */
79
80negative = cache_record->result != ccache_accept ||
81 (type[0] == 'd' && cache_record->postmaster_result == ccache_reject);
82expire = negative? negative_expire : positive_expire;
83now = time(NULL);
84
85if (now - cache_record->time_stamp > expire)
86 {
6f4d5ad3 87 HDEBUG(D_verify) debug_printf("callout cache: %s record expired for %s\n", type, key);
059ec3d9
PH
88 return NULL;
89 }
90
91/* If this is a non-reject domain record, check for the obsolete format version
92that doesn't have the postmaster and random timestamps, by looking at the
93length. If so, copy it to a new-style block, replicating the record's
94timestamp. Then check the additional timestamps. (There's no point wasting
95effort if connections are rejected.) */
96
97if (type[0] == 'd' && cache_record->result != ccache_reject)
98 {
99 if (length == sizeof(dbdata_callout_cache_obs))
100 {
101 dbdata_callout_cache *new = store_get(sizeof(dbdata_callout_cache));
102 memcpy(new, cache_record, length);
103 new->postmaster_stamp = new->random_stamp = new->time_stamp;
104 cache_record = new;
105 }
106
107 if (now - cache_record->postmaster_stamp > expire)
108 cache_record->postmaster_result = ccache_unknown;
109
110 if (now - cache_record->random_stamp > expire)
111 cache_record->random_result = ccache_unknown;
112 }
113
6f4d5ad3 114HDEBUG(D_verify) debug_printf("callout cache: found %s record for %s\n", type, key);
059ec3d9
PH
115return cache_record;
116}
117
118
119
707ee5b1
JH
120/* Check the callout cache.
121Options * pm_mailfrom may be modified by cache partial results.
059ec3d9 122
707ee5b1 123Return: TRUE if result found
059ec3d9
PH
124*/
125
707ee5b1
JH
126static BOOL
127cached_callout_lookup(address_item * addr, uschar * address_key,
128 uschar * from_address, int * opt_ptr, uschar ** pm_ptr,
129 int * yield, uschar ** failure_ptr,
130 dbdata_callout_cache * new_domain_record, int * old_domain_res)
059ec3d9 131{
707ee5b1 132int options = *opt_ptr;
059ec3d9
PH
133open_db dbblock;
134open_db *dbm_file = NULL;
059ec3d9
PH
135
136/* Open the callout cache database, it it exists, for reading only at this
137stage, unless caching has been disabled. */
138
8b9476ba 139if (options & vopt_callout_no_cache)
059ec3d9
PH
140 {
141 HDEBUG(D_verify) debug_printf("callout cache: disabled by no_cache\n");
142 }
707ee5b1 143else if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE)))
059ec3d9
PH
144 {
145 HDEBUG(D_verify) debug_printf("callout cache: not available\n");
146 }
707ee5b1 147else
059ec3d9 148 {
707ee5b1
JH
149 /* If a cache database is available see if we can avoid the need to do an
150 actual callout by making use of previously-obtained data. */
151
152 dbdata_callout_cache_address * cache_address_record;
153 dbdata_callout_cache * cache_record = get_callout_cache_record(dbm_file,
154 addr->domain, US"domain",
155 callout_cache_domain_positive_expire, callout_cache_domain_negative_expire);
059ec3d9
PH
156
157 /* If an unexpired cache record was found for this domain, see if the callout
158 process can be short-circuited. */
159
ff5929e3 160 if (cache_record)
059ec3d9 161 {
2b1c6e3a
PH
162 /* In most cases, if an early command (up to and including MAIL FROM:<>)
163 was rejected, there is no point carrying on. The callout fails. However, if
164 we are doing a recipient verification with use_sender or use_postmaster
165 set, a previous failure of MAIL FROM:<> doesn't count, because this time we
166 will be using a non-empty sender. We have to remember this situation so as
167 not to disturb the cached domain value if this whole verification succeeds
168 (we don't want it turning into "accept"). */
169
707ee5b1 170 *old_domain_res = cache_record->result;
2b1c6e3a 171
707ee5b1
JH
172 if ( cache_record->result == ccache_reject
173 || *from_address == 0 && cache_record->result == ccache_reject_mfnull)
059ec3d9 174 {
059ec3d9 175 HDEBUG(D_verify)
707ee5b1
JH
176 debug_printf("callout cache: domain gave initial rejection, or "
177 "does not accept HELO or MAIL FROM:<>\n");
059ec3d9
PH
178 setflag(addr, af_verify_nsfail);
179 addr->user_message = US"(result of an earlier callout reused).";
707ee5b1 180 *yield = FAIL;
8e669ac1 181 *failure_ptr = US"mail";
707ee5b1
JH
182 dbfn_close(dbm_file);
183 return TRUE;
059ec3d9
PH
184 }
185
186 /* If a previous check on a "random" local part was accepted, we assume
187 that the server does not do any checking on local parts. There is therefore
188 no point in doing the callout, because it will always be successful. If a
189 random check previously failed, arrange not to do it again, but preserve
190 the data in the new record. If a random check is required but hasn't been
191 done, skip the remaining cache processing. */
192
8b9476ba 193 if (options & vopt_callout_random) switch(cache_record->random_result)
059ec3d9
PH
194 {
195 case ccache_accept:
8b9476ba
JH
196 HDEBUG(D_verify)
197 debug_printf("callout cache: domain accepts random addresses\n");
2ddb4094 198 *failure_ptr = US"random";
707ee5b1
JH
199 dbfn_close(dbm_file);
200 return TRUE; /* Default yield is OK */
059ec3d9
PH
201
202 case ccache_reject:
8b9476ba
JH
203 HDEBUG(D_verify)
204 debug_printf("callout cache: domain rejects random addresses\n");
707ee5b1
JH
205 *opt_ptr = options & ~vopt_callout_random;
206 new_domain_record->random_result = ccache_reject;
207 new_domain_record->random_stamp = cache_record->random_stamp;
8b9476ba 208 break;
059ec3d9
PH
209
210 default:
8b9476ba
JH
211 HDEBUG(D_verify)
212 debug_printf("callout cache: need to check random address handling "
213 "(not cached or cache expired)\n");
707ee5b1
JH
214 dbfn_close(dbm_file);
215 return FALSE;
059ec3d9
PH
216 }
217
218 /* If a postmaster check is requested, but there was a previous failure,
219 there is again no point in carrying on. If a postmaster check is required,
220 but has not been done before, we are going to have to do a callout, so skip
221 remaining cache processing. */
222
707ee5b1 223 if (*pm_ptr)
059ec3d9
PH
224 {
225 if (cache_record->postmaster_result == ccache_reject)
707ee5b1
JH
226 {
227 setflag(addr, af_verify_pmfail);
228 HDEBUG(D_verify)
229 debug_printf("callout cache: domain does not accept "
230 "RCPT TO:<postmaster@domain>\n");
231 *yield = FAIL;
232 *failure_ptr = US"postmaster";
233 setflag(addr, af_verify_pmfail);
234 addr->user_message = US"(result of earlier verification reused).";
235 dbfn_close(dbm_file);
236 return TRUE;
237 }
059ec3d9 238 if (cache_record->postmaster_result == ccache_unknown)
707ee5b1
JH
239 {
240 HDEBUG(D_verify)
241 debug_printf("callout cache: need to check RCPT "
242 "TO:<postmaster@domain> (not cached or cache expired)\n");
243 dbfn_close(dbm_file);
244 return FALSE;
245 }
059ec3d9
PH
246
247 /* If cache says OK, set pm_mailfrom NULL to prevent a redundant
248 postmaster check if the address itself has to be checked. Also ensure
249 that the value in the cache record is preserved (with its old timestamp).
250 */
251
252 HDEBUG(D_verify) debug_printf("callout cache: domain accepts RCPT "
707ee5b1
JH
253 "TO:<postmaster@domain>\n");
254 *pm_ptr = NULL;
255 new_domain_record->postmaster_result = ccache_accept;
256 new_domain_record->postmaster_stamp = cache_record->postmaster_stamp;
059ec3d9
PH
257 }
258 }
259
260 /* We can't give a result based on information about the domain. See if there
261 is an unexpired cache record for this specific address (combined with the
262 sender address if we are doing a recipient callout with a non-empty sender).
263 */
264
707ee5b1
JH
265 if (!(cache_address_record = (dbdata_callout_cache_address *)
266 get_callout_cache_record(dbm_file, address_key, US"address",
267 callout_cache_positive_expire, callout_cache_negative_expire)))
268 {
269 dbfn_close(dbm_file);
270 return FALSE;
271 }
059ec3d9 272
707ee5b1 273 if (cache_address_record->result == ccache_accept)
059ec3d9 274 {
707ee5b1
JH
275 HDEBUG(D_verify)
276 debug_printf("callout cache: address record is positive\n");
277 }
278 else
279 {
280 HDEBUG(D_verify)
281 debug_printf("callout cache: address record is negative\n");
282 addr->user_message = US"Previous (cached) callout verification failure";
283 *failure_ptr = US"recipient";
284 *yield = FAIL;
059ec3d9
PH
285 }
286
287 /* Close the cache database while we actually do the callout for real. */
288
059ec3d9 289 dbfn_close(dbm_file);
707ee5b1
JH
290 return TRUE;
291 }
292return FALSE;
293}
294
295
296/* Write results to callout cache
297*/
298static void
299cache_callout_write(dbdata_callout_cache * dom_rec, const uschar * domain,
300 int done, dbdata_callout_cache_address * addr_rec, uschar * address_key)
301{
302open_db dbblock;
303open_db *dbm_file = NULL;
304
305/* If we get here with done == TRUE, a successful callout happened, and yield
306will be set OK or FAIL according to the response to the RCPT command.
307Otherwise, we looped through the hosts but couldn't complete the business.
308However, there may be domain-specific information to cache in both cases.
309
310The value of the result field in the new_domain record is ccache_unknown if
311there was an error before or with MAIL FROM:, and errno was not zero,
312implying some kind of I/O error. We don't want to write the cache in that case.
313Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */
314
315if (dom_rec->result != ccache_unknown)
316 if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE)))
317 {
318 HDEBUG(D_verify) debug_printf("callout cache: not available\n");
319 }
320 else
321 {
322 (void)dbfn_write(dbm_file, domain, dom_rec,
323 (int)sizeof(dbdata_callout_cache));
324 HDEBUG(D_verify) debug_printf("wrote callout cache domain record for %s:\n"
325 " result=%d postmaster=%d random=%d\n",
326 domain,
327 dom_rec->result,
328 dom_rec->postmaster_result,
329 dom_rec->random_result);
330 }
331
332/* If a definite result was obtained for the callout, cache it unless caching
333is disabled. */
334
335if (done && addr_rec->result != ccache_unknown)
336 {
337 if (!dbm_file)
338 dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE);
339 if (!dbm_file)
340 {
341 HDEBUG(D_verify) debug_printf("no callout cache available\n");
342 }
343 else
344 {
345 (void)dbfn_write(dbm_file, address_key, addr_rec,
346 (int)sizeof(dbdata_callout_cache_address));
347 HDEBUG(D_verify) debug_printf("wrote %s callout cache address record for %s\n",
348 addr_rec->result == ccache_accept ? "positive" : "negative",
349 address_key);
350 }
351 }
352
353if (dbm_file) dbfn_close(dbm_file);
354}
355
356
c4c940fd
JH
357/* Cutthrough-multi. If the existing cached cutthrough connection matches
358the one we would make for a subsequent recipient, use it. Send the RCPT TO
359and check the result, nonpipelined as it may be wanted immediately for
360recipient-verification.
361
362It seems simpler to deal with this case separately from the main callout loop.
363We will need to remember it has sent, or not, so that rcpt-acl tail code
364can do it there for the non-rcpt-verify case. For this we keep an addresscount.
365
366Return: TRUE for a definitive result for the recipient
367*/
368static int
369cutthrough_multi(address_item * addr, host_item * host_list,
370 transport_feedback * tf, int * yield)
371{
372BOOL done = FALSE;
373host_item * host;
374
375if (addr->transport == cutthrough.addr.transport)
376 for (host = host_list; host; host = host->next)
377 if (Ustrcmp(host->address, cutthrough.host.address) == 0)
378 {
379 int host_af;
380 uschar *interface = NULL; /* Outgoing interface to use; NULL => any */
381 int port = 25;
382
383 deliver_host = host->name;
384 deliver_host_address = host->address;
385 deliver_host_port = host->port;
386 deliver_domain = addr->domain;
387 transport_name = addr->transport->name;
388
57cc2785 389 host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
c4c940fd 390
f83a760f
JH
391 if ( !smtp_get_interface(tf->interface, host_af, addr, &interface,
392 US"callout")
393 || !smtp_get_port(tf->port, addr, &port, US"callout")
394 )
c4c940fd
JH
395 log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
396 addr->message);
397
f83a760f
JH
398 smtp_port_for_connect(host, port);
399
c4c940fd
JH
400 if ( ( interface == cutthrough.interface
401 || ( interface
402 && cutthrough.interface
403 && Ustrcmp(interface, cutthrough.interface) == 0
404 ) )
f83a760f 405 && host->port == cutthrough.host.port
c4c940fd
JH
406 )
407 {
408 uschar * resp = NULL;
409
410 /* Match! Send the RCPT TO, set done from the response */
411 done =
74f1a423
JH
412 smtp_write_command(&ctblock, SCMD_FLUSH, "RCPT TO:<%.1000s>\r\n",
413 transport_rcpt_address(addr,
414 addr->transport->rcpt_include_affixes)) >= 0
415 && cutthrough_response(&cutthrough.cctx, '2', &resp,
416 CUTTHROUGH_DATA_TIMEOUT) == '2';
c4c940fd
JH
417
418 /* This would go horribly wrong if a callout fail was ignored by ACL.
419 We punt by abandoning cutthrough on a reject, like the
420 first-rcpt does. */
421
422 if (done)
423 {
424 address_item * na = store_get(sizeof(address_item));
425 *na = cutthrough.addr;
426 cutthrough.addr = *addr;
427 cutthrough.addr.host_used = &cutthrough.host;
428 cutthrough.addr.next = na;
429
430 cutthrough.nrcpt++;
431 }
432 else
433 {
57cc2785 434 cancel_cutthrough_connection(TRUE, US"recipient rejected");
c4c940fd
JH
435 if (!resp || errno == ETIMEDOUT)
436 {
437 HDEBUG(D_verify) debug_printf("SMTP timeout\n");
438 }
439 else if (errno == 0)
440 {
441 if (*resp == 0)
442 Ustrcpy(resp, US"connection dropped");
443
444 addr->message =
e9166683
JH
445 string_sprintf("response to \"%s\" was: %s",
446 big_buffer, string_printing(resp));
c4c940fd
JH
447
448 addr->user_message =
449 string_sprintf("Callout verification failed:\n%s", resp);
450
451 /* Hard rejection ends the process */
452
453 if (resp[0] == '5') /* Address rejected */
454 {
455 *yield = FAIL;
456 done = TRUE;
457 }
458 }
459 }
460 }
461 break; /* host_list */
462 }
463if (!done)
57cc2785 464 cancel_cutthrough_connection(TRUE, US"incompatible connection");
c4c940fd
JH
465return done;
466}
467
468
707ee5b1
JH
469/*************************************************
470* Do callout verification for an address *
471*************************************************/
472
473/* This function is called from verify_address() when the address has routed to
474a host list, and a callout has been requested. Callouts are expensive; that is
475why a cache is used to improve the efficiency.
476
477Arguments:
478 addr the address that's been routed
479 host_list the list of hosts to try
480 tf the transport feedback block
481
482 ifstring "interface" option from transport, or NULL
483 portstring "port" option from transport, or NULL
484 protocolstring "protocol" option from transport, or NULL
485 callout the per-command callout timeout
486 callout_overall the overall callout timeout (if < 0 use 4*callout)
487 callout_connect the callout connection timeout (if < 0 use callout)
488 options the verification options - these bits are used:
489 vopt_is_recipient => this is a recipient address
490 vopt_callout_no_cache => don't use callout cache
491 vopt_callout_fullpm => if postmaster check, do full one
492 vopt_callout_random => do the "random" thing
493 vopt_callout_recipsender => use real sender for recipient
494 vopt_callout_recippmaster => use postmaster for recipient
57cc2785 495 vopt_callout_hold => lazy close connection
707ee5b1
JH
496 se_mailfrom MAIL FROM address for sender verify; NULL => ""
497 pm_mailfrom if non-NULL, do the postmaster check with this sender
498
499Returns: OK/FAIL/DEFER
500*/
501
502static int
503do_callout(address_item *addr, host_item *host_list, transport_feedback *tf,
504 int callout, int callout_overall, int callout_connect, int options,
505 uschar *se_mailfrom, uschar *pm_mailfrom)
506{
507int yield = OK;
508int old_domain_cache_result = ccache_accept;
509BOOL done = FALSE;
510uschar *address_key;
511uschar *from_address;
512uschar *random_local_part = NULL;
513const uschar *save_deliver_domain = deliver_domain;
514uschar **failure_ptr = options & vopt_is_recipient
515 ? &recipient_verify_failure : &sender_verify_failure;
516dbdata_callout_cache new_domain_record;
517dbdata_callout_cache_address new_address_record;
707ee5b1
JH
518time_t callout_start_time;
519
520new_domain_record.result = ccache_unknown;
521new_domain_record.postmaster_result = ccache_unknown;
522new_domain_record.random_result = ccache_unknown;
523
524memset(&new_address_record, 0, sizeof(new_address_record));
525
526/* For a recipient callout, the key used for the address cache record must
527include the sender address if we are using the real sender in the callout,
528because that may influence the result of the callout. */
529
530if (options & vopt_is_recipient)
531 if (options & vopt_callout_recipsender)
532 {
533 from_address = sender_address;
534 address_key = string_sprintf("%s/<%s>", addr->address, sender_address);
535 if (cutthrough.delivery) options |= vopt_callout_no_cache;
536 }
537 else if (options & vopt_callout_recippmaster)
538 {
539 from_address = string_sprintf("postmaster@%s", qualify_domain_sender);
540 address_key = string_sprintf("%s/<postmaster@%s>", addr->address,
541 qualify_domain_sender);
542 }
543 else
544 {
545 from_address = US"";
546 address_key = addr->address;
547 }
548
549/* For a sender callout, we must adjust the key if the mailfrom address is not
550empty. */
551
552else
553 {
554 from_address = se_mailfrom ? se_mailfrom : US"";
555 address_key = *from_address
556 ? string_sprintf("%s/<%s>", addr->address, from_address) : addr->address;
059ec3d9
PH
557 }
558
707ee5b1
JH
559if (cached_callout_lookup(addr, address_key, from_address,
560 &options, &pm_mailfrom, &yield, failure_ptr,
561 &new_domain_record, &old_domain_cache_result))
57cc2785
JH
562 {
563 cancel_cutthrough_connection(TRUE, US"cache-hit");
707ee5b1 564 goto END_CALLOUT;
57cc2785 565 }
707ee5b1 566
193e3acd 567if (!addr->transport)
059ec3d9 568 {
193e3acd 569 HDEBUG(D_verify) debug_printf("cannot callout via null transport\n");
059ec3d9 570 }
6681531a
HSHR
571else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0)
572 log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp",
573 addr->transport->name, addr->transport->driver_name);
193e3acd
JH
574else
575 {
576 smtp_transport_options_block *ob =
9d9c3746 577 (smtp_transport_options_block *)addr->transport->options_block;
c4c940fd 578 host_item * host;
059ec3d9 579
193e3acd
JH
580 /* The information wasn't available in the cache, so we have to do a real
581 callout and save the result in the cache for next time, unless no_cache is set,
582 or unless we have a previously cached negative random result. If we are to test
583 with a random local part, ensure that such a local part is available. If not,
4c04137d 584 log the fact, but carry on without randomising. */
059ec3d9 585
707ee5b1 586 if (options & vopt_callout_random && callout_random_local_part)
65f1c92a 587 if (!(random_local_part = expand_string(callout_random_local_part)))
193e3acd
JH
588 log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand "
589 "callout_random_local_part: %s", expand_string_message);
059ec3d9 590
193e3acd
JH
591 /* Default the connect and overall callout timeouts if not set, and record the
592 time we are starting so that we can enforce it. */
4c590bd1 593
193e3acd
JH
594 if (callout_overall < 0) callout_overall = 4 * callout;
595 if (callout_connect < 0) callout_connect = callout;
596 callout_start_time = time(NULL);
4c590bd1 597
193e3acd
JH
598 /* Before doing a real callout, if this is an SMTP connection, flush the SMTP
599 output because a callout might take some time. When PIPELINING is active and
600 there are many recipients, the total time for doing lots of callouts can add up
601 and cause the client to time out. So in this case we forgo the PIPELINING
602 optimization. */
817d9f57 603
ff5929e3 604 if (smtp_out && !disable_callout_flush) mac_smtp_fflush();
059ec3d9 605
c4c940fd
JH
606 clearflag(addr, af_verify_pmfail); /* postmaster callout flag */
607 clearflag(addr, af_verify_nsfail); /* null sender callout flag */
608
5032d1cf
JH
609/* cutthrough-multi: if a nonfirst rcpt has the same routing as the first,
610and we are holding a cutthrough conn open, we can just append the rcpt to
611that conn for verification purposes (and later delivery also). Simplest
c4c940fd 612coding means skipping this whole loop and doing the append separately. */
5032d1cf
JH
613
614 /* Can we re-use an open cutthrough connection? */
74f1a423 615 if ( cutthrough.cctx.sock >= 0
5032d1cf
JH
616 && (options & (vopt_callout_recipsender | vopt_callout_recippmaster))
617 == vopt_callout_recipsender
618 && !random_local_part
619 && !pm_mailfrom
620 )
c4c940fd 621 done = cutthrough_multi(addr, host_list, tf, &yield);
5032d1cf 622
c4c940fd
JH
623 /* If we did not use a cached connection, make connections to the hosts
624 and do real callouts. The list of hosts is passed in as an argument. */
059ec3d9 625
ff5929e3 626 for (host = host_list; host && !done; host = host->next)
059ec3d9 627 {
193e3acd
JH
628 int host_af;
629 int port = 25;
193e3acd 630 uschar *interface = NULL; /* Outgoing interface to use; NULL => any */
02b41d71 631 smtp_context sx;
193e3acd 632
ff5929e3 633 if (!host->address)
193e3acd
JH
634 {
635 DEBUG(D_verify) debug_printf("no IP address for host name %s: skipping\n",
636 host->name);
637 continue;
638 }
059ec3d9 639
193e3acd 640 /* Check the overall callout timeout */
059ec3d9 641
193e3acd
JH
642 if (time(NULL) - callout_start_time >= callout_overall)
643 {
644 HDEBUG(D_verify) debug_printf("overall timeout for callout exceeded\n");
645 break;
646 }
059ec3d9 647
193e3acd 648 /* Set IPv4 or IPv6 */
059ec3d9 649
e9166683 650 host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
de3a88fb 651
193e3acd
JH
652 /* Expand and interpret the interface and port strings. The latter will not
653 be used if there is a host-specific port (e.g. from a manualroute router).
654 This has to be delayed till now, because they may expand differently for
655 different hosts. If there's a failure, log it, but carry on with the
656 defaults. */
de3a88fb 657
193e3acd
JH
658 deliver_host = host->name;
659 deliver_host_address = host->address;
a7538db1 660 deliver_host_port = host->port;
193e3acd 661 deliver_domain = addr->domain;
aec45841 662 transport_name = addr->transport->name;
059ec3d9 663
6f6dedcc 664 if ( !smtp_get_interface(tf->interface, host_af, addr, &interface,
bf7aabb4
JH
665 US"callout")
666 || !smtp_get_port(tf->port, addr, &port, US"callout")
667 )
193e3acd
JH
668 log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
669 addr->message);
059ec3d9 670
02b41d71
JH
671 sx.addrlist = addr;
672 sx.host = host;
673 sx.host_af = host_af,
674 sx.port = port;
675 sx.interface = interface;
676 sx.helo_data = tf->helo_data;
677 sx.tblock = addr->transport;
e9166683 678 sx.verify = TRUE;
02b41d71
JH
679
680tls_retry_connection:
681 /* Set the address state so that errors are recorded in it */
682
683 addr->transport_return = PENDING_DEFER;
684 ob->connect_timeout = callout_connect;
685 ob->command_timeout = callout;
686
687 /* Get the channel set up ready for a message (MAIL FROM being the next
688 SMTP command to send. If we tried TLS but it failed, try again without
689 if permitted */
690
f10e3ea3
JH
691 yield = smtp_setup_conn(&sx, FALSE);
692#ifdef SUPPORT_TLS
693 if ( yield == DEFER
02b41d71
JH
694 && addr->basic_errno == ERRNO_TLSFAILURE
695 && ob->tls_tempfail_tryclear
696 && verify_check_given_host(&ob->hosts_require_tls, host) != OK
697 )
193e3acd 698 {
cf0c6164
JH
699 log_write(0, LOG_MAIN,
700 "%s: callout unencrypted to %s [%s] (not in hosts_require_tls)",
701 addr->message, host->name, host->address);
e9166683
JH
702 addr->transport_return = PENDING_DEFER;
703 yield = smtp_setup_conn(&sx, TRUE);
02b41d71 704 }
f10e3ea3 705#endif
02b41d71
JH
706 if (yield != OK)
707 {
02b41d71 708 errno = addr->basic_errno;
aec45841 709 transport_name = NULL;
193e3acd
JH
710 deliver_host = deliver_host_address = NULL;
711 deliver_domain = save_deliver_domain;
bf7aabb4 712
02b41d71
JH
713 /* Failure to accept HELO is cached; this blocks the whole domain for all
714 senders. I/O errors and defer responses are not cached. */
770747fd 715
02b41d71 716 if (yield == FAIL && (errno == 0 || errno == ERRNO_SMTPCLOSED))
a7538db1 717 {
02b41d71
JH
718 setflag(addr, af_verify_nsfail);
719 new_domain_record.result = ccache_reject;
720 done = TRUE;
a7538db1 721 }
193e3acd 722 else
02b41d71
JH
723 done = FALSE;
724 goto no_conn;
3c8b3577 725 }
9bfc60eb 726
02b41d71
JH
727 /* If we needed to authenticate, smtp_setup_conn() did that. Copy
728 the AUTH info for logging */
fcc8e047 729
02b41d71
JH
730 addr->authenticator = client_authenticator;
731 addr->auth_id = client_authenticated_id;
b4a2b536 732
e9166683
JH
733 sx.from_addr = from_address;
734 sx.first_addr = sx.sync_addr = addr;
735 sx.ok = FALSE; /*XXX these 3 last might not be needed for verify? */
736 sx.send_rset = TRUE;
737 sx.completed_addr = FALSE;
b4a2b536 738
cf0c6164 739 new_domain_record.result = old_domain_cache_result == ccache_reject_mfnull
e9166683 740 ? ccache_reject_mfnull : ccache_accept;
c4c940fd 741
e9166683
JH
742 /* Do the random local part check first. Temporarily replace the recipient
743 with the "random" value */
02b41d71 744
e9166683 745 if (random_local_part)
059ec3d9 746 {
e9166683 747 uschar * main_address = addr->address;
921dfc11
JH
748 const uschar * rcpt_domain = addr->domain;
749
8c5d388a 750#ifdef SUPPORT_I18N
921dfc11
JH
751 uschar * errstr = NULL;
752 if ( testflag(addr, af_utf8_downcvt)
753 && (rcpt_domain = string_domain_utf8_to_alabel(rcpt_domain,
754 &errstr), errstr)
755 )
756 {
757 addr->message = errstr;
758 errno = ERRNO_EXPANDFAIL;
759 setflag(addr, af_verify_nsfail);
760 done = FALSE;
761 rcpt_domain = US""; /*XXX errorhandling! */
762 }
763#endif
764
57cc2785
JH
765 /* This would be ok for 1st rcpt of a cutthrough (the case handled here;
766 subsequents are done in cutthrough_multi()), but no way to
767 handle a subsequent because of the RSET vaporising the MAIL FROM.
768 So refuse to support any. Most cutthrough use will not involve
769 random_local_part, so no loss. */
770 cancel_cutthrough_connection(TRUE, US"random-recipient");
059ec3d9 771
e9166683
JH
772 addr->address = string_sprintf("%s@%.1000s",
773 random_local_part, rcpt_domain);
774 done = FALSE;
902fbd69
JH
775
776 /* If accepted, we aren't going to do any further tests below.
777 Otherwise, cache a real negative response, and get back to the right
778 state to send RCPT. Unless there's some problem such as a dropped
779 connection, we expect to succeed, because the commands succeeded above.
780 However, some servers drop the connection after responding to an
781 invalid recipient, so on (any) error we drop and remake the connection.
782 XXX We don't care about that for postmaster_full. Should we?
783
784 XXX could we add another flag to the context, and have the common
785 code emit the RSET too? Even pipelined after the RCPT...
786 Then the main-verify call could use it if there's to be a subsequent
787 postmaster-verify.
788 The sync_responses() would need to be taught about it and we'd
789 need another return code filtering out to here.
14de8063 790
d6e7df90 791 Avoid using a SIZE option on the MAIL for all random-rcpt checks.
902fbd69
JH
792 */
793
14de8063
JH
794 sx.avoid_option = OPTION_SIZE;
795
a65c4156
JH
796 /* Remember when we last did a random test */
797 new_domain_record.random_stamp = time(NULL);
798
e9166683
JH
799 if (smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0)
800 switch(addr->transport_return)
801 {
b6323c75 802 case PENDING_OK: /* random was accepted, unfortunately */
e9166683 803 new_domain_record.random_result = ccache_accept;
b6323c75 804 yield = OK; /* Only usable verify result we can return */
fae8970d 805 done = TRUE;
2ddb4094 806 *failure_ptr = US"random";
fae8970d 807 goto no_conn;
b6323c75 808 case FAIL: /* rejected: the preferred result */
e9166683 809 new_domain_record.random_result = ccache_reject;
14de8063 810 sx.avoid_option = 0;
193e3acd 811
e9166683 812 /* Between each check, issue RSET, because some servers accept only
902fbd69
JH
813 one recipient after MAIL FROM:<>.
814 XXX We don't care about that for postmaster_full. Should we? */
193e3acd 815
e9166683 816 if ((done =
4e910c01 817 smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0 &&
e9166683
JH
818 smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
819 '2', callout)))
820 break;
65f1c92a 821
65f1c92a 822 HDEBUG(D_acl|D_v)
e1d04f48 823 debug_printf_indent("problem after random/rset/mfrom; reopen conn\n");
65f1c92a
JH
824 random_local_part = NULL;
825#ifdef SUPPORT_TLS
74f1a423 826 tls_close(sx.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
65f1c92a 827#endif
e1d04f48 828 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
74f1a423
JH
829 (void)close(sx.cctx.sock);
830 sx.cctx.sock = -1;
0cbf2b82 831#ifndef DISABLE_EVENT
65f1c92a
JH
832 (void) event_raise(addr->transport->event_action,
833 US"tcp:close", NULL);
834#endif
a65c4156
JH
835 addr->address = main_address;
836 addr->transport_return = PENDING_DEFER;
837 sx.first_addr = sx.sync_addr = addr;
838 sx.ok = FALSE;
839 sx.send_rset = TRUE;
840 sx.completed_addr = FALSE;
65f1c92a 841 goto tls_retry_connection;
b6323c75
JH
842 case DEFER: /* 4xx response to random */
843 break; /* Just to be clear. ccache_unknown, !done. */
e9166683 844 }
059ec3d9 845
e9166683
JH
846 /* Re-setup for main verify, or for the error message when failing */
847 addr->address = main_address;
848 addr->transport_return = PENDING_DEFER;
849 sx.first_addr = sx.sync_addr = addr;
850 sx.ok = FALSE;
851 sx.send_rset = TRUE;
852 sx.completed_addr = FALSE;
853 }
854 else
855 done = TRUE;
921dfc11 856
14de8063
JH
857 /* Main verify. For rcpt-verify use SIZE if we know it and we're not cacheing;
858 for sndr-verify never use it. */
e9166683
JH
859
860 if (done)
861 {
14de8063
JH
862 if (!(options & vopt_is_recipient && options & vopt_callout_no_cache))
863 sx.avoid_option = OPTION_SIZE;
864
e9166683
JH
865 done = FALSE;
866 switch(smtp_write_mail_and_rcpt_cmds(&sx, &yield))
867 {
868 case 0: switch(addr->transport_return) /* ok so far */
869 {
870 case PENDING_OK: done = TRUE;
871 new_address_record.result = ccache_accept;
872 break;
14de8063 873 case FAIL: done = TRUE;
e9166683
JH
874 yield = FAIL;
875 *failure_ptr = US"recipient";
876 new_address_record.result = ccache_reject;
877 break;
14de8063 878 default: break;
e9166683
JH
879 }
880 break;
881
882 case -1: /* MAIL response error */
883 *failure_ptr = US"mail";
884 if (errno == 0 && sx.buffer[0] == '5')
885 {
886 setflag(addr, af_verify_nsfail);
887 if (from_address[0] == 0)
888 new_domain_record.result = ccache_reject_mfnull;
889 }
890 break;
891 /* non-MAIL read i/o error */
892 /* non-MAIL response timeout */
893 /* internal error; channel still usable */
894 default: break; /* transmit failed */
895 }
896 }
897
898 addr->auth_sndr = client_authenticated_sender;
899
900 deliver_host = deliver_host_address = NULL;
901 deliver_domain = save_deliver_domain;
902
903 /* Do postmaster check if requested; if a full check is required, we
904 check for RCPT TO:<postmaster> (no domain) in accordance with RFC 821. */
905
906 if (done && pm_mailfrom)
907 {
908 /* Could possibly shift before main verify, just above, and be ok
909 for cutthrough. But no way to handle a subsequent rcpt, so just
910 refuse any */
57cc2785 911 cancel_cutthrough_connection(TRUE, US"postmaster verify");
e1d04f48 912 HDEBUG(D_acl|D_v) debug_printf_indent("Cutthrough cancelled by presence of postmaster verify\n");
e9166683 913
4e910c01 914 done = smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0
e9166683
JH
915 && smtp_read_response(&sx.inblock, sx.buffer,
916 sizeof(sx.buffer), '2', callout);
917
918 if (done)
919 {
920 uschar * main_address = addr->address;
921
922 /*XXX oops, affixes */
923 addr->address = string_sprintf("postmaster@%.1000s", addr->domain);
924 addr->transport_return = PENDING_DEFER;
925
926 sx.from_addr = pm_mailfrom;
927 sx.first_addr = sx.sync_addr = addr;
928 sx.ok = FALSE;
929 sx.send_rset = TRUE;
930 sx.completed_addr = FALSE;
14de8063 931 sx.avoid_option = OPTION_SIZE;
e9166683
JH
932
933 if( smtp_write_mail_and_rcpt_cmds(&sx, &yield) == 0
934 && addr->transport_return == PENDING_OK
935 )
936 done = TRUE;
921dfc11 937 else
e9166683 938 done = (options & vopt_callout_fullpm) != 0
4e910c01 939 && smtp_write_command(&sx.outblock, SCMD_FLUSH,
e9166683
JH
940 "RCPT TO:<postmaster>\r\n") >= 0
941 && smtp_read_response(&sx.inblock, sx.buffer,
942 sizeof(sx.buffer), '2', callout);
921dfc11 943
e9166683 944 /* Sort out the cache record */
2a4be8f9 945
e9166683
JH
946 new_domain_record.postmaster_stamp = time(NULL);
947
948 if (done)
949 new_domain_record.postmaster_result = ccache_accept;
950 else if (errno == 0 && sx.buffer[0] == '5')
951 {
952 *failure_ptr = US"postmaster";
953 setflag(addr, af_verify_pmfail);
954 new_domain_record.postmaster_result = ccache_reject;
955 }
956
957 addr->address = main_address;
958 }
959 }
193e3acd
JH
960 /* For any failure of the main check, other than a negative response, we just
961 close the connection and carry on. We can identify a negative response by the
962 fact that errno is zero. For I/O errors it will be non-zero
2a4be8f9 963
193e3acd
JH
964 Set up different error texts for logging and for sending back to the caller
965 as an SMTP response. Log in all cases, using a one-line format. For sender
966 callouts, give a full response to the caller, but for recipient callouts,
967 don't give the IP address because this may be an internal host whose identity
968 is not to be widely broadcast. */
2a4be8f9 969
02b41d71 970no_conn:
e9166683 971 switch(errno)
193e3acd 972 {
02b41d71
JH
973 case ETIMEDOUT:
974 HDEBUG(D_verify) debug_printf("SMTP timeout\n");
975 sx.send_quit = FALSE;
976 break;
977
8c5d388a 978#ifdef SUPPORT_I18N
02b41d71 979 case ERRNO_UTF8_FWD:
9bfc60eb
JH
980 {
981 extern int acl_where; /* src/acl.c */
982 errno = 0;
983 addr->message = string_sprintf(
e9166683 984 "response to \"EHLO\" did not include SMTPUTF8");
02b41d71
JH
985 addr->user_message = acl_where == ACL_WHERE_RCPT
986 ? US"533 no support for internationalised mailbox name"
9bfc60eb
JH
987 : US"550 mailbox unavailable";
988 yield = FAIL;
989 done = TRUE;
990 }
02b41d71 991 break;
9bfc60eb 992#endif
8ac90765
JH
993#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS)
994 case ERRNO_REQUIRETLS:
995 addr->user_message = US"530 5.7.4 REQUIRETLS support required";
996 yield = FAIL;
997 done = TRUE;
998 break;
999#endif
02b41d71
JH
1000 case ECONNREFUSED:
1001 sx.send_quit = FALSE;
1002 break;
2a4be8f9 1003
02b41d71 1004 case 0:
e9166683 1005 if (*sx.buffer == 0) Ustrcpy(sx.buffer, US"connection dropped");
059ec3d9 1006
e9166683 1007 /*XXX test here is ugly; seem to have a split of responsibility for
b6323c75 1008 building this message. Need to rationalise. Where is it done
e9166683
JH
1009 before here, and when not?
1010 Not == 5xx resp to MAIL on main-verify
1011 */
1012 if (!addr->message) addr->message =
1013 string_sprintf("response to \"%s\" was: %s",
1014 big_buffer, string_printing(sx.buffer));
02b41d71
JH
1015
1016 addr->user_message = options & vopt_is_recipient
e9166683 1017 ? string_sprintf("Callout verification failed:\n%s", sx.buffer)
02b41d71 1018 : string_sprintf("Called: %s\nSent: %s\nResponse: %s",
e9166683 1019 host->address, big_buffer, sx.buffer);
059ec3d9 1020
02b41d71 1021 /* Hard rejection ends the process */
193e3acd 1022
e9166683 1023 if (sx.buffer[0] == '5') /* Address rejected */
02b41d71
JH
1024 {
1025 yield = FAIL;
1026 done = TRUE;
1027 }
1028 break;
193e3acd 1029 }
059ec3d9 1030
193e3acd
JH
1031 /* End the SMTP conversation and close the connection. */
1032
4c04137d 1033 /* Cutthrough - on a successful connect and recipient-verify with
5032d1cf 1034 use-sender and we are 1st rcpt and have no cutthrough conn so far
57cc2785
JH
1035 here is where we want to leave the conn open. Ditto for a lazy-close
1036 verify. */
1037
857eaf37
JH
1038 if (cutthrough.delivery)
1039 {
1040 if (addr->transport->filter_command)
1041 {
1042 cutthrough.delivery= FALSE;
1043 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n");
1044 }
72cb765f
JH
1045#ifndef DISABLE_DKIM
1046 if (ob->dkim.dkim_domain)
857eaf37
JH
1047 {
1048 cutthrough.delivery= FALSE;
72cb765f 1049 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n");
857eaf37 1050 }
72cb765f
JH
1051#endif
1052#ifdef EXPERIMENTAL_ARC
1053 if (ob->arc_sign)
1054 {
1055 cutthrough.delivery= FALSE;
1056 HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of ARC signing\n");
1057 }
1058#endif
857eaf37
JH
1059 }
1060
57cc2785 1061 if ( (cutthrough.delivery || options & vopt_callout_hold)
5032d1cf 1062 && rcpt_count == 1
193e3acd
JH
1063 && done
1064 && yield == OK
98c82a3d
JH
1065 && (options & (vopt_callout_recipsender|vopt_callout_recippmaster|vopt_success_on_redirect))
1066 == vopt_callout_recipsender
193e3acd
JH
1067 && !random_local_part
1068 && !pm_mailfrom
74f1a423 1069 && cutthrough.cctx.sock < 0
02b41d71 1070 && !sx.lmtp
193e3acd 1071 )
059ec3d9 1072 {
8a6b4e02
JH
1073 address_item * parent, * caddr;
1074
57cc2785
JH
1075 HDEBUG(D_acl|D_v) debug_printf_indent("holding verify callout open for %s\n",
1076 cutthrough.delivery
1077 ? "cutthrough delivery" : "potential further verifies and delivery");
1078
1079 cutthrough.callout_hold_only = !cutthrough.delivery;
74f1a423
JH
1080 cutthrough.is_tls = tls_out.active.sock >= 0;
1081 /* We assume no buffer in use in the outblock */
1082 cutthrough.cctx = sx.cctx;
57cc2785
JH
1083 cutthrough.nrcpt = 1;
1084 cutthrough.transport = addr->transport->name;
1085 cutthrough.interface = interface;
1086 cutthrough.snd_port = sending_port;
1087 cutthrough.peer_options = smtp_peer_options;
1088 cutthrough.host = *host;
1089 {
1090 int oldpool = store_pool;
1091 store_pool = POOL_PERM;
1092 cutthrough.snd_ip = string_copy(sending_ip_address);
1093 cutthrough.host.name = string_copy(host->name);
1094 cutthrough.host.address = string_copy(host->address);
1095 store_pool = oldpool;
1096 }
8a6b4e02
JH
1097
1098 /* Save the address_item and parent chain for later logging */
1099 cutthrough.addr = *addr;
57cc2785 1100 cutthrough.addr.next = NULL;
5032d1cf 1101 cutthrough.addr.host_used = &cutthrough.host;
8a6b4e02
JH
1102 for (caddr = &cutthrough.addr, parent = addr->parent;
1103 parent;
1104 parent = parent->parent)
1105 *(caddr->parent = store_get(sizeof(address_item))) = *parent;
1106
193e3acd
JH
1107 ctblock.buffer = ctbuffer;
1108 ctblock.buffersize = sizeof(ctbuffer);
1109 ctblock.ptr = ctbuffer;
1110 /* ctblock.cmd_count = 0; ctblock.authenticating = FALSE; */
74f1a423 1111 ctblock.cctx = &cutthrough.cctx;
059ec3d9 1112 }
193e3acd 1113 else
059ec3d9 1114 {
57cc2785 1115 /* Ensure no cutthrough on multiple verifies that were incompatible */
193e3acd 1116 if (options & vopt_callout_recipsender)
57cc2785 1117 cancel_cutthrough_connection(TRUE, US"not usable for cutthrough");
02b41d71 1118 if (sx.send_quit)
2760b518 1119 {
4e910c01 1120 (void) smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n");
059ec3d9 1121
2760b518 1122 /* Wait a short time for response, and discard it */
e9166683 1123 smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
2760b518
JH
1124 '2', 1);
1125 }
1126
74f1a423 1127 if (sx.cctx.sock >= 0)
02b41d71 1128 {
a7538db1 1129#ifdef SUPPORT_TLS
74f1a423
JH
1130 if (sx.cctx.tls_ctx)
1131 {
1132 tls_close(sx.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
1133 sx.cctx.tls_ctx = NULL;
1134 }
a7538db1 1135#endif
e1d04f48 1136 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
74f1a423
JH
1137 (void)close(sx.cctx.sock);
1138 sx.cctx.sock = -1;
0cbf2b82 1139#ifndef DISABLE_EVENT
02b41d71 1140 (void) event_raise(addr->transport->event_action, US"tcp:close", NULL);
a7538db1 1141#endif
02b41d71 1142 }
059ec3d9 1143 }
059ec3d9 1144
e9166683
JH
1145 if (!done || yield != OK)
1146 addr->message = string_sprintf("%s [%s] : %s", host->name, host->address,
1147 addr->message);
193e3acd
JH
1148 } /* Loop through all hosts, while !done */
1149 }
059ec3d9
PH
1150
1151/* If we get here with done == TRUE, a successful callout happened, and yield
1152will be set OK or FAIL according to the response to the RCPT command.
1153Otherwise, we looped through the hosts but couldn't complete the business.
707ee5b1 1154However, there may be domain-specific information to cache in both cases. */
059ec3d9 1155
707ee5b1
JH
1156if (!(options & vopt_callout_no_cache))
1157 cache_callout_write(&new_domain_record, addr->domain,
1158 done, &new_address_record, address_key);
059ec3d9
PH
1159
1160/* Failure to connect to any host, or any response other than 2xx or 5xx is a
1161temporary error. If there was only one host, and a response was received, leave
1162it alone if supplying details. Otherwise, give a generic response. */
1163
707ee5b1 1164if (!done)
059ec3d9 1165 {
ff5929e3 1166 uschar * dullmsg = string_sprintf("Could not complete %s verify callout",
8b9476ba 1167 options & vopt_is_recipient ? "recipient" : "sender");
059ec3d9
PH
1168 yield = DEFER;
1169
02b41d71
JH
1170 addr->message = host_list->next || !addr->message
1171 ? dullmsg : string_sprintf("%s: %s", dullmsg, addr->message);
059ec3d9 1172
ff5929e3
JH
1173 addr->user_message = smtp_return_error_details
1174 ? string_sprintf("%s for <%s>.\n"
059ec3d9
PH
1175 "The mail server(s) for the domain may be temporarily unreachable, or\n"
1176 "they may be permanently unreachable from this server. In the latter case,\n%s",
1177 dullmsg, addr->address,
8b9476ba 1178 options & vopt_is_recipient
ff5929e3 1179 ? "the address will never be accepted."
8b9476ba
JH
1180 : "you need to change the address or create an MX record for its domain\n"
1181 "if it is supposed to be generally accessible from the Internet.\n"
ff5929e3
JH
1182 "Talk to your mail administrator for details.")
1183 : dullmsg;
059ec3d9
PH
1184
1185 /* Force a specific error code */
1186
1187 addr->basic_errno = ERRNO_CALLOUTDEFER;
1188 }
1189
1190/* Come here from within the cache-reading code on fast-track exit. */
1191
1192END_CALLOUT:
02b41d71 1193tls_modify_variables(&tls_in);
059ec3d9
PH
1194return yield;
1195}
1196
1197
1198
817d9f57
JH
1199/* Called after recipient-acl to get a cutthrough connection open when
1200 one was requested and a recipient-verify wasn't subsequently done.
1201*/
f9334a28 1202int
8a6b4e02 1203open_cutthrough_connection(address_item * addr)
e4bdf652
JH
1204{
1205address_item addr2;
f9334a28 1206int rc;
e4bdf652
JH
1207
1208/* Use a recipient-verify-callout to set up the cutthrough connection. */
1209/* We must use a copy of the address for verification, because it might
1210get rewritten. */
1211
1212addr2 = *addr;
e1d04f48 1213HDEBUG(D_acl) debug_printf_indent("----------- %s cutthrough setup ------------\n",
5032d1cf 1214 rcpt_count > 1 ? "more" : "start");
ff5929e3 1215rc = verify_address(&addr2, NULL,
e4bdf652
JH
1216 vopt_is_recipient | vopt_callout_recipsender | vopt_callout_no_cache,
1217 CUTTHROUGH_CMD_TIMEOUT, -1, -1,
1218 NULL, NULL, NULL);
ff5929e3
JH
1219addr->message = addr2.message;
1220addr->user_message = addr2.user_message;
e1d04f48 1221HDEBUG(D_acl) debug_printf_indent("----------- end cutthrough setup ------------\n");
f9334a28 1222return rc;
e4bdf652
JH
1223}
1224
1225
e4bdf652 1226
817d9f57
JH
1227/* Send given number of bytes from the buffer */
1228static BOOL
1229cutthrough_send(int n)
e4bdf652 1230{
74f1a423 1231if(cutthrough.cctx.sock < 0)
817d9f57 1232 return TRUE;
e4bdf652 1233
817d9f57
JH
1234if(
1235#ifdef SUPPORT_TLS
74f1a423
JH
1236 cutthrough.is_tls
1237 ? tls_write(cutthrough.cctx.tls_ctx, ctblock.buffer, n, FALSE)
1238 :
817d9f57 1239#endif
74f1a423 1240 send(cutthrough.cctx.sock, ctblock.buffer, n, 0) > 0
817d9f57
JH
1241 )
1242{
1243 transport_count += n;
1244 ctblock.ptr= ctblock.buffer;
1245 return TRUE;
1246}
e4bdf652 1247
e1d04f48 1248HDEBUG(D_transport|D_acl) debug_printf_indent("cutthrough_send failed: %s\n", strerror(errno));
817d9f57 1249return FALSE;
e4bdf652
JH
1250}
1251
1252
1253
817d9f57
JH
1254static BOOL
1255_cutthrough_puts(uschar * cp, int n)
1256{
1257while(n--)
1258 {
1259 if(ctblock.ptr >= ctblock.buffer+ctblock.buffersize)
1260 if(!cutthrough_send(ctblock.buffersize))
1261 return FALSE;
1262
1263 *ctblock.ptr++ = *cp++;
1264 }
1265return TRUE;
1266}
1267
1268/* Buffered output of counted data block. Return boolean success */
57cc2785 1269static BOOL
e4bdf652
JH
1270cutthrough_puts(uschar * cp, int n)
1271{
74f1a423
JH
1272if (cutthrough.cctx.sock < 0) return TRUE;
1273if (_cutthrough_puts(cp, n)) return TRUE;
57cc2785 1274cancel_cutthrough_connection(TRUE, US"transmit failed");
817d9f57
JH
1275return FALSE;
1276}
e4bdf652 1277
6851a9c5 1278void
57cc2785
JH
1279cutthrough_data_puts(uschar * cp, int n)
1280{
6851a9c5 1281if (cutthrough.delivery) (void) cutthrough_puts(cp, n);
78a3bbd5 1282return;
57cc2785
JH
1283}
1284
e4bdf652 1285
817d9f57 1286static BOOL
5032d1cf 1287_cutthrough_flush_send(void)
817d9f57 1288{
57cc2785 1289int n = ctblock.ptr - ctblock.buffer;
e4bdf652 1290
817d9f57
JH
1291if(n>0)
1292 if(!cutthrough_send(n))
1293 return FALSE;
1294return TRUE;
e4bdf652
JH
1295}
1296
817d9f57
JH
1297
1298/* Send out any bufferred output. Return boolean success. */
e4bdf652 1299BOOL
5032d1cf 1300cutthrough_flush_send(void)
e4bdf652 1301{
817d9f57 1302if (_cutthrough_flush_send()) return TRUE;
57cc2785 1303cancel_cutthrough_connection(TRUE, US"transmit failed");
e4bdf652
JH
1304return FALSE;
1305}
1306
1307
57cc2785 1308static BOOL
5032d1cf 1309cutthrough_put_nl(void)
e4bdf652
JH
1310{
1311return cutthrough_puts(US"\r\n", 2);
1312}
1313
1314
6851a9c5 1315void
57cc2785
JH
1316cutthrough_data_put_nl(void)
1317{
6851a9c5 1318cutthrough_data_puts(US"\r\n", 2);
57cc2785
JH
1319}
1320
1321
e4bdf652
JH
1322/* Get and check response from cutthrough target */
1323static uschar
74f1a423 1324cutthrough_response(client_conn_ctx * cctx, char expect, uschar ** copy, int timeout)
e4bdf652
JH
1325{
1326smtp_inblock inblock;
1327uschar inbuffer[4096];
1328uschar responsebuffer[4096];
1329
1330inblock.buffer = inbuffer;
1331inblock.buffersize = sizeof(inbuffer);
1332inblock.ptr = inbuffer;
1333inblock.ptrend = inbuffer;
74f1a423 1334inblock.cctx = cctx;
2760b518 1335if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, timeout))
57cc2785 1336 cancel_cutthrough_connection(TRUE, US"target timeout on read");
e4bdf652 1337
57cc2785 1338if(copy)
e4bdf652
JH
1339 {
1340 uschar * cp;
5032d1cf 1341 *copy = cp = string_copy(responsebuffer);
e4bdf652
JH
1342 /* Trim the trailing end of line */
1343 cp += Ustrlen(responsebuffer);
1344 if(cp > *copy && cp[-1] == '\n') *--cp = '\0';
1345 if(cp > *copy && cp[-1] == '\r') *--cp = '\0';
1346 }
1347
1348return responsebuffer[0];
1349}
1350
1351
1352/* Negotiate dataphase with the cutthrough target, returning success boolean */
1353BOOL
5032d1cf 1354cutthrough_predata(void)
e4bdf652 1355{
74f1a423 1356if(cutthrough.cctx.sock < 0 || cutthrough.callout_hold_only)
e4bdf652
JH
1357 return FALSE;
1358
e1d04f48 1359HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> DATA\n");
817d9f57
JH
1360cutthrough_puts(US"DATA\r\n", 6);
1361cutthrough_flush_send();
e4bdf652
JH
1362
1363/* Assume nothing buffered. If it was it gets ignored. */
74f1a423 1364return cutthrough_response(&cutthrough.cctx, '3', NULL, CUTTHROUGH_DATA_TIMEOUT) == '3';
e4bdf652
JH
1365}
1366
1367
42055a33 1368/* tctx arg only to match write_chunk() */
511a6c14 1369static BOOL
42055a33 1370cutthrough_write_chunk(transport_ctx * tctx, uschar * s, int len)
511a6c14
JH
1371{
1372uschar * s2;
1373while(s && (s2 = Ustrchr(s, '\n')))
1374 {
1375 if(!cutthrough_puts(s, s2-s) || !cutthrough_put_nl())
1376 return FALSE;
1377 s = s2+1;
1378 }
1379return TRUE;
1380}
1381
1382
e4bdf652 1383/* Buffered send of headers. Return success boolean. */
817d9f57 1384/* Expands newlines to wire format (CR,NL). */
e4bdf652 1385/* Also sends header-terminating blank line. */
e4bdf652 1386BOOL
5032d1cf 1387cutthrough_headers_send(void)
e4bdf652 1388{
6d5c916c
JH
1389transport_ctx tctx;
1390
74f1a423 1391if(cutthrough.cctx.sock < 0 || cutthrough.callout_hold_only)
e4bdf652
JH
1392 return FALSE;
1393
511a6c14
JH
1394/* We share a routine with the mainline transport to handle header add/remove/rewrites,
1395 but having a separate buffered-output function (for now)
1396*/
e1d04f48 1397HDEBUG(D_acl) debug_printf_indent("----------- start cutthrough headers send -----------\n");
e4bdf652 1398
74f1a423 1399tctx.u.fd = cutthrough.cctx.sock;
6d5c916c
JH
1400tctx.tblock = cutthrough.addr.transport;
1401tctx.addr = &cutthrough.addr;
1402tctx.check_string = US".";
1403tctx.escape_string = US"..";
328c5688 1404/*XXX check under spool_files_wireformat. Might be irrelevant */
6d5c916c
JH
1405tctx.options = topt_use_crlf;
1406
42055a33 1407if (!transport_headers_send(&tctx, &cutthrough_write_chunk))
511a6c14
JH
1408 return FALSE;
1409
e1d04f48 1410HDEBUG(D_acl) debug_printf_indent("----------- done cutthrough headers send ------------\n");
511a6c14 1411return TRUE;
817d9f57
JH
1412}
1413
1414
1415static void
78a3bbd5 1416close_cutthrough_connection(const uschar * why)
817d9f57 1417{
74f1a423 1418int fd = cutthrough.cctx.sock;
57cc2785 1419if(fd >= 0)
817d9f57
JH
1420 {
1421 /* We could be sending this after a bunch of data, but that is ok as
1422 the only way to cancel the transfer in dataphase is to drop the tcp
1423 conn before the final dot.
1424 */
74f1a423 1425 client_conn_ctx tmp_ctx = cutthrough.cctx;
817d9f57 1426 ctblock.ptr = ctbuffer;
e1d04f48 1427 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n");
817d9f57
JH
1428 _cutthrough_puts(US"QUIT\r\n", 6); /* avoid recursion */
1429 _cutthrough_flush_send();
74f1a423 1430 cutthrough.cctx.sock = -1; /* avoid recursion via read timeout */
06fdb9f7 1431 cutthrough.nrcpt = 0; /* permit re-cutthrough on subsequent message */
2760b518
JH
1432
1433 /* Wait a short time for response, and discard it */
74f1a423 1434 cutthrough_response(&tmp_ctx, '2', NULL, 1);
817d9f57 1435
57cc2785 1436#ifdef SUPPORT_TLS
74f1a423
JH
1437 if (cutthrough.is_tls)
1438 {
1439 tls_close(cutthrough.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
1440 cutthrough.cctx.tls_ctx = NULL;
1441 cutthrough.is_tls = FALSE;
1442 }
57cc2785 1443#endif
e1d04f48 1444 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
57cc2785 1445 (void)close(fd);
e1d04f48 1446 HDEBUG(D_acl) debug_printf_indent("----------- cutthrough shutdown (%s) ------------\n", why);
817d9f57
JH
1447 }
1448ctblock.ptr = ctbuffer;
e4bdf652
JH
1449}
1450
817d9f57 1451void
57cc2785
JH
1452cancel_cutthrough_connection(BOOL close_noncutthrough_verifies, const uschar * why)
1453{
1454if (cutthrough.delivery || close_noncutthrough_verifies)
1455 close_cutthrough_connection(why);
1456cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
1457}
1458
1459
1460void
1461release_cutthrough_connection(const uschar * why)
817d9f57 1462{
74f1a423 1463if (cutthrough.cctx.sock < 0) return;
57cc2785 1464HDEBUG(D_acl) debug_printf_indent("release cutthrough conn: %s\n", why);
74f1a423
JH
1465cutthrough.cctx.sock = -1;
1466cutthrough.cctx.tls_ctx = NULL;
57cc2785 1467cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
817d9f57
JH
1468}
1469
1470
1471
e4bdf652
JH
1472
1473/* Have senders final-dot. Send one to cutthrough target, and grab the response.
1474 Log an OK response as a transmission.
817d9f57 1475 Close the connection.
e4bdf652 1476 Return smtp response-class digit.
e4bdf652
JH
1477*/
1478uschar *
5032d1cf 1479cutthrough_finaldot(void)
e4bdf652 1480{
5032d1cf
JH
1481uschar res;
1482address_item * addr;
e1d04f48 1483HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> .\n");
e4bdf652
JH
1484
1485/* Assume data finshed with new-line */
5032d1cf
JH
1486if( !cutthrough_puts(US".", 1)
1487 || !cutthrough_put_nl()
1488 || !cutthrough_flush_send()
1489 )
1490 return cutthrough.addr.message;
e4bdf652 1491
74f1a423
JH
1492res = cutthrough_response(&cutthrough.cctx, '2', &cutthrough.addr.message,
1493 CUTTHROUGH_DATA_TIMEOUT);
5032d1cf 1494for (addr = &cutthrough.addr; addr; addr = addr->next)
817d9f57 1495 {
5032d1cf
JH
1496 addr->message = cutthrough.addr.message;
1497 switch(res)
1498 {
1499 case '2':
1500 delivery_log(LOG_MAIN, addr, (int)'>', NULL);
78a3bbd5 1501 close_cutthrough_connection(US"delivered");
5032d1cf 1502 break;
817d9f57 1503
5032d1cf
JH
1504 case '4':
1505 delivery_log(LOG_MAIN, addr, 0,
1506 US"tmp-reject from cutthrough after DATA:");
1507 break;
e4bdf652 1508
5032d1cf
JH
1509 case '5':
1510 delivery_log(LOG_MAIN|LOG_REJECT, addr, 0,
1511 US"rejected after DATA:");
1512 break;
e4bdf652 1513
5032d1cf
JH
1514 default:
1515 break;
1516 }
817d9f57 1517 }
5032d1cf 1518return cutthrough.addr.message;
e4bdf652
JH
1519}
1520
1521
817d9f57 1522
059ec3d9
PH
1523/*************************************************
1524* Copy error to toplevel address *
1525*************************************************/
1526
1527/* This function is used when a verify fails or defers, to ensure that the
1528failure or defer information is in the original toplevel address. This applies
1529when an address is redirected to a single new address, and the failure or
1530deferral happens to the child address.
1531
1532Arguments:
1533 vaddr the verify address item
1534 addr the final address item
1535 yield FAIL or DEFER
1536
1537Returns: the value of YIELD
1538*/
1539
1540static int
1541copy_error(address_item *vaddr, address_item *addr, int yield)
1542{
1543if (addr != vaddr)
1544 {
1545 vaddr->message = addr->message;
1546 vaddr->user_message = addr->user_message;
1547 vaddr->basic_errno = addr->basic_errno;
1548 vaddr->more_errno = addr->more_errno;
d43cbe25 1549 vaddr->prop.address_data = addr->prop.address_data;
42855d71 1550 copyflag(vaddr, addr, af_pass_message);
059ec3d9
PH
1551 }
1552return yield;
1553}
1554
1555
1556
1557
ce552449
NM
1558/**************************************************
1559* printf that automatically handles TLS if needed *
1560***************************************************/
1561
1562/* This function is used by verify_address() as a substitute for all fprintf()
1563calls; a direct fprintf() will not produce output in a TLS SMTP session, such
1564as a response to an EXPN command. smtp_in.c makes smtp_printf available but
1565that assumes that we always use the smtp_out FILE* when not using TLS or the
1566ssl buffer when we are. Instead we take a FILE* parameter and check to see if
1567that is smtp_out; if so, smtp_printf() with TLS support, otherwise regular
1568fprintf().
1569
1570Arguments:
1571 f the candidate FILE* to write to
1572 format format string
1573 ... optional arguments
1574
1575Returns:
1576 nothing
1577*/
1578
1579static void PRINTF_FUNCTION(2,3)
1ba28e2b 1580respond_printf(FILE *f, const char *format, ...)
ce552449
NM
1581{
1582va_list ap;
1583
1584va_start(ap, format);
1585if (smtp_out && (f == smtp_out))
925ac8e4 1586 smtp_vprintf(format, FALSE, ap);
ce552449 1587else
513afc6a 1588 vfprintf(f, format, ap);
ce552449
NM
1589va_end(ap);
1590}
1591
1592
1593
059ec3d9
PH
1594/*************************************************
1595* Verify an email address *
1596*************************************************/
1597
1598/* This function is used both for verification (-bv and at other times) and
1599address testing (-bt), which is indicated by address_test_mode being set.
1600
1601Arguments:
1602 vaddr contains the address to verify; the next field in this block
1603 must be NULL
1604 f if not NULL, write the result to this file
1605 options various option bits:
1606 vopt_fake_sender => this sender verify is not for the real
1607 sender (it was verify=sender=xxxx or an address from a
1608 header line) - rewriting must not change sender_address
1609 vopt_is_recipient => this is a recipient address, otherwise
1610 it's a sender address - this affects qualification and
1611 rewriting and messages from callouts
1612 vopt_qualify => qualify an unqualified address; else error
1613 vopt_expn => called from SMTP EXPN command
eafd343b
TK
1614 vopt_success_on_redirect => when a new address is generated
1615 the verification instantly succeeds
059ec3d9
PH
1616
1617 These ones are used by do_callout() -- the options variable
1618 is passed to it.
1619
2a4be8f9 1620 vopt_callout_fullpm => if postmaster check, do full one
059ec3d9
PH
1621 vopt_callout_no_cache => don't use callout cache
1622 vopt_callout_random => do the "random" thing
1623 vopt_callout_recipsender => use real sender for recipient
1624 vopt_callout_recippmaster => use postmaster for recipient
1625
1626 callout if > 0, specifies that callout is required, and gives timeout
4deaf07d 1627 for individual commands
059ec3d9
PH
1628 callout_overall if > 0, gives overall timeout for the callout function;
1629 if < 0, a default is used (see do_callout())
8e669ac1 1630 callout_connect the connection timeout for callouts
059ec3d9
PH
1631 se_mailfrom when callout is requested to verify a sender, use this
1632 in MAIL FROM; NULL => ""
1633 pm_mailfrom when callout is requested, if non-NULL, do the postmaster
1634 thing and use this as the sender address (may be "")
1635
1636 routed if not NULL, set TRUE if routing succeeded, so we can
1637 distinguish between routing failed and callout failed
1638
1639Returns: OK address verified
1640 FAIL address failed to verify
1641 DEFER can't tell at present
1642*/
1643
1644int
1645verify_address(address_item *vaddr, FILE *f, int options, int callout,
8e669ac1 1646 int callout_overall, int callout_connect, uschar *se_mailfrom,
4deaf07d 1647 uschar *pm_mailfrom, BOOL *routed)
059ec3d9
PH
1648{
1649BOOL allok = TRUE;
1650BOOL full_info = (f == NULL)? FALSE : (debug_selector != 0);
059ec3d9 1651BOOL expn = (options & vopt_expn) != 0;
eafd343b 1652BOOL success_on_redirect = (options & vopt_success_on_redirect) != 0;
059ec3d9
PH
1653int i;
1654int yield = OK;
1655int verify_type = expn? v_expn :
1656 address_test_mode? v_none :
8b9476ba 1657 options & vopt_is_recipient? v_recipient : v_sender;
059ec3d9
PH
1658address_item *addr_list;
1659address_item *addr_new = NULL;
1660address_item *addr_remote = NULL;
1661address_item *addr_local = NULL;
1662address_item *addr_succeed = NULL;
8b9476ba
JH
1663uschar **failure_ptr = options & vopt_is_recipient
1664 ? &recipient_verify_failure : &sender_verify_failure;
059ec3d9
PH
1665uschar *ko_prefix, *cr;
1666uschar *address = vaddr->address;
1667uschar *save_sender;
1668uschar null_sender[] = { 0 }; /* Ensure writeable memory */
1669
2c7db3f5
PH
1670/* Clear, just in case */
1671
1672*failure_ptr = NULL;
1673
059ec3d9
PH
1674/* Set up a prefix and suffix for error message which allow us to use the same
1675output statements both in EXPN mode (where an SMTP response is needed) and when
1676debugging with an output file. */
1677
1678if (expn)
1679 {
1680 ko_prefix = US"553 ";
1681 cr = US"\r";
1682 }
1683else ko_prefix = cr = US"";
1684
1685/* Add qualify domain if permitted; otherwise an unqualified address fails. */
1686
1687if (parse_find_at(address) == NULL)
1688 {
2ddb4094 1689 if (!(options & vopt_qualify))
059ec3d9 1690 {
2ddb4094 1691 if (f)
ce552449
NM
1692 respond_printf(f, "%sA domain is required for \"%s\"%s\n",
1693 ko_prefix, address, cr);
8e669ac1 1694 *failure_ptr = US"qualify";
059ec3d9
PH
1695 return FAIL;
1696 }
8b9476ba 1697 address = rewrite_address_qualify(address, options & vopt_is_recipient);
059ec3d9
PH
1698 }
1699
1700DEBUG(D_verify)
1701 {
1702 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1703 debug_printf("%s %s\n", address_test_mode? "Testing" : "Verifying", address);
1704 }
1705
1706/* Rewrite and report on it. Clear the domain and local part caches - these
1707may have been set by domains and local part tests during an ACL. */
1708
2ddb4094 1709if (global_rewrite_rules)
059ec3d9
PH
1710 {
1711 uschar *old = address;
8b9476ba 1712 address = rewrite_address(address, options & vopt_is_recipient, FALSE,
059ec3d9
PH
1713 global_rewrite_rules, rewrite_existflags);
1714 if (address != old)
1715 {
1716 for (i = 0; i < (MAX_NAMED_LIST * 2)/32; i++) vaddr->localpart_cache[i] = 0;
1717 for (i = 0; i < (MAX_NAMED_LIST * 2)/32; i++) vaddr->domain_cache[i] = 0;
2ddb4094 1718 if (f && !expn) fprintf(f, "Address rewritten as: %s\n", address);
059ec3d9
PH
1719 }
1720 }
1721
1722/* If this is the real sender address, we must update sender_address at
1723this point, because it may be referred to in the routers. */
1724
2ddb4094 1725if (!(options & (vopt_fake_sender|vopt_is_recipient)))
059ec3d9
PH
1726 sender_address = address;
1727
1728/* If the address was rewritten to <> no verification can be done, and we have
1729to return OK. This rewriting is permitted only for sender addresses; for other
1730addresses, such rewriting fails. */
1731
2ddb4094 1732if (!address[0]) return OK;
059ec3d9 1733
d9b2312b
JH
1734/* Flip the legacy TLS-related variables over to the outbound set in case
1735they're used in the context of a transport used by verification. Reset them
ea90b718 1736at exit from this routine (so no returns allowed from here on). */
d9b2312b 1737
35aba663 1738tls_modify_variables(&tls_out);
d9b2312b 1739
059ec3d9
PH
1740/* Save a copy of the sender address for re-instating if we change it to <>
1741while verifying a sender address (a nice bit of self-reference there). */
1742
1743save_sender = sender_address;
1744
ea90b718
JH
1745/* Observability variable for router/transport use */
1746
8b9476ba 1747verify_mode = options & vopt_is_recipient ? US"R" : US"S";
ea90b718 1748
059ec3d9
PH
1749/* Update the address structure with the possibly qualified and rewritten
1750address. Set it up as the starting address on the chain of new addresses. */
1751
1752vaddr->address = address;
1753addr_new = vaddr;
1754
1755/* We need a loop, because an address can generate new addresses. We must also
1756cope with generated pipes and files at the top level. (See also the code and
1757comment in deliver.c.) However, it is usually the case that the router for
1758user's .forward files has its verify flag turned off.
1759
1760If an address generates more than one child, the loop is used only when
1761full_info is set, and this can only be set locally. Remote enquiries just get
1762information about the top level address, not anything that it generated. */
1763
ea90b718 1764while (addr_new)
059ec3d9
PH
1765 {
1766 int rc;
1767 address_item *addr = addr_new;
1768
1769 addr_new = addr->next;
1770 addr->next = NULL;
1771
1772 DEBUG(D_verify)
1773 {
1774 debug_printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1775 debug_printf("Considering %s\n", addr->address);
1776 }
1777
1778 /* Handle generated pipe, file or reply addresses. We don't get these
1779 when handling EXPN, as it does only one level of expansion. */
1780
1781 if (testflag(addr, af_pfr))
1782 {
1783 allok = FALSE;
2ddb4094 1784 if (f)
059ec3d9
PH
1785 {
1786 BOOL allow;
1787
1788 if (addr->address[0] == '>')
1789 {
1790 allow = testflag(addr, af_allow_reply);
1791 fprintf(f, "%s -> mail %s", addr->parent->address, addr->address + 1);
1792 }
1793 else
1794 {
2ddb4094
JH
1795 allow = addr->address[0] == '|'
1796 ? testflag(addr, af_allow_pipe) : testflag(addr, af_allow_file);
059ec3d9
PH
1797 fprintf(f, "%s -> %s", addr->parent->address, addr->address);
1798 }
1799
1800 if (addr->basic_errno == ERRNO_BADTRANSPORT)
1801 fprintf(f, "\n*** Error in setting up pipe, file, or autoreply:\n"
1802 "%s\n", addr->message);
1803 else if (allow)
1804 fprintf(f, "\n transport = %s\n", addr->transport->name);
1805 else
1806 fprintf(f, " *** forbidden ***\n");
1807 }
1808 continue;
1809 }
1810
1811 /* Just in case some router parameter refers to it. */
1812
2f682e45
JH
1813 return_path = addr->prop.errors_address
1814 ? addr->prop.errors_address : sender_address;
059ec3d9
PH
1815
1816 /* Split the address into domain and local part, handling the %-hack if
1817 necessary, and then route it. While routing a sender address, set
1818 $sender_address to <> because that is what it will be if we were trying to
1819 send a bounce to the sender. */
1820
2f682e45 1821 if (routed) *routed = FALSE;
059ec3d9
PH
1822 if ((rc = deliver_split_address(addr)) == OK)
1823 {
8b9476ba 1824 if (!(options & vopt_is_recipient)) sender_address = null_sender;
059ec3d9
PH
1825 rc = route_address(addr, &addr_local, &addr_remote, &addr_new,
1826 &addr_succeed, verify_type);
1827 sender_address = save_sender; /* Put back the real sender */
1828 }
1829
1830 /* If routing an address succeeded, set the flag that remembers, for use when
1831 an ACL cached a sender verify (in case a callout fails). Then if routing set
1832 up a list of hosts or the transport has a host list, and the callout option
1833 is set, and we aren't in a host checking run, do the callout verification,
1834 and set another flag that notes that a callout happened. */
1835
1836 if (rc == OK)
1837 {
2f682e45 1838 if (routed) *routed = TRUE;
059ec3d9
PH
1839 if (callout > 0)
1840 {
08f3b11b 1841 transport_instance * tp;
2f682e45 1842 host_item * host_list = addr->host_list;
059ec3d9 1843
26da7e20
PH
1844 /* Make up some data for use in the case where there is no remote
1845 transport. */
1846
1847 transport_feedback tf = {
f2ed27cf
JH
1848 .interface = NULL, /* interface (=> any) */
1849 .port = US"smtp",
1850 .protocol = US"smtp",
1851 .hosts = NULL,
1852 .helo_data = US"$smtp_active_hostname",
1853 .hosts_override = FALSE,
1854 .hosts_randomize = FALSE,
1855 .gethostbyname = FALSE,
1856 .qualify_single = TRUE,
1857 .search_parents = FALSE
26da7e20 1858 };
059ec3d9
PH
1859
1860 /* If verification yielded a remote transport, we want to use that
1861 transport's options, so as to mimic what would happen if we were really
1862 sending a message to this address. */
1863
08f3b11b 1864 if ((tp = addr->transport) && !tp->info->local)
059ec3d9 1865 {
08f3b11b 1866 (void)(tp->setup)(tp, addr, &tf, 0, 0, NULL);
059ec3d9
PH
1867
1868 /* If the transport has hosts and the router does not, or if the
1869 transport is configured to override the router's hosts, we must build a
1870 host list of the transport's hosts, and find the IP addresses */
1871
2f682e45 1872 if (tf.hosts && (!host_list || tf.hosts_override))
059ec3d9
PH
1873 {
1874 uschar *s;
55414b25 1875 const uschar *save_deliver_domain = deliver_domain;
750af86e 1876 uschar *save_deliver_localpart = deliver_localpart;
059ec3d9
PH
1877
1878 host_list = NULL; /* Ignore the router's hosts */
1879
1880 deliver_domain = addr->domain;
1881 deliver_localpart = addr->local_part;
1882 s = expand_string(tf.hosts);
750af86e
PH
1883 deliver_domain = save_deliver_domain;
1884 deliver_localpart = save_deliver_localpart;
059ec3d9 1885
2f682e45 1886 if (!s)
059ec3d9
PH
1887 {
1888 log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand list of hosts "
1889 "\"%s\" in %s transport for callout: %s", tf.hosts,
08f3b11b 1890 tp->name, expand_string_message);
059ec3d9
PH
1891 }
1892 else
1893 {
322050c2 1894 int flags;
d8ef3577 1895 host_item *host, *nexthost;
059ec3d9
PH
1896 host_build_hostlist(&host_list, s, tf.hosts_randomize);
1897
1898 /* Just ignore failures to find a host address. If we don't manage
8e669ac1
PH
1899 to find any addresses, the callout will defer. Note that more than
1900 one address may be found for a single host, which will result in
1901 additional host items being inserted into the chain. Hence we must
d8ef3577 1902 save the next host first. */
059ec3d9 1903
66387a73 1904 flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA;
322050c2
PH
1905 if (tf.qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE;
1906 if (tf.search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
1907
2f682e45 1908 for (host = host_list; host; host = nexthost)
059ec3d9 1909 {
d8ef3577 1910 nexthost = host->next;
8e669ac1 1911 if (tf.gethostbyname ||
7e66e54d 1912 string_is_ip_address(host->name, NULL) != 0)
55414b25 1913 (void)host_find_byname(host, NULL, flags, NULL, TRUE);
059ec3d9 1914 else
9d9c3746 1915 {
8ac90765 1916 const dnssec_domains * dsp = NULL;
08f3b11b 1917 if (Ustrcmp(tp->driver_name, "smtp") == 0)
9d9c3746
JH
1918 {
1919 smtp_transport_options_block * ob =
08f3b11b 1920 (smtp_transport_options_block *) tp->options_block;
8ac90765 1921 dsp = &ob->dnssec;
9d9c3746
JH
1922 }
1923
2546388c 1924 (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
8ac90765 1925 dsp, NULL, NULL);
9d9c3746 1926 }
059ec3d9
PH
1927 }
1928 }
1929 }
1930 }
1931
8e669ac1 1932 /* Can only do a callout if we have at least one host! If the callout
2c7db3f5 1933 fails, it will have set ${sender,recipient}_verify_failure. */
059ec3d9 1934
2f682e45 1935 if (host_list)
059ec3d9
PH
1936 {
1937 HDEBUG(D_verify) debug_printf("Attempting full verification using callout\n");
1938 if (host_checking && !host_checking_callout)
1939 {
1940 HDEBUG(D_verify)
1941 debug_printf("... callout omitted by default when host testing\n"
1942 "(Use -bhc if you want the callouts to happen.)\n");
1943 }
1944 else
1945 {
4ed8d31a
JH
1946#ifdef SUPPORT_TLS
1947 deliver_set_expansions(addr);
1948#endif
059ec3d9 1949 rc = do_callout(addr, host_list, &tf, callout, callout_overall,
4deaf07d 1950 callout_connect, options, se_mailfrom, pm_mailfrom);
8a6b4e02
JH
1951#ifdef SUPPORT_TLS
1952 deliver_set_expansions(NULL);
1953#endif
059ec3d9
PH
1954 }
1955 }
1956 else
1957 {
1958 HDEBUG(D_verify) debug_printf("Cannot do callout: neither router nor "
5ffb5d81 1959 "transport provided a host list, or transport is not smtp\n");
059ec3d9
PH
1960 }
1961 }
1962 }
8e669ac1 1963
2c7db3f5 1964 /* Otherwise, any failure is a routing failure */
8e669ac1
PH
1965
1966 else *failure_ptr = US"route";
059ec3d9
PH
1967
1968 /* A router may return REROUTED if it has set up a child address as a result
1969 of a change of domain name (typically from widening). In this case we always
1970 want to continue to verify the new child. */
1971
1972 if (rc == REROUTED) continue;
8e669ac1 1973
059ec3d9
PH
1974 /* Handle hard failures */
1975
1976 if (rc == FAIL)
1977 {
1978 allok = FALSE;
2f682e45 1979 if (f)
059ec3d9 1980 {
e6f6568e
PH
1981 address_item *p = addr->parent;
1982
ce552449 1983 respond_printf(f, "%s%s %s", ko_prefix,
2f682e45
JH
1984 full_info ? addr->address : address,
1985 address_test_mode ? "is undeliverable" : "failed to verify");
059ec3d9
PH
1986 if (!expn && admin_user)
1987 {
1988 if (addr->basic_errno > 0)
ce552449 1989 respond_printf(f, ": %s", strerror(addr->basic_errno));
2f682e45 1990 if (addr->message)
ce552449 1991 respond_printf(f, ": %s", addr->message);
e6f6568e
PH
1992 }
1993
1994 /* Show parents iff doing full info */
1995
2f682e45 1996 if (full_info) while (p)
e6f6568e 1997 {
ce552449 1998 respond_printf(f, "%s\n <-- %s", cr, p->address);
e6f6568e 1999 p = p->parent;
059ec3d9 2000 }
ce552449 2001 respond_printf(f, "%s\n", cr);
059ec3d9 2002 }
57cc2785 2003 cancel_cutthrough_connection(TRUE, US"routing hard fail");
059ec3d9 2004
d9b2312b 2005 if (!full_info)
2f682e45 2006 {
d9b2312b
JH
2007 yield = copy_error(vaddr, addr, FAIL);
2008 goto out;
2f682e45
JH
2009 }
2010 yield = FAIL;
059ec3d9
PH
2011 }
2012
2013 /* Soft failure */
2014
2015 else if (rc == DEFER)
2016 {
2017 allok = FALSE;
2f682e45 2018 if (f)
059ec3d9 2019 {
e6f6568e 2020 address_item *p = addr->parent;
ce552449 2021 respond_printf(f, "%s%s cannot be resolved at this time", ko_prefix,
322050c2 2022 full_info? addr->address : address);
059ec3d9
PH
2023 if (!expn && admin_user)
2024 {
2025 if (addr->basic_errno > 0)
ce552449 2026 respond_printf(f, ": %s", strerror(addr->basic_errno));
2f682e45 2027 if (addr->message)
ce552449 2028 respond_printf(f, ": %s", addr->message);
059ec3d9 2029 else if (addr->basic_errno <= 0)
ce552449 2030 respond_printf(f, ": unknown error");
059ec3d9
PH
2031 }
2032
e6f6568e
PH
2033 /* Show parents iff doing full info */
2034
2f682e45 2035 if (full_info) while (p)
e6f6568e 2036 {
ce552449 2037 respond_printf(f, "%s\n <-- %s", cr, p->address);
e6f6568e
PH
2038 p = p->parent;
2039 }
ce552449 2040 respond_printf(f, "%s\n", cr);
059ec3d9 2041 }
57cc2785 2042 cancel_cutthrough_connection(TRUE, US"routing soft fail");
e4bdf652 2043
d9b2312b
JH
2044 if (!full_info)
2045 {
2046 yield = copy_error(vaddr, addr, DEFER);
2047 goto out;
2048 }
2f682e45 2049 if (yield == OK) yield = DEFER;
059ec3d9
PH
2050 }
2051
2052 /* If we are handling EXPN, we do not want to continue to route beyond
e6f6568e 2053 the top level (whose address is in "address"). */
059ec3d9
PH
2054
2055 else if (expn)
2056 {
2057 uschar *ok_prefix = US"250-";
2f682e45
JH
2058
2059 if (!addr_new)
2060 if (!addr_local && !addr_remote)
ce552449 2061 respond_printf(f, "250 mail to <%s> is discarded\r\n", address);
059ec3d9 2062 else
ce552449 2063 respond_printf(f, "250 <%s>\r\n", address);
2f682e45
JH
2064
2065 else do
059ec3d9
PH
2066 {
2067 address_item *addr2 = addr_new;
2068 addr_new = addr2->next;
2f682e45 2069 if (!addr_new) ok_prefix = US"250 ";
ce552449 2070 respond_printf(f, "%s<%s>\r\n", ok_prefix, addr2->address);
2f682e45 2071 } while (addr_new);
d9b2312b
JH
2072 yield = OK;
2073 goto out;
059ec3d9
PH
2074 }
2075
2076 /* Successful routing other than EXPN. */
2077
2078 else
2079 {
2080 /* Handle successful routing when short info wanted. Otherwise continue for
2081 other (generated) addresses. Short info is the operational case. Full info
2082 can be requested only when debug_selector != 0 and a file is supplied.
2083
2084 There is a conflict between the use of aliasing as an alternate email
2085 address, and as a sort of mailing list. If an alias turns the incoming
2086 address into just one address (e.g. J.Caesar->jc44) you may well want to
2087 carry on verifying the generated address to ensure it is valid when
2088 checking incoming mail. If aliasing generates multiple addresses, you
2089 probably don't want to do this. Exim therefore treats the generation of
2090 just a single new address as a special case, and continues on to verify the
2091 generated address. */
2092
2f682e45
JH
2093 if ( !full_info /* Stop if short info wanted AND */
2094 && ( ( !addr_new /* No new address OR */
2095 || addr_new->next /* More than one new address OR */
2096 || testflag(addr_new, af_pfr) /* New address is pfr */
2097 )
2098 || /* OR */
2099 ( addr_new /* At least one new address AND */
2100 && success_on_redirect /* success_on_redirect is set */
2101 ) )
2102 )
059ec3d9 2103 {
2f682e45
JH
2104 if (f) fprintf(f, "%s %s\n",
2105 address, address_test_mode ? "is deliverable" : "verified");
059ec3d9
PH
2106
2107 /* If we have carried on to verify a child address, we want the value
2108 of $address_data to be that of the child */
2109
d43cbe25 2110 vaddr->prop.address_data = addr->prop.address_data;
98c82a3d
JH
2111
2112 /* If stopped because more than one new address, cannot cutthrough */
2113
2114 if (addr_new && addr_new->next)
57cc2785 2115 cancel_cutthrough_connection(TRUE, US"multiple addresses from routing");
98c82a3d 2116
d9b2312b
JH
2117 yield = OK;
2118 goto out;
059ec3d9
PH
2119 }
2120 }
2121 } /* Loop for generated addresses */
2122
2123/* Display the full results of the successful routing, including any generated
2124addresses. Control gets here only when full_info is set, which requires f not
2125to be NULL, and this occurs only when a top-level verify is called with the
2126debugging switch on.
2127
2128If there are no local and no remote addresses, and there were no pipes, files,
2129or autoreplies, and there were no errors or deferments, the message is to be
2130discarded, usually because of the use of :blackhole: in an alias file. */
2131
2f682e45 2132if (allok && !addr_local && !addr_remote)
dbcef0ea 2133 {
059ec3d9 2134 fprintf(f, "mail to %s is discarded\n", address);
d9b2312b 2135 goto out;
dbcef0ea 2136 }
059ec3d9 2137
dbcef0ea 2138for (addr_list = addr_local, i = 0; i < 2; addr_list = addr_remote, i++)
08f3b11b 2139 while (addr_list)
059ec3d9
PH
2140 {
2141 address_item *addr = addr_list;
2142 address_item *p = addr->parent;
08f3b11b
JH
2143 transport_instance * tp = addr->transport;
2144
059ec3d9
PH
2145 addr_list = addr->next;
2146
2147 fprintf(f, "%s", CS addr->address);
384152a6 2148#ifdef EXPERIMENTAL_SRS
d43cbe25
JH
2149 if(addr->prop.srs_sender)
2150 fprintf(f, " [srs = %s]", addr->prop.srs_sender);
384152a6 2151#endif
dbcef0ea
PH
2152
2153 /* If the address is a duplicate, show something about it. */
2154
2155 if (!testflag(addr, af_pfr))
2156 {
2157 tree_node *tnode;
08f3b11b 2158 if ((tnode = tree_search(tree_duplicates, addr->unique)))
dbcef0ea
PH
2159 fprintf(f, " [duplicate, would not be delivered]");
2160 else tree_add_duplicate(addr->unique, addr);
2161 }
2162
2163 /* Now show its parents */
2164
08f3b11b 2165 for (p = addr->parent; p; p = p->parent)
059ec3d9 2166 fprintf(f, "\n <-- %s", p->address);
059ec3d9
PH
2167 fprintf(f, "\n ");
2168
2169 /* Show router, and transport */
2170
08f3b11b
JH
2171 fprintf(f, "router = %s, transport = %s\n",
2172 addr->router->name, tp ? tp->name : US"unset");
059ec3d9
PH
2173
2174 /* Show any hosts that are set up by a router unless the transport
2175 is going to override them; fiddle a bit to get a nice format. */
2176
08f3b11b 2177 if (addr->host_list && tp && !tp->overrides_hosts)
059ec3d9
PH
2178 {
2179 host_item *h;
2180 int maxlen = 0;
2181 int maxaddlen = 0;
08f3b11b
JH
2182 for (h = addr->host_list; h; h = h->next)
2183 { /* get max lengths of host names, addrs */
059ec3d9
PH
2184 int len = Ustrlen(h->name);
2185 if (len > maxlen) maxlen = len;
08f3b11b 2186 len = h->address ? Ustrlen(h->address) : 7;
059ec3d9
PH
2187 if (len > maxaddlen) maxaddlen = len;
2188 }
08f3b11b
JH
2189 for (h = addr->host_list; h; h = h->next)
2190 {
2191 fprintf(f, " host %-*s ", maxlen, h->name);
2192
2193 if (h->address)
2194 fprintf(f, "[%s%-*c", h->address, maxaddlen+1 - Ustrlen(h->address), ']');
2195 else if (tp->info->local)
2196 fprintf(f, " %-*s ", maxaddlen, ""); /* Omit [unknown] for local */
2197 else
2198 fprintf(f, "[%s%-*c", "unknown", maxaddlen+1 - 7, ']');
2199
2200 if (h->mx >= 0) fprintf(f, " MX=%d", h->mx);
059ec3d9 2201 if (h->port != PORT_NONE) fprintf(f, " port=%d", h->port);
08f3b11b
JH
2202 if (running_in_test_harness && h->dnssec == DS_YES) fputs(" AD", f);
2203 if (h->status == hstatus_unusable) fputs(" ** unusable **", f);
2204 fputc('\n', f);
059ec3d9
PH
2205 }
2206 }
2207 }
059ec3d9 2208
d9b2312b 2209/* Yield will be DEFER or FAIL if any one address has, only for full_info (which is
2c7db3f5
PH
2210the -bv or -bt case). */
2211
d9b2312b 2212out:
ea90b718 2213verify_mode = NULL;
35aba663 2214tls_modify_variables(&tls_in);
d9b2312b 2215
8e669ac1 2216return yield;
059ec3d9
PH
2217}
2218
2219
2220
2221
2222/*************************************************
2223* Check headers for syntax errors *
2224*************************************************/
2225
2226/* This function checks those header lines that contain addresses, and verifies
db57e575 2227that all the addresses therein are 5322-syntactially correct.
059ec3d9
PH
2228
2229Arguments:
2230 msgptr where to put an error message
2231
2232Returns: OK
2233 FAIL
2234*/
2235
2236int
2237verify_check_headers(uschar **msgptr)
2238{
2239header_line *h;
2240uschar *colon, *s;
1eccaa59 2241int yield = OK;
059ec3d9 2242
db57e575 2243for (h = header_list; h && yield == OK; h = h->next)
059ec3d9
PH
2244 {
2245 if (h->type != htype_from &&
2246 h->type != htype_reply_to &&
2247 h->type != htype_sender &&
2248 h->type != htype_to &&
2249 h->type != htype_cc &&
2250 h->type != htype_bcc)
2251 continue;
2252
2253 colon = Ustrchr(h->text, ':');
2254 s = colon + 1;
2255 while (isspace(*s)) s++;
2256
1eccaa59
PH
2257 /* Loop for multiple addresses in the header, enabling group syntax. Note
2258 that we have to reset this after the header has been scanned. */
059ec3d9 2259
1eccaa59 2260 parse_allow_group = TRUE;
059ec3d9 2261
db57e575 2262 while (*s)
059ec3d9
PH
2263 {
2264 uschar *ss = parse_find_address_end(s, FALSE);
2265 uschar *recipient, *errmess;
2266 int terminator = *ss;
2267 int start, end, domain;
2268
2269 /* Temporarily terminate the string at this point, and extract the
1eccaa59 2270 operative address within, allowing group syntax. */
059ec3d9
PH
2271
2272 *ss = 0;
2273 recipient = parse_extract_address(s,&errmess,&start,&end,&domain,FALSE);
2274 *ss = terminator;
2275
2276 /* Permit an unqualified address only if the message is local, or if the
2277 sending host is configured to be permitted to send them. */
2278
db57e575 2279 if (recipient && !domain)
059ec3d9
PH
2280 {
2281 if (h->type == htype_from || h->type == htype_sender)
2282 {
2283 if (!allow_unqualified_sender) recipient = NULL;
2284 }
2285 else
2286 {
2287 if (!allow_unqualified_recipient) recipient = NULL;
2288 }
2289 if (recipient == NULL) errmess = US"unqualified address not permitted";
2290 }
2291
2292 /* It's an error if no address could be extracted, except for the special
2293 case of an empty address. */
2294
db57e575 2295 if (!recipient && Ustrcmp(errmess, "empty address") != 0)
059ec3d9
PH
2296 {
2297 uschar *verb = US"is";
2298 uschar *t = ss;
1ab95fa6 2299 uschar *tt = colon;
059ec3d9
PH
2300 int len;
2301
2302 /* Arrange not to include any white space at the end in the
1ab95fa6 2303 error message or the header name. */
059ec3d9
PH
2304
2305 while (t > s && isspace(t[-1])) t--;
1ab95fa6 2306 while (tt > h->text && isspace(tt[-1])) tt--;
059ec3d9 2307
1ab95fa6 2308 /* Add the address that failed to the error message, since in a
059ec3d9
PH
2309 header with very many addresses it is sometimes hard to spot
2310 which one is at fault. However, limit the amount of address to
2311 quote - cases have been seen where, for example, a missing double
2312 quote in a humungous To: header creates an "address" that is longer
2313 than string_sprintf can handle. */
2314
2315 len = t - s;
2316 if (len > 1024)
2317 {
2318 len = 1024;
2319 verb = US"begins";
2320 }
2321
55414b25
JH
2322 /* deconst cast ok as we're passing a non-const to string_printing() */
2323 *msgptr = US string_printing(
1ab95fa6 2324 string_sprintf("%s: failing address in \"%.*s:\" header %s: %.*s",
bb07bcd3 2325 errmess, (int)(tt - h->text), h->text, verb, len, s));
059ec3d9 2326
1eccaa59
PH
2327 yield = FAIL;
2328 break; /* Out of address loop */
059ec3d9
PH
2329 }
2330
2331 /* Advance to the next address */
2332
db57e575 2333 s = ss + (terminator ? 1 : 0);
059ec3d9
PH
2334 while (isspace(*s)) s++;
2335 } /* Next address */
059ec3d9 2336
1eccaa59
PH
2337 parse_allow_group = FALSE;
2338 parse_found_group = FALSE;
2339 } /* Next header unless yield has been set FALSE */
2340
2341return yield;
059ec3d9
PH
2342}
2343
2344
770747fd
MFM
2345/*************************************************
2346* Check header names for 8-bit characters *
2347*************************************************/
2348
4c04137d 2349/* This function checks for invalid characters in header names. See
770747fd
MFM
2350RFC 5322, 2.2. and RFC 6532, 3.
2351
2352Arguments:
2353 msgptr where to put an error message
2354
2355Returns: OK
2356 FAIL
2357*/
2358
2359int
2360verify_check_header_names_ascii(uschar **msgptr)
2361{
2362header_line *h;
2363uschar *colon, *s;
2364
57cc2785 2365for (h = header_list; h; h = h->next)
770747fd 2366 {
57cc2785
JH
2367 colon = Ustrchr(h->text, ':');
2368 for(s = h->text; s < colon; s++)
2369 if ((*s < 33) || (*s > 126))
2370 {
2371 *msgptr = string_sprintf("Invalid character in header \"%.*s\" found",
2372 colon - h->text, h->text);
2373 return FAIL;
2374 }
770747fd
MFM
2375 }
2376return OK;
2377}
059ec3d9 2378
1c41c9cc
PH
2379/*************************************************
2380* Check for blind recipients *
2381*************************************************/
2382
2383/* This function checks that every (envelope) recipient is mentioned in either
2384the To: or Cc: header lines, thus detecting blind carbon copies.
2385
2386There are two ways of scanning that could be used: either scan the header lines
2387and tick off the recipients, or scan the recipients and check the header lines.
2388The original proposed patch did the former, but I have chosen to do the latter,
2389because (a) it requires no memory and (b) will use fewer resources when there
2390are many addresses in To: and/or Cc: and only one or two envelope recipients.
2391
2392Arguments: none
2393Returns: OK if there are no blind recipients
2394 FAIL if there is at least one blind recipient
2395*/
2396
2397int
2398verify_check_notblind(void)
2399{
2400int i;
2401for (i = 0; i < recipients_count; i++)
2402 {
2403 header_line *h;
2404 BOOL found = FALSE;
2405 uschar *address = recipients_list[i].address;
2406
2407 for (h = header_list; !found && h != NULL; h = h->next)
2408 {
2409 uschar *colon, *s;
2410
2411 if (h->type != htype_to && h->type != htype_cc) continue;
2412
2413 colon = Ustrchr(h->text, ':');
2414 s = colon + 1;
2415 while (isspace(*s)) s++;
2416
1eccaa59
PH
2417 /* Loop for multiple addresses in the header, enabling group syntax. Note
2418 that we have to reset this after the header has been scanned. */
1c41c9cc 2419
1eccaa59 2420 parse_allow_group = TRUE;
1c41c9cc
PH
2421
2422 while (*s != 0)
2423 {
2424 uschar *ss = parse_find_address_end(s, FALSE);
2425 uschar *recipient,*errmess;
2426 int terminator = *ss;
2427 int start, end, domain;
2428
2429 /* Temporarily terminate the string at this point, and extract the
1eccaa59 2430 operative address within, allowing group syntax. */
1c41c9cc
PH
2431
2432 *ss = 0;
2433 recipient = parse_extract_address(s,&errmess,&start,&end,&domain,FALSE);
2434 *ss = terminator;
2435
2436 /* If we found a valid recipient that has a domain, compare it with the
2437 envelope recipient. Local parts are compared case-sensitively, domains
2438 case-insensitively. By comparing from the start with length "domain", we
2439 include the "@" at the end, which ensures that we are comparing the whole
2440 local part of each address. */
2441
2442 if (recipient != NULL && domain != 0)
2443 {
2444 found = Ustrncmp(recipient, address, domain) == 0 &&
2445 strcmpic(recipient + domain, address + domain) == 0;
2446 if (found) break;
2447 }
2448
2449 /* Advance to the next address */
2450
2451 s = ss + (terminator? 1:0);
2452 while (isspace(*s)) s++;
2453 } /* Next address */
1eccaa59
PH
2454
2455 parse_allow_group = FALSE;
2456 parse_found_group = FALSE;
1c41c9cc
PH
2457 } /* Next header (if found is false) */
2458
2459 if (!found) return FAIL;
2460 } /* Next recipient */
2461
2462return OK;
2463}
2464
2465
059ec3d9
PH
2466
2467/*************************************************
2468* Find if verified sender *
2469*************************************************/
2470
2471/* Usually, just a single address is verified as the sender of the message.
2472However, Exim can be made to verify other addresses as well (often related in
2473some way), and this is useful in some environments. There may therefore be a
2474chain of such addresses that have previously been tested. This function finds
2475whether a given address is on the chain.
2476
2477Arguments: the address to be verified
2478Returns: pointer to an address item, or NULL
2479*/
2480
2481address_item *
2482verify_checked_sender(uschar *sender)
2483{
2484address_item *addr;
2485for (addr = sender_verified_list; addr != NULL; addr = addr->next)
2486 if (Ustrcmp(sender, addr->address) == 0) break;
2487return addr;
2488}
2489
2490
2491
2492
2493
2494/*************************************************
2495* Get valid header address *
2496*************************************************/
2497
2498/* Scan the originator headers of the message, looking for an address that
2499verifies successfully. RFC 822 says:
2500
2501 o The "Sender" field mailbox should be sent notices of
2502 any problems in transport or delivery of the original
2503 messages. If there is no "Sender" field, then the
2504 "From" field mailbox should be used.
2505
2506 o If the "Reply-To" field exists, then the reply should
2507 go to the addresses indicated in that field and not to
2508 the address(es) indicated in the "From" field.
2509
2510So we check a Sender field if there is one, else a Reply_to field, else a From
2511field. As some strange messages may have more than one of these fields,
2512especially if they are resent- fields, check all of them if there is more than
2513one.
2514
2515Arguments:
2516 user_msgptr points to where to put a user error message
2517 log_msgptr points to where to put a log error message
2518 callout timeout for callout check (passed to verify_address())
2519 callout_overall overall callout timeout (ditto)
8e669ac1 2520 callout_connect connect callout timeout (ditto)
059ec3d9
PH
2521 se_mailfrom mailfrom for verify; NULL => ""
2522 pm_mailfrom sender for pm callout check (passed to verify_address())
2523 options callout options (passed to verify_address())
8e669ac1 2524 verrno where to put the address basic_errno
059ec3d9
PH
2525
2526If log_msgptr is set to something without setting user_msgptr, the caller
2527normally uses log_msgptr for both things.
2528
2529Returns: result of the verification attempt: OK, FAIL, or DEFER;
2530 FAIL is given if no appropriate headers are found
2531*/
2532
2533int
2534verify_check_header_address(uschar **user_msgptr, uschar **log_msgptr,
8e669ac1 2535 int callout, int callout_overall, int callout_connect, uschar *se_mailfrom,
fe5b5d0b 2536 uschar *pm_mailfrom, int options, int *verrno)
059ec3d9
PH
2537{
2538static int header_types[] = { htype_sender, htype_reply_to, htype_from };
1eccaa59 2539BOOL done = FALSE;
059ec3d9
PH
2540int yield = FAIL;
2541int i;
2542
1eccaa59 2543for (i = 0; i < 3 && !done; i++)
059ec3d9
PH
2544 {
2545 header_line *h;
1eccaa59 2546 for (h = header_list; h != NULL && !done; h = h->next)
059ec3d9
PH
2547 {
2548 int terminator, new_ok;
2549 uschar *s, *ss, *endname;
2550
2551 if (h->type != header_types[i]) continue;
2552 s = endname = Ustrchr(h->text, ':') + 1;
2553
1eccaa59
PH
2554 /* Scan the addresses in the header, enabling group syntax. Note that we
2555 have to reset this after the header has been scanned. */
2556
2557 parse_allow_group = TRUE;
2558
059ec3d9
PH
2559 while (*s != 0)
2560 {
2561 address_item *vaddr;
2562
2563 while (isspace(*s) || *s == ',') s++;
2564 if (*s == 0) break; /* End of header */
2565
2566 ss = parse_find_address_end(s, FALSE);
2567
2568 /* The terminator is a comma or end of header, but there may be white
2569 space preceding it (including newline for the last address). Move back
2570 past any white space so we can check against any cached envelope sender
2571 address verifications. */
2572
2573 while (isspace(ss[-1])) ss--;
2574 terminator = *ss;
2575 *ss = 0;
2576
2577 HDEBUG(D_verify) debug_printf("verifying %.*s header address %s\n",
2578 (int)(endname - h->text), h->text, s);
2579
2580 /* See if we have already verified this address as an envelope sender,
2581 and if so, use the previous answer. */
2582
2583 vaddr = verify_checked_sender(s);
2584
2585 if (vaddr != NULL && /* Previously checked */
2586 (callout <= 0 || /* No callout needed; OR */
2587 vaddr->special_action > 256)) /* Callout was done */
2588 {
2589 new_ok = vaddr->special_action & 255;
2590 HDEBUG(D_verify) debug_printf("previously checked as envelope sender\n");
2591 *ss = terminator; /* Restore shortened string */
2592 }
2593
2594 /* Otherwise we run the verification now. We must restore the shortened
2595 string before running the verification, so the headers are correct, in
2596 case there is any rewriting. */
2597
2598 else
2599 {
2600 int start, end, domain;
1eccaa59
PH
2601 uschar *address = parse_extract_address(s, log_msgptr, &start, &end,
2602 &domain, FALSE);
059ec3d9
PH
2603
2604 *ss = terminator;
2605
1eccaa59
PH
2606 /* If we found an empty address, just carry on with the next one, but
2607 kill the message. */
2608
2609 if (address == NULL && Ustrcmp(*log_msgptr, "empty address") == 0)
2610 {
2611 *log_msgptr = NULL;
2612 s = ss;
2613 continue;
2614 }
2615
059ec3d9
PH
2616 /* If verification failed because of a syntax error, fail this
2617 function, and ensure that the failing address gets added to the error
2618 message. */
2619
2620 if (address == NULL)
2621 {
2622 new_ok = FAIL;
1eccaa59
PH
2623 while (ss > s && isspace(ss[-1])) ss--;
2624 *log_msgptr = string_sprintf("syntax error in '%.*s' header when "
2625 "scanning for sender: %s in \"%.*s\"",
bb07bcd3 2626 (int)(endname - h->text), h->text, *log_msgptr, (int)(ss - s), s);
1eccaa59
PH
2627 yield = FAIL;
2628 done = TRUE;
2629 break;
059ec3d9
PH
2630 }
2631
2f6603e1 2632 /* Else go ahead with the sender verification. But it isn't *the*
059ec3d9
PH
2633 sender of the message, so set vopt_fake_sender to stop sender_address
2634 being replaced after rewriting or qualification. */
2635
2636 else
2637 {
2638 vaddr = deliver_make_addr(address, FALSE);
2639 new_ok = verify_address(vaddr, NULL, options | vopt_fake_sender,
8e669ac1 2640 callout, callout_overall, callout_connect, se_mailfrom,
4deaf07d 2641 pm_mailfrom, NULL);
059ec3d9
PH
2642 }
2643 }
2644
2645 /* We now have the result, either newly found, or cached. If we are
2646 giving out error details, set a specific user error. This means that the
2647 last of these will be returned to the user if all three fail. We do not
2648 set a log message - the generic one below will be used. */
2649
fe5b5d0b 2650 if (new_ok != OK)
059ec3d9 2651 {
8e669ac1 2652 *verrno = vaddr->basic_errno;
fe5b5d0b 2653 if (smtp_return_error_details)
fe5b5d0b
PH
2654 *user_msgptr = string_sprintf("Rejected after DATA: "
2655 "could not verify \"%.*s\" header address\n%s: %s",
bb07bcd3 2656 (int)(endname - h->text), h->text, vaddr->address, vaddr->message);
8e669ac1 2657 }
059ec3d9
PH
2658
2659 /* Success or defer */
2660
1eccaa59
PH
2661 if (new_ok == OK)
2662 {
2663 yield = OK;
2664 done = TRUE;
2665 break;
2666 }
2667
059ec3d9
PH
2668 if (new_ok == DEFER) yield = DEFER;
2669
2670 /* Move on to any more addresses in the header */
2671
2672 s = ss;
1eccaa59
PH
2673 } /* Next address */
2674
2675 parse_allow_group = FALSE;
2676 parse_found_group = FALSE;
2677 } /* Next header, unless done */
2678 } /* Next header type unless done */
059ec3d9
PH
2679
2680if (yield == FAIL && *log_msgptr == NULL)
2681 *log_msgptr = US"there is no valid sender in any header line";
2682
2683if (yield == DEFER && *log_msgptr == NULL)
2684 *log_msgptr = US"all attempts to verify a sender in a header line deferred";
2685
2686return yield;
2687}
2688
2689
2690
2691
2692/*************************************************
2693* Get RFC 1413 identification *
2694*************************************************/
2695
2696/* Attempt to get an id from the sending machine via the RFC 1413 protocol. If
2697the timeout is set to zero, then the query is not done. There may also be lists
2698of hosts and nets which are exempt. To guard against malefactors sending
2699non-printing characters which could, for example, disrupt a message's headers,
2700make sure the string consists of printing characters only.
2701
2702Argument:
2703 port the port to connect to; usually this is IDENT_PORT (113), but when
2704 running in the test harness with -bh a different value is used.
2705
2706Returns: nothing
2707
2708Side effect: any received ident value is put in sender_ident (NULL otherwise)
2709*/
2710
2711void
2712verify_get_ident(int port)
2713{
74f1a423
JH
2714client_conn_ctx ident_conn_ctx = {0};
2715int host_af, qlen;
059ec3d9
PH
2716int received_sender_port, received_interface_port, n;
2717uschar *p;
44e6c20c 2718blob early_data;
059ec3d9
PH
2719uschar buffer[2048];
2720
2721/* Default is no ident. Check whether we want to do an ident check for this
2722host. */
2723
2724sender_ident = NULL;
2725if (rfc1413_query_timeout <= 0 || verify_check_host(&rfc1413_hosts) != OK)
2726 return;
2727
2728DEBUG(D_ident) debug_printf("doing ident callback\n");
2729
2730/* Set up a connection to the ident port of the remote host. Bind the local end
2731to the incoming interface address. If the sender host address is an IPv6
2732address, the incoming interface address will also be IPv6. */
2733
fb05276a 2734host_af = Ustrchr(sender_host_address, ':') == NULL ? AF_INET : AF_INET6;
74f1a423 2735if ((ident_conn_ctx.sock = ip_socket(SOCK_STREAM, host_af)) < 0) return;
059ec3d9 2736
74f1a423 2737if (ip_bind(ident_conn_ctx.sock, host_af, interface_address, 0) < 0)
059ec3d9
PH
2738 {
2739 DEBUG(D_ident) debug_printf("bind socket for ident failed: %s\n",
2740 strerror(errno));
2741 goto END_OFF;
2742 }
2743
44e6c20c
JH
2744/* Construct and send the query. */
2745
2746qlen = snprintf(CS buffer, sizeof(buffer), "%d , %d\r\n",
2747 sender_host_port, interface_port);
2748early_data.data = buffer;
2749early_data.len = qlen;
2750
74f1a423 2751if (ip_connect(ident_conn_ctx.sock, host_af, sender_host_address, port,
44e6c20c 2752 rfc1413_query_timeout, &early_data) < 0)
059ec3d9 2753 {
6c6d6e48 2754 if (errno == ETIMEDOUT && LOGGING(ident_timeout))
059ec3d9
PH
2755 log_write(0, LOG_MAIN, "ident connection to %s timed out",
2756 sender_host_address);
059ec3d9 2757 else
059ec3d9
PH
2758 DEBUG(D_ident) debug_printf("ident connection to %s failed: %s\n",
2759 sender_host_address, strerror(errno));
059ec3d9
PH
2760 goto END_OFF;
2761 }
2762
059ec3d9
PH
2763/* Read a response line. We put it into the rest of the buffer, using several
2764recv() calls if necessary. */
2765
2766p = buffer + qlen;
2767
2768for (;;)
2769 {
2770 uschar *pp;
2771 int count;
2772 int size = sizeof(buffer) - (p - buffer);
2773
2774 if (size <= 0) goto END_OFF; /* Buffer filled without seeing \n. */
74f1a423 2775 count = ip_recv(&ident_conn_ctx, p, size, rfc1413_query_timeout);
059ec3d9
PH
2776 if (count <= 0) goto END_OFF; /* Read error or EOF */
2777
2778 /* Scan what we just read, to see if we have reached the terminating \r\n. Be
2779 generous, and accept a plain \n terminator as well. The only illegal
2780 character is 0. */
2781
2782 for (pp = p; pp < p + count; pp++)
2783 {
2784 if (*pp == 0) goto END_OFF; /* Zero octet not allowed */
2785 if (*pp == '\n')
2786 {
2787 if (pp[-1] == '\r') pp--;
2788 *pp = 0;
2789 goto GOT_DATA; /* Break out of both loops */
2790 }
2791 }
2792
2793 /* Reached the end of the data without finding \n. Let the loop continue to
2794 read some more, if there is room. */
2795
2796 p = pp;
2797 }
2798
2799GOT_DATA:
2800
2801/* We have received a line of data. Check it carefully. It must start with the
2802same two port numbers that we sent, followed by data as defined by the RFC. For
2803example,
2804
2805 12345 , 25 : USERID : UNIX :root
2806
2807However, the amount of white space may be different to what we sent. In the
2808"osname" field there may be several sub-fields, comma separated. The data we
2809actually want to save follows the third colon. Some systems put leading spaces
2810in it - we discard those. */
2811
2812if (sscanf(CS buffer + qlen, "%d , %d%n", &received_sender_port,
2813 &received_interface_port, &n) != 2 ||
2814 received_sender_port != sender_host_port ||
2815 received_interface_port != interface_port)
2816 goto END_OFF;
2817
2818p = buffer + qlen + n;
2819while(isspace(*p)) p++;
2820if (*p++ != ':') goto END_OFF;
2821while(isspace(*p)) p++;
2822if (Ustrncmp(p, "USERID", 6) != 0) goto END_OFF;
2823p += 6;
2824while(isspace(*p)) p++;
2825if (*p++ != ':') goto END_OFF;
2826while (*p != 0 && *p != ':') p++;
2827if (*p++ == 0) goto END_OFF;
2828while(isspace(*p)) p++;
2829if (*p == 0) goto END_OFF;
2830
2831/* The rest of the line is the data we want. We turn it into printing
2832characters when we save it, so that it cannot mess up the format of any logging
2833or Received: lines into which it gets inserted. We keep a maximum of 127
55414b25 2834characters. The deconst cast is ok as we fed a nonconst to string_printing() */
059ec3d9 2835
55414b25 2836sender_ident = US string_printing(string_copyn(p, 127));
059ec3d9
PH
2837DEBUG(D_ident) debug_printf("sender_ident = %s\n", sender_ident);
2838
2839END_OFF:
74f1a423 2840(void)close(ident_conn_ctx.sock);
059ec3d9
PH
2841return;
2842}
2843
2844
2845
2846
2847/*************************************************
2848* Match host to a single host-list item *
2849*************************************************/
2850
2851/* This function compares a host (name or address) against a single item
2852from a host list. The host name gets looked up if it is needed and is not
2853already known. The function is called from verify_check_this_host() via
2854match_check_list(), which is why most of its arguments are in a single block.
2855
2856Arguments:
2857 arg the argument block (see below)
2858 ss the host-list item
2859 valueptr where to pass back looked up data, or NULL
2860 error for error message when returning ERROR
2861
2862The block contains:
32d668a5
PH
2863 host_name (a) the host name, or
2864 (b) NULL, implying use sender_host_name and
2865 sender_host_aliases, looking them up if required, or
2866 (c) the empty string, meaning that only IP address matches
2867 are permitted
059ec3d9
PH
2868 host_address the host address
2869 host_ipv4 the IPv4 address taken from an IPv6 one
2870
2871Returns: OK matched
2872 FAIL did not match
2873 DEFER lookup deferred
32d668a5
PH
2874 ERROR (a) failed to find the host name or IP address, or
2875 (b) unknown lookup type specified, or
2876 (c) host name encountered when only IP addresses are
2877 being matched
059ec3d9
PH
2878*/
2879
32d668a5 2880int
55414b25 2881check_host(void *arg, const uschar *ss, const uschar **valueptr, uschar **error)
059ec3d9
PH
2882{
2883check_host_block *cb = (check_host_block *)arg;
32d668a5 2884int mlen = -1;
059ec3d9 2885int maskoffset;
32d668a5 2886BOOL iplookup = FALSE;
059ec3d9 2887BOOL isquery = FALSE;
32d668a5 2888BOOL isiponly = cb->host_name != NULL && cb->host_name[0] == 0;
55414b25 2889const uschar *t;
32d668a5 2890uschar *semicolon;
059ec3d9
PH
2891uschar **aliases;
2892
2893/* Optimize for the special case when the pattern is "*". */
2894
2895if (*ss == '*' && ss[1] == 0) return OK;
2896
2897/* If the pattern is empty, it matches only in the case when there is no host -
2898this can occur in ACL checking for SMTP input using the -bs option. In this
2899situation, the host address is the empty string. */
2900
2901if (cb->host_address[0] == 0) return (*ss == 0)? OK : FAIL;
2902if (*ss == 0) return FAIL;
2903
32d668a5
PH
2904/* If the pattern is precisely "@" then match against the primary host name,
2905provided that host name matching is permitted; if it's "@[]" match against the
2906local host's IP addresses. */
059ec3d9
PH
2907
2908if (*ss == '@')
2909 {
32d668a5
PH
2910 if (ss[1] == 0)
2911 {
2912 if (isiponly) return ERROR;
2913 ss = primary_hostname;
2914 }
059ec3d9
PH
2915 else if (Ustrcmp(ss, "@[]") == 0)
2916 {
2917 ip_address_item *ip;
2918 for (ip = host_find_interfaces(); ip != NULL; ip = ip->next)
2919 if (Ustrcmp(ip->address, cb->host_address) == 0) return OK;
2920 return FAIL;
2921 }
2922 }
2923
2924/* If the pattern is an IP address, optionally followed by a bitmask count, do
4c04137d 2925a (possibly masked) comparison with the current IP address. */
059ec3d9 2926
7e66e54d 2927if (string_is_ip_address(ss, &maskoffset) != 0)
059ec3d9
PH
2928 return (host_is_in_net(cb->host_address, ss, maskoffset)? OK : FAIL);
2929
1688f43b
PH
2930/* The pattern is not an IP address. A common error that people make is to omit
2931one component of an IPv4 address, either by accident, or believing that, for
2932example, 1.2.3/24 is the same as 1.2.3.0/24, or 1.2.3 is the same as 1.2.3.0,
2933which it isn't. (Those applications that do accept 1.2.3 as an IP address
2934interpret it as 1.2.0.3 because the final component becomes 16-bit - this is an
2935ancient specification.) To aid in debugging these cases, we give a specific
2936error if the pattern contains only digits and dots or contains a slash preceded
2937only by digits and dots (a slash at the start indicates a file name and of
2938course slashes may be present in lookups, but not preceded only by digits and
2939dots). */
2940
2941for (t = ss; isdigit(*t) || *t == '.'; t++);
2942if (*t == 0 || (*t == '/' && t != ss))
2943 {
2944 *error = US"malformed IPv4 address or address mask";
2945 return ERROR;
2946 }
2947
32d668a5 2948/* See if there is a semicolon in the pattern */
059ec3d9 2949
32d668a5
PH
2950semicolon = Ustrchr(ss, ';');
2951
2952/* If we are doing an IP address only match, then all lookups must be IP
df199fec 2953address lookups, even if there is no "net-". */
32d668a5
PH
2954
2955if (isiponly)
059ec3d9 2956 {
32d668a5
PH
2957 iplookup = semicolon != NULL;
2958 }
059ec3d9 2959
32d668a5 2960/* Otherwise, if the item is of the form net[n]-lookup;<file|query> then it is
df199fec
PH
2961a lookup on a masked IP network, in textual form. We obey this code even if we
2962have already set iplookup, so as to skip over the "net-" prefix and to set the
2963mask length. The net- stuff really only applies to single-key lookups where the
2964key is implicit. For query-style lookups the key is specified in the query.
2965From release 4.30, the use of net- for query style is no longer needed, but we
2966retain it for backward compatibility. */
2967
2968if (Ustrncmp(ss, "net", 3) == 0 && semicolon != NULL)
32d668a5
PH
2969 {
2970 mlen = 0;
2971 for (t = ss + 3; isdigit(*t); t++) mlen = mlen * 10 + *t - '0';
2972 if (mlen == 0 && t == ss+3) mlen = -1; /* No mask supplied */
2973 iplookup = (*t++ == '-');
2974 }
1688f43b 2975else t = ss;
059ec3d9 2976
32d668a5 2977/* Do the IP address lookup if that is indeed what we have */
059ec3d9 2978
32d668a5
PH
2979if (iplookup)
2980 {
2981 int insize;
2982 int search_type;
2983 int incoming[4];
2984 void *handle;
2985 uschar *filename, *key, *result;
2986 uschar buffer[64];
059ec3d9 2987
32d668a5 2988 /* Find the search type */
059ec3d9 2989
32d668a5 2990 search_type = search_findtype(t, semicolon - t);
059ec3d9 2991
32d668a5
PH
2992 if (search_type < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s",
2993 search_error_message);
059ec3d9 2994
13b685f9
PH
2995 /* Adjust parameters for the type of lookup. For a query-style lookup, there
2996 is no file name, and the "key" is just the query. For query-style with a file
2997 name, we have to fish the file off the start of the query. For a single-key
2998 lookup, the key is the current IP address, masked appropriately, and
2999 reconverted to text form, with the mask appended. For IPv6 addresses, specify
6a3bceb1
PH
3000 dot separators instead of colons, except when the lookup type is "iplsearch".
3001 */
059ec3d9 3002
13b685f9
PH
3003 if (mac_islookup(search_type, lookup_absfilequery))
3004 {
3005 filename = semicolon + 1;
3006 key = filename;
3007 while (*key != 0 && !isspace(*key)) key++;
3008 filename = string_copyn(filename, key - filename);
3009 while (isspace(*key)) key++;
3010 }
3011 else if (mac_islookup(search_type, lookup_querystyle))
32d668a5
PH
3012 {
3013 filename = NULL;
3014 key = semicolon + 1;
3015 }
6a3bceb1 3016 else /* Single-key style */
32d668a5 3017 {
e6d225ae 3018 int sep = (Ustrcmp(lookup_list[search_type]->name, "iplsearch") == 0)?
6a3bceb1 3019 ':' : '.';
32d668a5
PH
3020 insize = host_aton(cb->host_address, incoming);
3021 host_mask(insize, incoming, mlen);
6a3bceb1 3022 (void)host_nmtoa(insize, incoming, mlen, buffer, sep);
32d668a5
PH
3023 key = buffer;
3024 filename = semicolon + 1;
059ec3d9 3025 }
32d668a5
PH
3026
3027 /* Now do the actual lookup; note that there is no search_close() because
3028 of the caching arrangements. */
3029
d4ff61d1
JH
3030 if (!(handle = search_open(filename, search_type, 0, NULL, NULL)))
3031 log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);
3032
32d668a5
PH
3033 result = search_find(handle, filename, key, -1, NULL, 0, 0, NULL);
3034 if (valueptr != NULL) *valueptr = result;
3035 return (result != NULL)? OK : search_find_defer? DEFER: FAIL;
059ec3d9
PH
3036 }
3037
3038/* The pattern is not an IP address or network reference of any kind. That is,
32d668a5
PH
3039it is a host name pattern. If this is an IP only match, there's an error in the
3040host list. */
3041
3042if (isiponly)
3043 {
3044 *error = US"cannot match host name in match_ip list";
3045 return ERROR;
3046 }
3047
3048/* Check the characters of the pattern to see if they comprise only letters,
3049digits, full stops, and hyphens (the constituents of domain names). Allow
3050underscores, as they are all too commonly found. Sigh. Also, if
3051allow_utf8_domains is set, allow top-bit characters. */
059ec3d9
PH
3052
3053for (t = ss; *t != 0; t++)
3054 if (!isalnum(*t) && *t != '.' && *t != '-' && *t != '_' &&
3055 (!allow_utf8_domains || *t < 128)) break;
3056
3057/* If the pattern is a complete domain name, with no fancy characters, look up
3058its IP address and match against that. Note that a multi-homed host will add
3059items to the chain. */
3060
3061if (*t == 0)
3062 {
3063 int rc;
3064 host_item h;
3065 h.next = NULL;
3066 h.name = ss;
3067 h.address = NULL;
3068 h.mx = MX_NONE;
9b8fadde 3069
1f155f8e
JH
3070 /* Using byname rather than bydns here means we cannot determine dnssec
3071 status. On the other hand it is unclear how that could be either
3072 propagated up or enforced. */
3073
322050c2 3074 rc = host_find_byname(&h, NULL, HOST_FIND_QUALIFY_SINGLE, NULL, FALSE);
059ec3d9
PH
3075 if (rc == HOST_FOUND || rc == HOST_FOUND_LOCAL)
3076 {
3077 host_item *hh;
3078 for (hh = &h; hh != NULL; hh = hh->next)
3079 {
96776534 3080 if (host_is_in_net(hh->address, cb->host_address, 0)) return OK;
059ec3d9
PH
3081 }
3082 return FAIL;
3083 }
3084 if (rc == HOST_FIND_AGAIN) return DEFER;
3085 *error = string_sprintf("failed to find IP address for %s", ss);
3086 return ERROR;
3087 }
3088
3089/* Almost all subsequent comparisons require the host name, and can be done
3090using the general string matching function. When this function is called for
3091outgoing hosts, the name is always given explicitly. If it is NULL, it means we
3092must use sender_host_name and its aliases, looking them up if necessary. */
3093
3094if (cb->host_name != NULL) /* Explicit host name given */
3095 return match_check_string(cb->host_name, ss, -1, TRUE, TRUE, TRUE,
3096 valueptr);
3097
3098/* Host name not given; in principle we need the sender host name and its
3099aliases. However, for query-style lookups, we do not need the name if the
3100query does not contain $sender_host_name. From release 4.23, a reference to
3101$sender_host_name causes it to be looked up, so we don't need to do the lookup
3102on spec. */
3103
3104if ((semicolon = Ustrchr(ss, ';')) != NULL)
3105 {
55414b25 3106 const uschar *affix;
059ec3d9
PH
3107 int partial, affixlen, starflags, id;
3108
3109 *semicolon = 0;
3110 id = search_findtype_partial(ss, &partial, &affix, &affixlen, &starflags);
3111 *semicolon=';';
3112
3113 if (id < 0) /* Unknown lookup type */
3114 {
3115 log_write(0, LOG_MAIN|LOG_PANIC, "%s in host list item \"%s\"",
3116 search_error_message, ss);
3117 return DEFER;
3118 }
13b685f9 3119 isquery = mac_islookup(id, lookup_querystyle|lookup_absfilequery);
059ec3d9
PH
3120 }
3121
3122if (isquery)
3123 {
3124 switch(match_check_string(US"", ss, -1, TRUE, TRUE, TRUE, valueptr))
3125 {
3126 case OK: return OK;
3127 case DEFER: return DEFER;
3128 default: return FAIL;
3129 }
3130 }
3131
3132/* Not a query-style lookup; must ensure the host name is present, and then we
3133do a check on the name and all its aliases. */
3134
2d0dc929 3135if (!sender_host_name)
059ec3d9
PH
3136 {
3137 HDEBUG(D_host_lookup)
3138 debug_printf("sender host name required, to match against %s\n", ss);
3139 if (host_lookup_failed || host_name_lookup() != OK)
3140 {
3141 *error = string_sprintf("failed to find host name for %s",
3142 sender_host_address);;
3143 return ERROR;
3144 }
3145 host_build_sender_fullhost();
3146 }
3147
3148/* Match on the sender host name, using the general matching function */
3149
2d0dc929 3150switch(match_check_string(sender_host_name, ss, -1, TRUE, TRUE, TRUE, valueptr))
059ec3d9
PH
3151 {
3152 case OK: return OK;
3153 case DEFER: return DEFER;
3154 }
3155
3156/* If there are aliases, try matching on them. */
3157
3158aliases = sender_host_aliases;
2d0dc929 3159while (*aliases)
059ec3d9
PH
3160 switch(match_check_string(*aliases++, ss, -1, TRUE, TRUE, TRUE, valueptr))
3161 {
3162 case OK: return OK;
3163 case DEFER: return DEFER;
3164 }
059ec3d9
PH
3165return FAIL;
3166}
3167
3168
3169
3170
3171/*************************************************
3172* Check a specific host matches a host list *
3173*************************************************/
3174
3175/* This function is passed a host list containing items in a number of
3176different formats and the identity of a host. Its job is to determine whether
3177the given host is in the set of hosts defined by the list. The host name is
3178passed as a pointer so that it can be looked up if needed and not already
3179known. This is commonly the case when called from verify_check_host() to check
3180an incoming connection. When called from elsewhere the host name should usually
3181be set.
3182
3183This function is now just a front end to match_check_list(), which runs common
3184code for scanning a list. We pass it the check_host() function to perform a
3185single test.
3186
3187Arguments:
3188 listptr pointer to the host list
3189 cache_bits pointer to cache for named lists, or NULL
3190 host_name the host name or NULL, implying use sender_host_name and
3191 sender_host_aliases, looking them up if required
3192 host_address the IP address
3193 valueptr if not NULL, data from a lookup is passed back here
3194
3195Returns: OK if the host is in the defined set
3196 FAIL if the host is not in the defined set,
3197 DEFER if a data lookup deferred (not a host lookup)
3198
3199If the host name was needed in order to make a comparison, and could not be
3200determined from the IP address, the result is FAIL unless the item
3201"+allow_unknown" was met earlier in the list, in which case OK is returned. */
3202
3203int
55414b25
JH
3204verify_check_this_host(const uschar **listptr, unsigned int *cache_bits,
3205 const uschar *host_name, const uschar *host_address, const uschar **valueptr)
059ec3d9 3206{
d4eb88df 3207int rc;
059ec3d9 3208unsigned int *local_cache_bits = cache_bits;
55414b25 3209const uschar *save_host_address = deliver_host_address;
0ab63f3d 3210check_host_block cb = { .host_name = host_name, .host_address = host_address };
059ec3d9 3211
0ab63f3d 3212if (valueptr) *valueptr = NULL;
059ec3d9
PH
3213
3214/* If the host address starts off ::ffff: it is an IPv6 address in
3215IPv4-compatible mode. Find the IPv4 part for checking against IPv4
3216addresses. */
3217
0ab63f3d
JH
3218cb.host_ipv4 = Ustrncmp(host_address, "::ffff:", 7) == 0
3219 ? host_address + 7 : host_address;
059ec3d9 3220
8e669ac1
PH
3221/* During the running of the check, put the IP address into $host_address. In
3222the case of calls from the smtp transport, it will already be there. However,
3223in other calls (e.g. when testing ignore_target_hosts), it won't. Just to be on
d4eb88df
PH
3224the safe side, any existing setting is preserved, though as I write this
3225(November 2004) I can't see any cases where it is actually needed. */
3226
3227deliver_host_address = host_address;
3228rc = match_check_list(
3229 listptr, /* the list */
3230 0, /* separator character */
3231 &hostlist_anchor, /* anchor pointer */
3232 &local_cache_bits, /* cache pointer */
3233 check_host, /* function for testing */
3234 &cb, /* argument for function */
3235 MCL_HOST, /* type of check */
8e669ac1 3236 (host_address == sender_host_address)?
d4eb88df
PH
3237 US"host" : host_address, /* text for debugging */
3238 valueptr); /* where to pass back data */
3239deliver_host_address = save_host_address;
8e669ac1 3240return rc;
059ec3d9
PH
3241}
3242
3243
3244
3245
3246/*************************************************
5130845b
JH
3247* Check the given host item matches a list *
3248*************************************************/
3249int
3250verify_check_given_host(uschar **listptr, host_item *host)
3251{
55414b25 3252return verify_check_this_host(CUSS listptr, NULL, host->name, host->address, NULL);
5130845b
JH
3253}
3254
3255/*************************************************
059ec3d9
PH
3256* Check the remote host matches a list *
3257*************************************************/
3258
3259/* This is a front end to verify_check_this_host(), created because checking
3260the remote host is a common occurrence. With luck, a good compiler will spot
3261the tail recursion and optimize it. If there's no host address, this is
3262command-line SMTP input - check against an empty string for the address.
3263
3264Arguments:
3265 listptr pointer to the host list
3266
3267Returns: the yield of verify_check_this_host(),
3268 i.e. OK, FAIL, or DEFER
3269*/
3270
3271int
3272verify_check_host(uschar **listptr)
3273{
55414b25 3274return verify_check_this_host(CUSS listptr, sender_host_cache, NULL,
059ec3d9
PH
3275 (sender_host_address == NULL)? US"" : sender_host_address, NULL);
3276}
3277
3278
3279
3280
3281
3282/*************************************************
83e029d5 3283* Invert an IP address *
059ec3d9
PH
3284*************************************************/
3285
83e029d5
PP
3286/* Originally just used for DNS xBL lists, now also used for the
3287reverse_ip expansion operator.
3288
059ec3d9
PH
3289Arguments:
3290 buffer where to put the answer
3291 address the address to invert
3292*/
3293
83e029d5 3294void
059ec3d9
PH
3295invert_address(uschar *buffer, uschar *address)
3296{
3297int bin[4];
3298uschar *bptr = buffer;
3299
3300/* If this is an IPv4 address mapped into IPv6 format, adjust the pointer
3301to the IPv4 part only. */
3302
3303if (Ustrncmp(address, "::ffff:", 7) == 0) address += 7;
3304
3305/* Handle IPv4 address: when HAVE_IPV6 is false, the result of host_aton() is
3306always 1. */
3307
3308if (host_aton(address, bin) == 1)
3309 {
3310 int i;
3311 int x = bin[0];
3312 for (i = 0; i < 4; i++)
3313 {
3314 sprintf(CS bptr, "%d.", x & 255);
3315 while (*bptr) bptr++;
3316 x >>= 8;
3317 }
3318 }
3319
3320/* Handle IPv6 address. Actually, as far as I know, there are no IPv6 addresses
3321in any DNS black lists, and the format in which they will be looked up is
3322unknown. This is just a guess. */
3323
3324#if HAVE_IPV6
3325else
3326 {
3327 int i, j;
3328 for (j = 3; j >= 0; j--)
3329 {
3330 int x = bin[j];
3331 for (i = 0; i < 8; i++)
3332 {
3333 sprintf(CS bptr, "%x.", x & 15);
3334 while (*bptr) bptr++;
3335 x >>= 4;
3336 }
3337 }
3338 }
3339#endif
d6f6e0dc
PH
3340
3341/* Remove trailing period -- this is needed so that both arbitrary
3342dnsbl keydomains and inverted addresses may be combined with the
3343same format string, "%s.%s" */
3344
3345*(--bptr) = 0;
059ec3d9
PH
3346}
3347
3348
3349
3350/*************************************************
0bcb2a0e
PH
3351* Perform a single dnsbl lookup *
3352*************************************************/
3353
d6f6e0dc
PH
3354/* This function is called from verify_check_dnsbl() below. It is also called
3355recursively from within itself when domain and domain_txt are different
3356pointers, in order to get the TXT record from the alternate domain.
0bcb2a0e
PH
3357
3358Arguments:
d6f6e0dc
PH
3359 domain the outer dnsbl domain
3360 domain_txt alternate domain to lookup TXT record on success; when the
3361 same domain is to be used, domain_txt == domain (that is,
3362 the pointers must be identical, not just the text)
8e669ac1 3363 keydomain the current keydomain (for debug message)
d6f6e0dc
PH
3364 prepend subdomain to lookup (like keydomain, but
3365 reversed if IP address)
3366 iplist the list of matching IP addresses, or NULL for "any"
8e669ac1 3367 bitmask true if bitmask matching is wanted
431b7361
PH
3368 match_type condition for 'succeed' result
3369 0 => Any RR in iplist (=)
3370 1 => No RR in iplist (!=)
3371 2 => All RRs in iplist (==)
3372 3 => Some RRs not in iplist (!==)
3373 the two bits are defined as MT_NOT and MT_ALL
8e669ac1 3374 defer_return what to return for a defer
0bcb2a0e
PH
3375
3376Returns: OK if lookup succeeded
3377 FAIL if not
3378*/
3379
3380static int
d6f6e0dc 3381one_check_dnsbl(uschar *domain, uschar *domain_txt, uschar *keydomain,
431b7361 3382 uschar *prepend, uschar *iplist, BOOL bitmask, int match_type,
d6f6e0dc 3383 int defer_return)
8e669ac1 3384{
0bcb2a0e
PH
3385dns_answer dnsa;
3386dns_scan dnss;
3387tree_node *t;
3388dnsbl_cache_block *cb;
3389int old_pool = store_pool;
d6f6e0dc
PH
3390uschar query[256]; /* DNS domain max length */
3391
3392/* Construct the specific query domainname */
3393
3394if (!string_format(query, sizeof(query), "%s.%s", prepend, domain))
3395 {
3396 log_write(0, LOG_MAIN|LOG_PANIC, "dnslist query is too long "
3397 "(ignored): %s...", query);
3398 return FAIL;
3399 }
0bcb2a0e
PH
3400
3401/* Look for this query in the cache. */
3402
14b3c5bc
JH
3403if ( (t = tree_search(dnsbl_cache, query))
3404 && (cb = t->data.ptr)->expiry > time(NULL)
3405 )
3406
3407/* Previous lookup was cached */
3408
3409 {
3410 HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
3411 }
0bcb2a0e
PH
3412
3413/* If not cached from a previous lookup, we must do a DNS lookup, and
3414cache the result in permanent memory. */
3415
14b3c5bc 3416else
0bcb2a0e 3417 {
e162fc97 3418 uint ttl = 3600;
14b3c5bc 3419
0bcb2a0e
PH
3420 store_pool = POOL_PERM;
3421
14b3c5bc
JH
3422 if (t)
3423 {
3424 HDEBUG(D_dnsbl) debug_printf("cached data found but past valid time; ");
3425 }
0bcb2a0e 3426
14b3c5bc
JH
3427 else
3428 { /* Set up a tree entry to cache the lookup */
3429 t = store_get(sizeof(tree_node) + Ustrlen(query));
3430 Ustrcpy(t->name, query);
3431 t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
3432 (void)tree_insertnode(&dnsbl_cache, t);
3433 }
0bcb2a0e 3434
4c04137d 3435 /* Do the DNS lookup . */
0bcb2a0e
PH
3436
3437 HDEBUG(D_dnsbl) debug_printf("new DNS lookup for %s\n", query);
3438 cb->rc = dns_basic_lookup(&dnsa, query, T_A);
3439 cb->text_set = FALSE;
3440 cb->text = NULL;
3441 cb->rhs = NULL;
3442
3443 /* If the lookup succeeded, cache the RHS address. The code allows for
3444 more than one address - this was for complete generality and the possible
d8d9f930
JH
3445 use of A6 records. However, A6 records are no longer supported. Leave the code
3446 here, just in case.
0bcb2a0e
PH
3447
3448 Quite apart from one A6 RR generating multiple addresses, there are DNS
3449 lists that return more than one A record, so we must handle multiple
e162fc97
JH
3450 addresses generated in that way as well.
3451
3452 Mark the cache entry with the "now" plus the minimum of the address TTLs,
3453 or some suitably far-future time if none were found. */
0bcb2a0e
PH
3454
3455 if (cb->rc == DNS_SUCCEED)
3456 {
3457 dns_record *rr;
3458 dns_address **addrp = &(cb->rhs);
3459 for (rr = dns_next_rr(&dnsa, &dnss, RESET_ANSWERS);
e1a3f32f 3460 rr;
0bcb2a0e 3461 rr = dns_next_rr(&dnsa, &dnss, RESET_NEXT))
0bcb2a0e
PH
3462 if (rr->type == T_A)
3463 {
3464 dns_address *da = dns_address_from_rr(&dnsa, rr);
e1a3f32f 3465 if (da)
0bcb2a0e
PH
3466 {
3467 *addrp = da;
2d0dc929
JH
3468 while (da->next) da = da->next;
3469 addrp = &da->next;
14b3c5bc 3470 if (ttl > rr->ttl) ttl = rr->ttl;
0bcb2a0e
PH
3471 }
3472 }
0bcb2a0e
PH
3473
3474 /* If we didn't find any A records, change the return code. This can
3475 happen when there is a CNAME record but there are no A records for what
3476 it points to. */
3477
2d0dc929 3478 if (!cb->rhs) cb->rc = DNS_NODATA;
0bcb2a0e
PH
3479 }
3480
14b3c5bc 3481 cb->expiry = time(NULL)+ttl;
0bcb2a0e
PH
3482 store_pool = old_pool;
3483 }
3484