ARC: fix crash induced by misordered headers. Bug 2493
[exim.git] / src / src / retry.c
CommitLineData
059ec3d9
PH
1/*************************************************
2* Exim - an Internet mail transport agent *
3*************************************************/
4
f9ba5e22 5/* Copyright (c) University of Cambridge 1995 - 2018 */
059ec3d9
PH
6/* See the file NOTICE for conditions of use and distribution. */
7
8/* Functions concerned with retrying unsuccessful deliveries. */
9
10
11#include "exim.h"
12
13
14
15/*************************************************
16* Check the ultimate address timeout *
17*************************************************/
18
19/* This function tests whether a message has been on the queue longer than
ba9af0af 20the maximum retry time for a particular host or address.
059ec3d9
PH
21
22Arguments:
ba9af0af 23 retry_key the key to look up a retry rule
059ec3d9 24 domain the domain to look up a domain retry rule
ba9af0af 25 retry_record contains error information for finding rule
059ec3d9
PH
26 now the time
27
28Returns: TRUE if the ultimate timeout has been reached
29*/
30
ba9af0af 31BOOL
55414b25 32retry_ultimate_address_timeout(uschar *retry_key, const uschar *domain,
ba9af0af 33 dbdata_retry *retry_record, time_t now)
059ec3d9 34{
ba9af0af 35BOOL address_timeout;
de6f74f2 36retry_config * retry;
ba9af0af
TF
37
38DEBUG(D_retry)
39 {
40 debug_printf("retry time not reached: checking ultimate address timeout\n");
de6f74f2
JH
41 debug_printf(" now=" TIME_T_FMT " first_failed=" TIME_T_FMT
42 " next_try=" TIME_T_FMT " expired=%c\n",
43 now, retry_record->first_failed,
44 retry_record->next_try, retry_record->expired ? 'T' : 'F');
ba9af0af 45 }
059ec3d9 46
de6f74f2 47retry = retry_find_config(retry_key+2, domain,
ba9af0af 48 retry_record->basic_errno, retry_record->more_errno);
059ec3d9 49
de6f74f2 50if (retry && retry->rules)
059ec3d9
PH
51 {
52 retry_rule *last_rule;
de6f74f2 53 for (last_rule = retry->rules; last_rule->next; last_rule = last_rule->next) ;
ba9af0af 54 DEBUG(D_retry)
de6f74f2
JH
55 debug_printf(" received_time=" TIME_T_FMT " diff=%d timeout=%d\n",
56 received_time.tv_sec, (int)(now - received_time.tv_sec), last_rule->timeout);
32dfdf8b 57 address_timeout = (now - received_time.tv_sec > last_rule->timeout);
059ec3d9 58 }
ea49d0e1
PH
59else
60 {
ba9af0af 61 DEBUG(D_retry)
ea49d0e1 62 debug_printf("no retry rule found: assume timed out\n");
ba9af0af 63 address_timeout = TRUE;
ea49d0e1 64 }
059ec3d9 65
ba9af0af
TF
66DEBUG(D_retry)
67 if (address_timeout)
68 debug_printf("on queue longer than maximum retry for address - "
69 "allowing delivery\n");
70
059ec3d9
PH
71return address_timeout;
72}
73
74
75
76/*************************************************
77* Set status of a host+address item *
78*************************************************/
79
80/* This function is passed a host_item which contains a host name and an
81IP address string. Its job is to set the status of the address if it is not
82already set (indicated by hstatus_unknown). The possible values are:
83
84 hstatus_usable the address is not listed in the unusable tree, and does
85 not have a retry record, OR the time is past the next
86 try time, OR the message has been on the queue for more
87 than the maximum retry time for a failing host
88
89 hstatus_unusable the address is listed in the unusable tree, or does have
90 a retry record, and the time is not yet at the next retry
91 time.
92
93 hstatus_unusable_expired as above, but also the retry time has expired
94 for this address.
95
96The reason a delivery is permitted when a message has been around for a very
97long time is to allow the ultimate address timeout to operate after a delivery
98failure. Otherwise some messages may stick around without being tried for too
99long.
100
101If a host retry record is retrieved from the hints database, the time of last
102trying is filled into the last_try field of the host block. If a host is
103generally usable, a check is made to see if there is a retry delay on this
104specific message at this host.
105
106If a non-standard port is being used, it is added to the retry key.
107
108Arguments:
109 domain the address domain
110 host pointer to a host item
111 portstring "" for standard port, ":xxxx" for a non-standard port
112 include_ip_address TRUE to include the address in the key - this is
113 usual, but sometimes is not wanted
114 retry_host_key where to put a pointer to the key for the host-specific
115 retry record, if one is read and the host is usable
116 retry_message_key where to put a pointer to the key for the message+host
117 retry record, if one is read and the host is usable
118
119Returns: TRUE if the host has expired but is usable because
120 its retry time has come
121*/
122
123BOOL
55414b25 124retry_check_address(const uschar *domain, host_item *host, uschar *portstring,
059ec3d9
PH
125 BOOL include_ip_address, uschar **retry_host_key, uschar **retry_message_key)
126{
127BOOL yield = FALSE;
128time_t now = time(NULL);
129uschar *host_key, *message_key;
130open_db dbblock;
131open_db *dbm_file;
132tree_node *node;
133dbdata_retry *host_retry_record, *message_retry_record;
134
135*retry_host_key = *retry_message_key = NULL;
136
137DEBUG(D_transport|D_retry) debug_printf("checking status of %s\n", host->name);
138
139/* Do nothing if status already set; otherwise initialize status as usable. */
140
141if (host->status != hstatus_unknown) return FALSE;
142host->status = hstatus_usable;
143
144/* Generate the host key for the unusable tree and the retry database. Ensure
145host names are lower cased (that's what %S does). */
146
147host_key = include_ip_address?
148 string_sprintf("T:%S:%s%s", host->name, host->address, portstring) :
149 string_sprintf("T:%S%s", host->name, portstring);
150
151/* Generate the message-specific key */
152
153message_key = string_sprintf("%s:%s", host_key, message_id);
154
155/* Search the tree of unusable IP addresses. This is filled in when deliveries
156fail, because the retry database itself is not updated until the end of all
157deliveries (so as to do it all in one go). The tree records addresses that have
158become unusable during this delivery process (i.e. those that will get put into
159the retry database when it is updated). */
160
36d295f1 161if ((node = tree_search(tree_unusable, host_key)))
059ec3d9
PH
162 {
163 DEBUG(D_transport|D_retry) debug_printf("found in tree of unusables\n");
164 host->status = (node->data.val > 255)?
165 hstatus_unusable_expired : hstatus_unusable;
166 host->why = node->data.val & 255;
167 return FALSE;
168 }
169
170/* Open the retry database, giving up if there isn't one. Otherwise, search for
171the retry records, and then close the database again. */
172
b10c87b3 173if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE)))
059ec3d9
PH
174 {
175 DEBUG(D_deliver|D_retry|D_hints_lookup)
176 debug_printf("no retry data available\n");
177 return FALSE;
178 }
179host_retry_record = dbfn_read(dbm_file, host_key);
180message_retry_record = dbfn_read(dbm_file, message_key);
181dbfn_close(dbm_file);
182
183/* Ignore the data if it is too old - too long since it was written */
184
36d295f1 185if (!host_retry_record)
059ec3d9
PH
186 {
187 DEBUG(D_transport|D_retry) debug_printf("no host retry record\n");
188 }
189else if (now - host_retry_record->time_stamp > retry_data_expire)
190 {
191 host_retry_record = NULL;
192 DEBUG(D_transport|D_retry) debug_printf("host retry record too old\n");
193 }
194
36d295f1 195if (!message_retry_record)
059ec3d9
PH
196 {
197 DEBUG(D_transport|D_retry) debug_printf("no message retry record\n");
198 }
199else if (now - message_retry_record->time_stamp > retry_data_expire)
200 {
201 message_retry_record = NULL;
202 DEBUG(D_transport|D_retry) debug_printf("message retry record too old\n");
203 }
204
205/* If there's a host-specific retry record, check for reaching the retry
206time (or forcing). If not, and the host is not expired, check for the message
207having been around for longer than the maximum retry time for this host or
208address. Allow the delivery if it has. Otherwise set the appropriate unusable
209flag and return FALSE. Otherwise arrange to return TRUE if this is an expired
210host. */
211
36d295f1 212if (host_retry_record)
059ec3d9
PH
213 {
214 *retry_host_key = host_key;
215
216 /* We have not reached the next try time. Check for the ultimate address
217 timeout if the host has not expired. */
218
8768d548 219 if (now < host_retry_record->next_try && !f.deliver_force)
059ec3d9 220 {
059ec3d9 221 if (!host_retry_record->expired &&
ba9af0af
TF
222 retry_ultimate_address_timeout(host_key, domain,
223 host_retry_record, now))
059ec3d9 224 return FALSE;
059ec3d9
PH
225
226 /* We have not hit the ultimate address timeout; host is unusable. */
227
228 host->status = (host_retry_record->expired)?
229 hstatus_unusable_expired : hstatus_unusable;
230 host->why = hwhy_retry;
231 host->last_try = host_retry_record->last_try;
232 return FALSE;
233 }
234
235 /* Host is usable; set return TRUE if expired. */
236
237 yield = host_retry_record->expired;
238 }
239
240/* It's OK to try the host. If there's a message-specific retry record, check
241for reaching its retry time (or forcing). If not, mark the host unusable,
242unless the ultimate address timeout has been reached. */
243
36d295f1 244if (message_retry_record)
059ec3d9
PH
245 {
246 *retry_message_key = message_key;
8768d548 247 if (now < message_retry_record->next_try && !f.deliver_force)
059ec3d9 248 {
ba9af0af
TF
249 if (!retry_ultimate_address_timeout(host_key, domain,
250 message_retry_record, now))
059ec3d9
PH
251 {
252 host->status = hstatus_unusable;
253 host->why = hwhy_retry;
254 }
059ec3d9
PH
255 return FALSE;
256 }
257 }
258
259return yield;
260}
261
262
263
264
265/*************************************************
266* Add a retry item to an address *
267*************************************************/
268
269/* Retry items are chained onto an address when it is deferred either by router
270or by a transport, or if it succeeds or fails and there was a previous retry
271item that now needs to be deleted. Sometimes there can be both kinds of item:
272for example, if routing was deferred but then succeeded, and delivery then
273deferred. In that case there is a delete item for the routing retry, and an
274updating item for the delivery.
275
276(But note that that is only visible at the outer level, because in remote
277delivery subprocesses, the address starts "clean", with no retry items carried
278in.)
279
280These items are used at the end of a delivery attempt to update the retry
281database. The keys start R: for routing delays and T: for transport delays.
282
283Arguments:
284 addr the address block onto which to hang the item
285 key the retry key
286 flags delete, host, and message flags, copied into the block
287
288Returns: nothing
289*/
290
291void
292retry_add_item(address_item *addr, uschar *key, int flags)
293{
f3ebb786 294retry_item *rti = store_get(sizeof(retry_item), FALSE);
c562fd30 295host_item * host = addr->host_used;
36d295f1 296
059ec3d9
PH
297rti->next = addr->retries;
298addr->retries = rti;
299rti->key = key;
300rti->basic_errno = addr->basic_errno;
301rti->more_errno = addr->more_errno;
c562fd30
JH
302rti->message = host
303 ? string_sprintf("H=%s [%s]: %s", host->name, host->address, addr->message)
304 : addr->message;
059ec3d9
PH
305rti->flags = flags;
306
307DEBUG(D_transport|D_retry)
308 {
309 int letter = rti->more_errno & 255;
310 debug_printf("added retry item for %s: errno=%d more_errno=", rti->key,
311 rti->basic_errno);
312 if (letter == 'A' || letter == 'M')
313 debug_printf("%d,%c", (rti->more_errno >> 8) & 255, letter);
314 else
315 debug_printf("%d", rti->more_errno);
316 debug_printf(" flags=%d\n", flags);
317 }
318}
319
320
321
322/*************************************************
323* Find retry configuration data *
324*************************************************/
325
326/* Search the in-store retry information for the first retry item that applies
327to a given destination. If the key contains an @ we are probably handling a
328local delivery and have a complete address to search for; this happens when
329retry_use_local_part is set on a router. Otherwise, the key is likely to be a
330host name for a remote delivery, or a domain name for a local delivery. We
331prepend *@ on the front of it so that it will match a retry item whose address
332item pattern is independent of the local part. The alternate key, if set, is
333always just a domain, so we treat it likewise.
334
335Arguments:
336 key key for which retry info is wanted
337 alternate alternative key, always just a domain
338 basic_errno specific error predicate on the retry rule, or zero
339 more_errno additional data for errno predicate
340
341Returns: pointer to retry rule, or NULL
342*/
343
344retry_config *
55414b25 345retry_find_config(const uschar *key, const uschar *alternate, int basic_errno,
059ec3d9
PH
346 int more_errno)
347{
55414b25 348const uschar *colon = Ustrchr(key, ':');
059ec3d9
PH
349retry_config *yield;
350
ea49d0e1
PH
351/* If there's a colon in the key, there are two possibilities:
352
353(1) This is a key for a host, ip address, and possibly port, in the format
354
355 hostname:ip+port
356
55414b25 357 In this case, we copy the host name.
ea49d0e1
PH
358
359(2) This is a key for a pipe, file, or autoreply delivery, in the format
360
361 pipe-or-file-or-auto:x@y
362
363 where x@y is the original address that provoked the delivery. The pipe or
364 file or auto will start with | or / or >, whereas a host name will start
365 with a letter or a digit. In this case we want to use the original address
366 to search for a retry rule. */
059ec3d9 367
55414b25
JH
368if (colon)
369 key = isalnum(*key)
370 ? string_copyn(key, colon-key) /* the hostname */
371 : Ustrrchr(key, ':') + 1; /* Take from the last colon */
059ec3d9
PH
372
373/* Sort out the keys */
374
55414b25
JH
375if (!Ustrchr(key, '@')) key = string_sprintf("*@%s", key);
376if (alternate) alternate = string_sprintf("*@%s", alternate);
059ec3d9
PH
377
378/* Scan the configured retry items. */
379
36d295f1 380for (yield = retries; yield; yield = yield->next)
059ec3d9 381 {
55414b25
JH
382 const uschar *plist = yield->pattern;
383 const uschar *slist = yield->senders;
059ec3d9
PH
384
385 /* If a specific error is set for this item, check that we are handling that
386 specific error, and if so, check any additional error information if
387 required. */
388
389 if (yield->basic_errno != 0)
390 {
391 /* Special code is required for quota errors, as these can either be system
392 quota errors, or Exim's own quota imposition, which has a different error
393 number. Full partitions are also treated in the same way as quota errors.
394 */
395
396 if (yield->basic_errno == ERRNO_EXIMQUOTA)
397 {
398 if ((basic_errno != ERRNO_EXIMQUOTA && basic_errno != errno_quota &&
399 basic_errno != ENOSPC) ||
400 (yield->more_errno != 0 && yield->more_errno > more_errno))
401 continue;
402 }
403
e97957bc
PH
404 /* The TLSREQUIRED error also covers TLSFAILURE. These are subtly different
405 errors, but not worth separating at this level. */
406
407 else if (yield->basic_errno == ERRNO_TLSREQUIRED)
408 {
409 if (basic_errno != ERRNO_TLSREQUIRED && basic_errno != ERRNO_TLSFAILURE)
410 continue;
411 }
412
413 /* Handle 4xx responses to MAIL, RCPT, or DATA. The code that was received
414 is in the 2nd least significant byte of more_errno (with 400 subtracted).
415 The required value is coded in the 2nd least significant byte of the
416 yield->more_errno field as follows:
059ec3d9
PH
417
418 255 => any 4xx code
419 >= 100 => the decade must match the value less 100
420 < 100 => the exact value must match
421 */
422
e97957bc
PH
423 else if (yield->basic_errno == ERRNO_MAIL4XX ||
424 yield->basic_errno == ERRNO_RCPT4XX ||
425 yield->basic_errno == ERRNO_DATA4XX)
059ec3d9
PH
426 {
427 int wanted;
e97957bc 428 if (basic_errno != yield->basic_errno) continue;
059ec3d9
PH
429 wanted = (yield->more_errno >> 8) & 255;
430 if (wanted != 255)
431 {
432 int evalue = (more_errno >> 8) & 255;
433 if (wanted >= 100)
434 {
435 if ((evalue/10)*10 != wanted - 100) continue;
436 }
437 else if (evalue != wanted) continue;
438 }
439 }
440
441 /* There are some special cases for timeouts */
442
443 else if (yield->basic_errno == ETIMEDOUT)
444 {
445 if (basic_errno != ETIMEDOUT) continue;
446
447 /* Just RTEF_CTOUT in the rule => don't care about 'A'/'M' addresses */
448 if (yield->more_errno == RTEF_CTOUT)
449 {
450 if ((more_errno & RTEF_CTOUT) == 0) continue;
451 }
452
453 else if (yield->more_errno != 0)
454 {
455 int cf_errno = more_errno;
456 if ((yield->more_errno & RTEF_CTOUT) == 0) cf_errno &= ~RTEF_CTOUT;
457 if (yield->more_errno != cf_errno) continue;
458 }
459 }
460
461 /* Default checks for exact match */
462
463 else
464 {
465 if (yield->basic_errno != basic_errno ||
466 (yield->more_errno != 0 && yield->more_errno != more_errno))
467 continue;
468 }
469 }
470
471 /* If the "senders" condition is set, check it. Note that sender_address may
472 be null during -brt checking, in which case we do not use this rule. */
473
36d295f1
JH
474 if ( slist
475 && ( !sender_address
476 || match_address_list_basic(sender_address, &slist, 0) != OK
477 ) )
059ec3d9
PH
478 continue;
479
480 /* Check for a match between the address list item at the start of this retry
481 rule and either the main or alternate keys. */
482
36d295f1
JH
483 if ( match_address_list_basic(key, &plist, UCHAR_MAX+1) == OK
484 || ( alternate
485 && match_address_list_basic(alternate, &plist, UCHAR_MAX+1) == OK
486 ) )
059ec3d9
PH
487 break;
488 }
489
059ec3d9
PH
490return yield;
491}
492
493
494
495
496/*************************************************
497* Update retry database *
498*************************************************/
499
500/* Update the retry data for any directing/routing/transporting that was
501deferred, or delete it for those that succeeded after a previous defer. This is
502done all in one go to minimize opening/closing/locking of the database file.
503
504Note that, because SMTP delivery involves a list of destinations to try, there
505may be defer-type retry information for some of them even when the message was
506successfully delivered. Likewise if it eventually failed.
507
508This function may move addresses from the defer to the failed queue if the
509ultimate retry time has expired.
510
511Arguments:
512 addr_defer queue of deferred addresses
513 addr_failed queue of failed addresses
514 addr_succeed queue of successful addresses
515
516Returns: nothing
517*/
518
519void
520retry_update(address_item **addr_defer, address_item **addr_failed,
521 address_item **addr_succeed)
522{
523open_db dbblock;
524open_db *dbm_file = NULL;
525time_t now = time(NULL);
059ec3d9
PH
526
527DEBUG(D_retry) debug_printf("Processing retry items\n");
528
529/* Three-times loop to handle succeeded, failed, and deferred addresses.
530Deferred addresses must be handled after failed ones, because some may be moved
531to the failed chain if they have timed out. */
532
d7978c0f 533for (int i = 0; i < 3; i++)
059ec3d9
PH
534 {
535 address_item *endaddr, *addr;
536 address_item *last_first = NULL;
7b4c8c1f
JH
537 address_item **paddr = i==0 ? addr_succeed :
538 i==1 ? addr_failed : addr_defer;
059ec3d9
PH
539 address_item **saved_paddr = NULL;
540
7b4c8c1f
JH
541 DEBUG(D_retry) debug_printf("%s addresses:\n",
542 i == 0 ? "Succeeded" : i == 1 ? "Failed" : "Deferred");
059ec3d9
PH
543
544 /* Loop for each address on the chain. For deferred addresses, the whole
545 address times out unless one of its retry addresses has a retry rule that
546 hasn't yet timed out. Deferred addresses should not be requesting deletion
547 of retry items, but just in case they do by accident, treat that case
548 as "not timed out".
549
550 As well as handling the addresses themselves, we must also process any
551 retry items for any parent addresses - these are typically "delete" items,
552 because the parent must have succeeded in order to generate the child. */
553
7b4c8c1f 554 while ((endaddr = *paddr))
059ec3d9
PH
555 {
556 BOOL timed_out = FALSE;
059ec3d9 557
7b4c8c1f 558 for (addr = endaddr; addr; addr = addr->parent)
059ec3d9
PH
559 {
560 int update_count = 0;
561 int timedout_count = 0;
562
7b4c8c1f
JH
563 DEBUG(D_retry) debug_printf(" %s%s\n", addr->address,
564 addr->retries ? "" : ": no retry items");
059ec3d9
PH
565
566 /* Loop for each retry item. */
567
d7978c0f 568 for (retry_item * rti = addr->retries; rti; rti = rti->next)
059ec3d9
PH
569 {
570 uschar *message;
571 int message_length, message_space, failing_interval, next_try;
572 retry_rule *rule, *final_rule;
573 retry_config *retry;
574 dbdata_retry *retry_record;
575
576 /* Open the retry database if it is not already open; failure to open
577 the file is logged, but otherwise ignored - deferred addresses will
578 get retried at the next opportunity. Not opening earlier than this saves
579 opening if no addresses have retry items - common when none have yet
580 reached their retry next try time. */
581
7b4c8c1f 582 if (!dbm_file)
b10c87b3 583 dbm_file = dbfn_open(US"retry", O_RDWR, &dbblock, TRUE, TRUE);
059ec3d9 584
7b4c8c1f 585 if (!dbm_file)
059ec3d9
PH
586 {
587 DEBUG(D_deliver|D_retry|D_hints_lookup)
588 debug_printf("retry database not available for updating\n");
589 return;
590 }
591
592 /* If there are no deferred addresses, that is, if this message is
593 completing, and the retry item is for a message-specific SMTP error,
594 force it to be deleted, because there's no point in keeping data for
595 no-longer-existing messages. This situation can occur when a domain has
596 two hosts and a message-specific error occurs for the first of them,
597 but the address gets delivered to the second one. This optimization
598 doesn't succeed in cleaning out all the dead entries, but it helps. */
599
7b4c8c1f 600 if (!*addr_defer && rti->flags & rf_message)
059ec3d9
PH
601 rti->flags |= rf_delete;
602
603 /* Handle the case of a request to delete the retry info for this
604 destination. */
605
7b4c8c1f 606 if (rti->flags & rf_delete)
059ec3d9
PH
607 {
608 (void)dbfn_delete(dbm_file, rti->key);
609 DEBUG(D_retry)
610 debug_printf("deleted retry information for %s\n", rti->key);
611 continue;
612 }
613
614 /* Count the number of non-delete retry items. This is so that we
615 can compare it to the count of timed_out ones, to check whether
616 all are timed out. */
617
618 update_count++;
619
620 /* Get the retry information for this destination and error code, if
621 any. If this item is for a remote host with ip address, then pass
622 the domain name as an alternative to search for. If no retry
623 information is found, we can't generate a retry time, so there is
624 no point updating the database. This retry item is timed out. */
625
7b4c8c1f
JH
626 if (!(retry = retry_find_config(rti->key + 2,
627 rti->flags & rf_host ? addr->domain : NULL,
628 rti->basic_errno, rti->more_errno)))
059ec3d9
PH
629 {
630 DEBUG(D_retry) debug_printf("No configured retry item for %s%s%s\n",
631 rti->key,
7b4c8c1f
JH
632 rti->flags & rf_host ? US" or " : US"",
633 rti->flags & rf_host ? addr->domain : US"");
059ec3d9
PH
634 if (addr == endaddr) timedout_count++;
635 continue;
636 }
637
638 DEBUG(D_retry)
7b4c8c1f 639 if (rti->flags & rf_host)
ea49d0e1
PH
640 debug_printf("retry for %s (%s) = %s %d %d\n", rti->key,
641 addr->domain, retry->pattern, retry->basic_errno,
642 retry->more_errno);
059ec3d9 643 else
ea49d0e1
PH
644 debug_printf("retry for %s = %s %d %d\n", rti->key, retry->pattern,
645 retry->basic_errno, retry->more_errno);
059ec3d9
PH
646
647 /* Set up the message for the database retry record. Because DBM
648 records have a maximum data length, we enforce a limit. There isn't
649 much point in keeping a huge message here, anyway. */
650
7b4c8c1f
JH
651 message = rti->basic_errno > 0
652 ? US strerror(rti->basic_errno)
653 : rti->message
654 ? US string_printing(rti->message)
655 : US"unknown error";
059ec3d9
PH
656 message_length = Ustrlen(message);
657 if (message_length > 150) message_length = 150;
658
659 /* Read a retry record from the database or construct a new one.
660 Ignore an old one if it is too old since it was last updated. */
661
662 retry_record = dbfn_read(dbm_file, rti->key);
7b4c8c1f
JH
663 if ( retry_record
664 && now - retry_record->time_stamp > retry_data_expire)
059ec3d9
PH
665 retry_record = NULL;
666
7b4c8c1f 667 if (!retry_record)
059ec3d9 668 {
f3ebb786
JH
669 retry_record = store_get(sizeof(dbdata_retry) + message_length,
670 is_tainted(message));
059ec3d9
PH
671 message_space = message_length;
672 retry_record->first_failed = now;
673 retry_record->last_try = now;
674 retry_record->next_try = now;
675 retry_record->expired = FALSE;
676 retry_record->text[0] = 0; /* just in case */
677 }
678 else message_space = Ustrlen(retry_record->text);
679
680 /* Compute how long this destination has been failing */
681
682 failing_interval = now - retry_record->first_failed;
727071f8
PH
683 DEBUG(D_retry) debug_printf("failing_interval=%d message_age=%d\n",
684 failing_interval, message_age);
685
dd16e114
PH
686 /* For a non-host error, if the message has been on the queue longer
687 than the recorded time of failure, use the message's age instead. This
688 can happen when some messages can be delivered and others cannot; a
689 successful delivery will reset the first_failed time, and this can lead
690 to a failing message being retried too often. */
727071f8 691
7b4c8c1f 692 if (!(rti->flags & rf_host) && message_age > failing_interval)
dd16e114 693 failing_interval = message_age;
059ec3d9
PH
694
695 /* Search for the current retry rule. The cutoff time of the
696 last rule is handled differently to the others. The rule continues
697 to operate for ever (the global maximum interval will eventually
698 limit the gaps) but its cutoff time determines when an individual
699 destination times out. If there are no retry rules, the destination
700 always times out, but we can't compute a retry time. */
701
702 final_rule = NULL;
7b4c8c1f 703 for (rule = retry->rules; rule; rule = rule->next)
059ec3d9
PH
704 {
705 if (failing_interval <= rule->timeout) break;
706 final_rule = rule;
707 }
708
709 /* If there's an un-timed out rule, the destination has not
710 yet timed out, so the address as a whole has not timed out (but we are
711 interested in this only for the end address). Make sure the expired
712 flag is false (can be forced via fixdb from outside, but ensure it is
713 consistent with the rules whenever we go through here). */
714
7b4c8c1f 715 if (rule)
059ec3d9 716 retry_record->expired = FALSE;
059ec3d9
PH
717
718 /* Otherwise, set the retry timeout expired, and set the final rule
719 as the one from which to compute the next retry time. Subsequent
720 messages will fail immediately until the retry time is reached (unless
721 there are other, still active, retries). */
722
723 else
724 {
725 rule = final_rule;
726 retry_record->expired = TRUE;
727 if (addr == endaddr) timedout_count++;
728 }
729
730 /* There is a special case to consider when some messages get through
731 to a destination and others don't. This can happen locally when a
732 large message pushes a user over quota, and it can happen remotely
733 when a machine is on a dodgy Internet connection. The messages that
734 get through wipe the retry information, causing those that don't to
735 stay on the queue longer than the final retry time. In order to
736 avoid this, we check, using the time of arrival of the message, to
737 see if it has been on the queue for more than the final cutoff time,
738 and if so, cause this retry item to time out, and the retry time to
739 be set to "now" so that any subsequent messages in the same condition
740 also get tried. We search for the last rule onwards from the one that
741 is in use. If there are no retry rules for the item, rule will be null
742 and timedout_count will already have been updated.
743
744 This implements "timeout this rule if EITHER the host (or routing or
745 directing) has been failing for more than the maximum time, OR if the
727071f8
PH
746 message has been on the queue for more than the maximum time."
747
748 February 2006: It is possible that this code is no longer needed
749 following the change to the retry calculation to use the message age if
750 it is larger than the time since first failure. It may be that the
751 expired flag is always set when the other conditions are met. However,
752 this is a small bit of code, and it does no harm to leave it in place,
753 just in case. */
059ec3d9 754
32dfdf8b 755 if ( received_time.tv_sec <= retry_record->first_failed
7b4c8c1f
JH
756 && addr == endaddr
757 && !retry_record->expired
758 && rule)
059ec3d9
PH
759 {
760 retry_rule *last_rule;
7b4c8c1f
JH
761 for (last_rule = rule; last_rule->next; last_rule = last_rule->next)
762 ;
32dfdf8b 763 if (now - received_time.tv_sec > last_rule->timeout)
059ec3d9
PH
764 {
765 DEBUG(D_retry) debug_printf("on queue longer than maximum retry\n");
766 timedout_count++;
767 rule = NULL;
768 }
769 }
770
771 /* Compute the next try time from the rule, subject to the global
772 maximum, and update the retry database. If rule == NULL it means
773 there were no rules at all (and the timeout will be set expired),
774 or we have a message that is older than the final timeout. In this
775 case set the next retry time to now, so that one delivery attempt
776 happens for subsequent messages. */
777
7b4c8c1f
JH
778 if (!rule)
779 next_try = now;
780 else
059ec3d9 781 {
7b4c8c1f
JH
782 if (rule->rule == 'F')
783 next_try = now + rule->p1;
6af56900 784 else /* rule = 'G' or 'H' */
059ec3d9
PH
785 {
786 int last_predicted_gap =
787 retry_record->next_try - retry_record->last_try;
788 int last_actual_gap = now - retry_record->last_try;
789 int lastgap = (last_predicted_gap < last_actual_gap)?
790 last_predicted_gap : last_actual_gap;
6af56900
PH
791 int next_gap = (lastgap * rule->p2)/1000;
792 if (rule->rule == 'G')
6af56900 793 next_try = now + ((lastgap < rule->p1)? rule->p1 : next_gap);
6af56900
PH
794 else /* The 'H' rule */
795 {
796 next_try = now + rule->p1;
797 if (next_gap > rule->p1)
3cd34f13
PH
798 next_try += random_number(next_gap - rule->p1)/2 +
799 (next_gap - rule->p1)/2;
6af56900 800 }
059ec3d9
PH
801 }
802 }
803
804 /* Impose a global retry max */
805
806 if (next_try - now > retry_interval_max)
807 next_try = now + retry_interval_max;
808
809 /* If the new message length is greater than the previous one, we
810 have to copy the record first. */
811
812 if (message_length > message_space)
813 {
f3ebb786 814 dbdata_retry *newr = store_get(sizeof(dbdata_retry) + message_length, FALSE);
059ec3d9
PH
815 memcpy(newr, retry_record, sizeof(dbdata_retry));
816 retry_record = newr;
817 }
818
819 /* Set up the retry record; message_length may be less than the string
820 length for very long error strings. */
821
822 retry_record->last_try = now;
823 retry_record->next_try = next_try;
824 retry_record->basic_errno = rti->basic_errno;
825 retry_record->more_errno = rti->more_errno;
826 Ustrncpy(retry_record->text, message, message_length);
827 retry_record->text[message_length] = 0;
828
829 DEBUG(D_retry)
830 {
831 int letter = retry_record->more_errno & 255;
832 debug_printf("Writing retry data for %s\n", rti->key);
833 debug_printf(" first failed=%d last try=%d next try=%d expired=%d\n",
834 (int)retry_record->first_failed, (int)retry_record->last_try,
835 (int)retry_record->next_try, retry_record->expired);
836 debug_printf(" errno=%d more_errno=", retry_record->basic_errno);
837 if (letter == 'A' || letter == 'M')
838 debug_printf("%d,%c", (retry_record->more_errno >> 8) & 255,
839 letter);
840 else
841 debug_printf("%d", retry_record->more_errno);
842 debug_printf(" %s\n", retry_record->text);
843 }
844
845 (void)dbfn_write(dbm_file, rti->key, retry_record,
846 sizeof(dbdata_retry) + message_length);
847 } /* Loop for each retry item */
848
849 /* If all the non-delete retry items are timed out, the address is
850 timed out, provided that we didn't skip any hosts because their retry
851 time was not reached (or because of hosts_max_try). */
852
853 if (update_count > 0 && update_count == timedout_count)
059ec3d9
PH
854 if (!testflag(endaddr, af_retry_skipped))
855 {
856 DEBUG(D_retry) debug_printf("timed out: all retries expired\n");
857 timed_out = TRUE;
858 }
859 else
059ec3d9
PH
860 DEBUG(D_retry)
861 debug_printf("timed out but some hosts were skipped\n");
059ec3d9
PH
862 } /* Loop for an address and its parents */
863
864 /* If this is a deferred address, and retry processing was requested by
865 means of one or more retry items, and they all timed out, move the address
866 to the failed queue, and restart this loop without updating paddr.
867
868 If there were several addresses batched in the same remote delivery, only
869 the original top one will have host retry items attached to it, but we want
870 to handle all the same. Each will have a pointer back to its "top" address,
871 and they will now precede the item with the retries because addresses are
872 inverted when added to these final queues. We have saved information about
873 them in passing (below) so they can all be cut out at once. */
874
875 if (i == 2) /* Handling defers */
876 {
7b4c8c1f 877 if (endaddr->retries && timed_out)
059ec3d9
PH
878 {
879 if (last_first == endaddr) paddr = saved_paddr;
880 addr = *paddr;
881 *paddr = endaddr->next;
882
883 endaddr->next = *addr_failed;
884 *addr_failed = addr;
885
886 for (;; addr = addr->next)
887 {
888 setflag(addr, af_retry_timedout);
13c7874e
JH
889 addr->message = addr->message
890 ? string_sprintf("%s: retry timeout exceeded", addr->message)
891 : US"retry timeout exceeded";
892 addr->user_message = addr->user_message
893 ? string_sprintf("%s: retry timeout exceeded", addr->user_message)
894 : US"retry timeout exceeded";
059ec3d9
PH
895 log_write(0, LOG_MAIN, "** %s%s%s%s: retry timeout exceeded",
896 addr->address,
13c7874e
JH
897 addr->parent ? US" <" : US"",
898 addr->parent ? addr->parent->address : US"",
899 addr->parent ? US">" : US"");
059ec3d9
PH
900
901 if (addr == endaddr) break;
902 }
903
904 continue; /* Restart from changed *paddr */
905 }
906
907 /* This address is to remain on the defer chain. If it has a "first"
908 pointer, save the pointer to it in case we want to fail the set of
909 addresses when we get to the first one. */
910
911 if (endaddr->first != last_first)
912 {
913 last_first = endaddr->first;
914 saved_paddr = paddr;
915 }
916 }
917
918 /* All cases (succeed, fail, defer left on queue) */
919
920 paddr = &(endaddr->next); /* Advance to next address */
921 } /* Loop for all addresses */
922 } /* Loop for succeed, fail, defer */
923
924/* Close and unlock the database */
925
7b4c8c1f 926if (dbm_file) dbfn_close(dbm_file);
059ec3d9
PH
927
928DEBUG(D_retry) debug_printf("end of retry processing\n");
929}
930
931/* End of retry.c */