From: Jeremy Harris <jgh146exb@wizmail.org>
Date: Sat, 12 Oct 2019 11:48:44 +0000 (+0100)
Subject: Reduce delivery process startup time
X-Git-Tag: exim-4_94_RC0~240^2~41
X-Git-Url: https://vcs.fsf.org/?p=exim.git;a=commitdiff_plain;h=d85cdeb5e554b59bf4c43c54461409c15c6ee9c5

Reduce delivery process startup time
---

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 93f4a1eb2..9a27e144c 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -5,6 +5,14 @@ affect Exim's operation, with an unchanged configuration file.  For new
 options, and new features, see the NewStuff file next to this ChangeLog.
 
 
+Exim version 4.next
+-------------------
+
+JH/01 Avoid costly startup code when not strictly needed.  This reduces time
+      for some exim process initialisations.  It does mean that the logging
+      of TLS configuration problems is only done for the daemon startup.
+
+
 Exim version 4.93
 -----------------
 
diff --git a/src/src/daemon.c b/src/src/daemon.c
index 99fa909d2..3fc73babe 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -1740,6 +1740,20 @@ else
 (eg: compile regex) */
 
 dns_pattern_init();
+smtp_deliver_init();	/* Used for callouts */
+
+#ifndef DISABLE_DKIM
+  {
+# ifdef MEASURE_TIMING
+  struct timeval t0;
+  gettimeofday(&t0, NULL);
+# endif
+  dkim_exim_init();
+# ifdef MEASURE_TIMING
+  report_time_since(&t0, US"dkim_exim_init (delta)");
+# endif
+  }
+#endif
 
 #ifdef WITH_CONTENT_SCAN
 malware_init();
diff --git a/src/src/deliver.c b/src/src/deliver.c
index e228a0bfd..5fc748141 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -7146,7 +7146,7 @@ if (addr_remote)
   /* Precompile some regex that are used to recognize parameters in response
   to an EHLO command, if they aren't already compiled. */
 
-  deliver_init();
+  smtp_deliver_init();
 
   /* Now sort the addresses if required, and do the deliveries. The yield of
   do_remote_deliveries is FALSE when mua_wrapper is set and all addresses
@@ -8484,52 +8484,13 @@ return final_yield;
 
 
 void
-deliver_init(void)
+tcp_init(void)
 {
 #ifdef EXIM_TFO_PROBE
 tfo_probe();
 #else
 f.tcp_fastopen_ok = TRUE;
 #endif
-
-
-if (!regex_PIPELINING) regex_PIPELINING =
-  regex_must_compile(US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)", FALSE, TRUE);
-
-if (!regex_SIZE) regex_SIZE =
-  regex_must_compile(US"\\n250[\\s\\-]SIZE(\\s|\\n|$)", FALSE, TRUE);
-
-if (!regex_AUTH) regex_AUTH =
-  regex_must_compile(AUTHS_REGEX, FALSE, TRUE);
-
-#ifndef DISABLE_TLS
-if (!regex_STARTTLS) regex_STARTTLS =
-  regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE);
-#endif
-
-if (!regex_CHUNKING) regex_CHUNKING =
-  regex_must_compile(US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)", FALSE, TRUE);
-
-#ifndef DISABLE_PRDR
-if (!regex_PRDR) regex_PRDR =
-  regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE);
-#endif
-
-#ifdef SUPPORT_I18N
-if (!regex_UTF8) regex_UTF8 =
-  regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE);
-#endif
-
-if (!regex_DSN) regex_DSN  =
-  regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE);
-
-if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA =
-  regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE);
-
-#ifdef SUPPORT_PIPE_CONNECT
-if (!regex_EARLY_PIPE) regex_EARLY_PIPE =
-  regex_must_compile(US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)", FALSE, TRUE);
-#endif
 }
 
 
diff --git a/src/src/dkim.c b/src/src/dkim.c
index 065170444..5c9d2279e 100644
--- a/src/src/dkim.c
+++ b/src/src/dkim.c
@@ -95,6 +95,8 @@ return NULL;	/*XXX better error detail?  logging? */
 void
 dkim_exim_init(void)
 {
+if (f.dkim_init_done) return;
+f.dkim_init_done = TRUE;
 pdkim_init();
 }
 
@@ -103,6 +105,8 @@ pdkim_init();
 void
 dkim_exim_verify_init(BOOL dot_stuffing)
 {
+dkim_exim_init();
+
 /* There is a store-reset between header & body reception
 so cannot use the main pool. Any allocs done by Exim
 memory-handling must use the perm pool. */
@@ -569,6 +573,8 @@ void
 dkim_exim_sign_init(void)
 {
 int old_pool = store_pool;
+
+dkim_exim_init();
 store_pool = POOL_MAIN;
 pdkim_init_context(&dkim_sign_ctx, FALSE, &dkim_exim_query_dns_txt);
 store_pool = old_pool;
diff --git a/src/src/exim.c b/src/src/exim.c
index 2b6297bf5..68734e35c 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -920,7 +920,7 @@ fprintf(fp, "Support for:");
   fprintf(fp, " DMARC");
 #endif
 #ifdef TCP_FASTOPEN
-  deliver_init();
+  tcp_init();
   if (f.tcp_fastopen_ok) fprintf(fp, " TCP_Fast_Open");
 #endif
 #ifdef EXPERIMENTAL_LMDB
@@ -4480,31 +4480,9 @@ if (list_config)
   }
 
 
-/* Initialise subsystems as required */
-#ifndef DISABLE_DKIM
-  {
-# ifdef MEASURE_TIMING
-  struct timeval t0;
-  gettimeofday(&t0, NULL);
-# endif
-  dkim_exim_init();
-# ifdef MEASURE_TIMING
-  report_time_since(&t0, US"dkim_exim_init (delta)");
-# endif
-  }
-#endif
-
-  {
-#ifdef MEASURE_TIMING
-  struct timeval t0;
-  gettimeofday(&t0, NULL);
-#endif
-  deliver_init();
-#ifdef MEASURE_TIMING
-  report_time_since(&t0, US"deliver_init (delta)");
-#endif
-  }
+/* Initialise subsystems as required. */
 
+tcp_init();
 
 /* Handle a request to deliver one or more messages that are already on the
 queue. Values of msg_action other than MSG_DELIVER and MSG_LOAD are dealt with
@@ -4699,6 +4677,21 @@ if (f.daemon_listen || f.inetd_wait_mode || queue_interval > 0)
     log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Daemon cannot be run when "
       "mua_wrapper is set");
     }
+
+  /* This also checks that the library linkage is working and we can call
+  routines in it, so call even if tls_require_ciphers is unset */
+    {
+#ifdef MEASURE_TIMING
+    struct timeval t0, diff;
+    (void)gettimeofday(&t0, NULL);
+#endif
+    if (!tls_dropprivs_validate_require_cipher(FALSE))
+      exit(1);
+#ifdef MEASURE_TIMING
+    report_time_since(&t0, US"validate_ciphers (delta)");
+#endif
+    }
+
   daemon_go();
   }
 
diff --git a/src/src/functions.h b/src/src/functions.h
index 37f6b1b6f..488e84c6c 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -54,6 +54,7 @@ extern BOOL    tls_client_start(client_conn_ctx *, smtp_connect_args *,
 extern void    tls_close(void *, int);
 extern BOOL    tls_could_read(void);
 extern void    tls_daemon_init(void);
+extern BOOL    tls_dropprivs_validate_require_cipher(BOOL);
 extern BOOL    tls_export_cert(uschar *, size_t, void *);
 extern int     tls_feof(void);
 extern int     tls_ferror(void);
@@ -175,7 +176,6 @@ extern void    debug_vprintf(int, const char *, va_list);
 extern void    decode_bits(unsigned int *, size_t, int *,
 	           uschar *, bit_table *, int, uschar *, int);
 extern address_item *deliver_make_addr(uschar *, BOOL);
-extern void    deliver_init(void);
 extern void    delivery_log(int, address_item *, int, uschar *);
 extern int     deliver_message(uschar *, BOOL, BOOL);
 extern void    deliver_msglog(const char *, ...) PRINTF_FUNCTION(1,2);
@@ -445,6 +445,7 @@ extern void    smtp_command_timeout_exit(void) NORETURN;
 extern void    smtp_command_sigterm_exit(void) NORETURN;
 extern void    smtp_data_timeout_exit(void) NORETURN;
 extern void    smtp_data_sigint_exit(void) NORETURN;
+extern void    smtp_deliver_init(void);
 extern uschar *smtp_cmd_hist(void);
 extern int     smtp_connect(smtp_connect_args *, const blob *);
 extern int     smtp_sock_connect(host_item *, int, int, uschar *,
@@ -538,6 +539,7 @@ extern int     strcmpic(const uschar *, const uschar *);
 extern int     strncmpic(const uschar *, const uschar *, int);
 extern uschar *strstric(uschar *, uschar *, BOOL);
 
+extern void    tcp_init(void);
 #ifdef EXIM_TFO_PROBE
 extern void    tfo_probe(void);
 #endif
diff --git a/src/src/globals.c b/src/src/globals.c
index 24281f239..302e18eb3 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -238,6 +238,7 @@ struct global_flags f =
 	.disable_logging        = FALSE,
 #ifndef DISABLE_DKIM
 	.dkim_disable_verify      = FALSE,
+	.dkim_init_done           = FALSE,
 #endif
 #ifdef SUPPORT_DMARC
 	.dmarc_has_been_checked  = FALSE,
diff --git a/src/src/globals.h b/src/src/globals.h
index e4725a719..27a4bd9e0 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -198,6 +198,7 @@ extern struct global_flags {
  BOOL   disable_logging			:1; /* Disables log writing when TRUE */
 #ifndef DISABLE_DKIM
  BOOL   dkim_disable_verify		:1; /* Set via ACL control statement. When set, DKIM verification is disabled for the current message */
+ BOOL   dkim_init_done			:1; /* lazy-init status */
 #endif
 #ifdef SUPPORT_DMARC
  BOOL   dmarc_has_been_checked		:1; /* Global variable to check if test has been called yet */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index 2f78cd746..daa88d010 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -3074,80 +3074,6 @@ log_write(0, LOG_MAIN|LOG_PANIC_DIE, "malformed ratelimit data: %s", s);
 
 
 
-/*************************************************
-*       Drop privs for checking TLS config      *
-*************************************************/
-
-/* We want to validate TLS options during readconf, but do not want to be
-root when we call into the TLS library, in case of library linkage errors
-which cause segfaults; before this check, those were always done as the Exim
-runtime user and it makes sense to continue with that.
-
-Assumes:  tls_require_ciphers has been set, if it will be
-          exim_user has been set, if it will be
-          exim_group has been set, if it will be
-
-Returns:  bool for "okay"; false will cause caller to immediately exit.
-*/
-
-#ifndef DISABLE_TLS
-static BOOL
-tls_dropprivs_validate_require_cipher(BOOL nowarn)
-{
-const uschar *errmsg;
-pid_t pid;
-int rc, status;
-void (*oldsignal)(int);
-
-/* If TLS will never be used, no point checking ciphers */
-
-if (  !tls_advertise_hosts
-   || !*tls_advertise_hosts
-   || Ustrcmp(tls_advertise_hosts, ":") == 0
-   )
-  return TRUE;
-else if (!nowarn && !tls_certificate)
-  log_write(0, LOG_MAIN,
-    "Warning: No server certificate defined; will use a selfsigned one.\n"
-    " Suggested action: either install a certificate or change tls_advertise_hosts option");
-
-oldsignal = signal(SIGCHLD, SIG_DFL);
-
-fflush(NULL);
-if ((pid = fork()) < 0)
-  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "fork failed for TLS check");
-
-if (pid == 0)
-  {
-  /* in some modes, will have dropped privilege already */
-  if (!geteuid())
-    exim_setugid(exim_uid, exim_gid, FALSE,
-        US"calling tls_validate_require_cipher");
-
-  if ((errmsg = tls_validate_require_cipher()))
-    log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
-        "tls_require_ciphers invalid: %s", errmsg);
-  fflush(NULL);
-  exim_underbar_exit(0);
-  }
-
-do {
-  rc = waitpid(pid, &status, 0);
-} while (rc < 0 && errno == EINTR);
-
-DEBUG(D_tls)
-  debug_printf("tls_validate_require_cipher child %d ended: status=0x%x\n",
-      (int)pid, status);
-
-signal(SIGCHLD, oldsignal);
-
-return status == 0;
-}
-#endif /*DISABLE_TLS*/
-
-
-
-
 /*************************************************
 *         Read main configuration options        *
 *************************************************/
@@ -3658,11 +3584,6 @@ if ((tls_verify_hosts || tls_try_verify_hosts) && !tls_verify_certificates)
     "tls_%sverify_hosts is set, but tls_verify_certificates is not set",
     tls_verify_hosts ? "" : "try_");
 
-/* This also checks that the library linkage is working and we can call
-routines in it, so call even if tls_require_ciphers is unset */
-if (!tls_dropprivs_validate_require_cipher(nowarn))
-  exit(1);
-
 /* Magic number: at time of writing, 1024 has been the long-standing value
 used by so many clients, and what Exim used to use always, that it makes
 sense to just min-clamp this max-clamp at that. */
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index deeb04253..6cd9bf75b 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2385,9 +2385,20 @@ and sent an SMTP response. */
 
 DEBUG(D_tls) debug_printf("initialising GnuTLS as a server\n");
 
-if ((rc = tls_init(NULL, tls_certificate, tls_privatekey,
-    NULL, tls_verify_certificates, tls_crl,
-    require_ciphers, &state, &tls_in, errstr)) != OK) return rc;
+  {
+#ifdef MEASURE_TIMING
+  struct timeval t0;
+  gettimeofday(&t0, NULL);
+#endif
+
+  if ((rc = tls_init(NULL, tls_certificate, tls_privatekey,
+      NULL, tls_verify_certificates, tls_crl,
+      require_ciphers, &state, &tls_in, errstr)) != OK) return rc;
+
+#ifdef MEASURE_TIMING
+  report_time_since(&t0, US"server tls_init (delta)");
+#endif
+  }
 
 #ifdef EXPERIMENTAL_TLS_RESUME
 tls_server_resume_prehandshake(state);
@@ -2821,10 +2832,21 @@ if (conn_args->dane && ob->dane_require_tls_ciphers)
 if (!cipher_list)
   cipher_list = ob->tls_require_ciphers;
 
-if (tls_init(host, ob->tls_certificate, ob->tls_privatekey,
-    ob->tls_sni, ob->tls_verify_certificates, ob->tls_crl,
-    cipher_list, &state, tlsp, errstr) != OK)
-  return FALSE;
+  {
+#ifdef MEASURE_TIMING
+  struct timeval t0;
+  gettimeofday(&t0, NULL);
+#endif
+
+  if (tls_init(host, ob->tls_certificate, ob->tls_privatekey,
+      ob->tls_sni, ob->tls_verify_certificates, ob->tls_crl,
+      cipher_list, &state, tlsp, errstr) != OK)
+    return FALSE;
+
+#ifdef MEASURE_TIMING
+  report_time_since(&t0, US"client tls_init (delta)");
+#endif
+  }
 
   {
   int dh_min_bits = ob->tls_dh_min_bits;
diff --git a/src/src/tls.c b/src/src/tls.c
index 796bc6d61..531d67950 100644
--- a/src/src/tls.c
+++ b/src/src/tls.c
@@ -369,6 +369,81 @@ else if ((subjdn = tls_cert_subject(cert, NULL)))
   }
 return FALSE;
 }
+
+
+
+/*************************************************
+*       Drop privs for checking TLS config      *
+*************************************************/
+
+/* We want to validate TLS options during readconf, but do not want to be
+root when we call into the TLS library, in case of library linkage errors
+which cause segfaults; before this check, those were always done as the Exim
+runtime user and it makes sense to continue with that.
+
+Assumes:  tls_require_ciphers has been set, if it will be
+          exim_user has been set, if it will be
+          exim_group has been set, if it will be
+
+Returns:  bool for "okay"; false will cause caller to immediately exit.
+*/
+
+BOOL
+tls_dropprivs_validate_require_cipher(BOOL nowarn)
+{
+const uschar *errmsg;
+pid_t pid;
+int rc, status;
+void (*oldsignal)(int);
+
+/* If TLS will never be used, no point checking ciphers */
+
+if (  !tls_advertise_hosts
+   || !*tls_advertise_hosts
+   || Ustrcmp(tls_advertise_hosts, ":") == 0
+   )
+  return TRUE;
+else if (!nowarn && !tls_certificate)
+  log_write(0, LOG_MAIN,
+    "Warning: No server certificate defined; will use a selfsigned one.\n"
+    " Suggested action: either install a certificate or change tls_advertise_hosts option");
+
+oldsignal = signal(SIGCHLD, SIG_DFL);
+
+fflush(NULL);
+if ((pid = fork()) < 0)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "fork failed for TLS check");
+
+if (pid == 0)
+  {
+  /* in some modes, will have dropped privilege already */
+  if (!geteuid())
+    exim_setugid(exim_uid, exim_gid, FALSE,
+        US"calling tls_validate_require_cipher");
+
+  if ((errmsg = tls_validate_require_cipher()))
+    log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
+        "tls_require_ciphers invalid: %s", errmsg);
+  fflush(NULL);
+  exim_underbar_exit(0);
+  }
+
+do {
+  rc = waitpid(pid, &status, 0);
+} while (rc < 0 && errno == EINTR);
+
+DEBUG(D_tls)
+  debug_printf("tls_validate_require_cipher child %d ended: status=0x%x\n",
+      (int)pid, status);
+
+signal(SIGCHLD, oldsignal);
+
+return status == 0;
+}
+
+
+
+
 #endif	/*!DISABLE_TLS*/
 #endif	/*!MACRO_PREDEF*/
 
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index c547c87fa..b45da05ad 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -357,6 +357,51 @@ static BOOL    pipelining_active;	/* current transaction is in pipe mode */
 static unsigned ehlo_response(uschar * buf, unsigned checks);
 
 
+/******************************************************************************/
+
+void
+smtp_deliver_init(void)
+{
+if (!regex_PIPELINING) regex_PIPELINING =
+  regex_must_compile(US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)", FALSE, TRUE);
+
+if (!regex_SIZE) regex_SIZE =
+  regex_must_compile(US"\\n250[\\s\\-]SIZE(\\s|\\n|$)", FALSE, TRUE);
+
+if (!regex_AUTH) regex_AUTH =
+  regex_must_compile(AUTHS_REGEX, FALSE, TRUE);
+
+#ifndef DISABLE_TLS
+if (!regex_STARTTLS) regex_STARTTLS =
+  regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE);
+#endif
+
+if (!regex_CHUNKING) regex_CHUNKING =
+  regex_must_compile(US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)", FALSE, TRUE);
+
+#ifndef DISABLE_PRDR
+if (!regex_PRDR) regex_PRDR =
+  regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE);
+#endif
+
+#ifdef SUPPORT_I18N
+if (!regex_UTF8) regex_UTF8 =
+  regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE);
+#endif
+
+if (!regex_DSN) regex_DSN  =
+  regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE);
+
+if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA =
+  regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE);
+
+#ifdef SUPPORT_PIPE_CONNECT
+if (!regex_EARLY_PIPE) regex_EARLY_PIPE =
+  regex_must_compile(US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)", FALSE, TRUE);
+#endif
+}
+
+
 /*************************************************
 *             Setup entry point                  *
 *************************************************/
diff --git a/src/src/verify.c b/src/src/verify.c
index 384739b2b..1a44de1ea 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -586,6 +586,10 @@ else
       log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand "
         "callout_random_local_part: %s", expand_string_message);
 
+  /* Compile regex' used by client-side smtp */
+
+  smtp_deliver_init();
+
   /* Default the connect and overall callout timeouts if not set, and record the
   time we are starting so that we can enforce it. */
 
diff --git a/test/log/2120 b/test/log/2120
index 6c4b0cac8..8057655d7 100644
--- a/test/log/2120
+++ b/test/log/2120
@@ -1,8 +1,4 @@
-1999-03-02 09:44:33 Warning: No server certificate defined; will use a selfsigned one.
- Suggested action: either install a certificate or change tls_advertise_hosts option
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@thishost.test.ex U=CALLER P=local S=sss
-1999-03-02 09:44:33 Warning: No server certificate defined; will use a selfsigned one.
- Suggested action: either install a certificate or change tls_advertise_hosts option
 1999-03-02 09:44:33 Start queue run: pid=pppp -qf
 1999-03-02 09:44:33 10HmaX-0005vi-00 [127.0.0.1] SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=Exim Developers/CN=thishost.test.ex
 1999-03-02 09:44:33 10HmaX-0005vi-00 => userx@thishost.test.ex R=abc T=t1 H=thishost.test.ex [127.0.0.1] X=TLS1.x:ke-RSA-AES256-SHAnnn:xxx CV=no DN="/C=UK/O=Exim Developers/CN=thishost.test.ex" C="250 OK id=10HmaY-0005vi-00"