Expanded EXPERIMENTAL_TPDA feature
[exim.git] / src / src / verify.c
index 39ef337871cec9a7ab38afe159663c4ec875cb53..8564aacc2c5bb6bbef7391f9d8b95da3c62d97ef 100644 (file)
@@ -2,7 +2,7 @@
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 
-/* Copyright (c) University of Cambridge 1995 - 2009 */
+/* Copyright (c) University of Cambridge 1995 - 2014 */
 /* See the file NOTICE for conditions of use and distribution. */
 
 /* Functions concerned with verifying things. The original code for callout
@@ -373,10 +373,13 @@ if (!addr->transport)
   {
   HDEBUG(D_verify) debug_printf("cannot callout via null transport\n");
   }
+else if (Ustrcmp(addr->transport->driver_name, "smtp") != 0)
+  log_write(0, LOG_MAIN|LOG_PANIC|LOG_CONFIG_FOR, "callout transport '%s': %s is non-smtp",
+    addr->transport->name, addr->transport->driver_name);
 else
   {
   smtp_transport_options_block *ob =
-    (smtp_transport_options_block *)(addr->transport->options_block);
+    (smtp_transport_options_block *)addr->transport->options_block;
 
   /* The information wasn't available in the cache, so we have to do a real
   callout and save the result in the cache for next time, unless no_cache is set,
@@ -407,13 +410,6 @@ else
 
   if (smtp_out != NULL && !disable_callout_flush) mac_smtp_fflush();
 
-  /* Precompile some regex that are used to recognize parameters in response
-  to an EHLO command, if they aren't already compiled. */
-  #ifdef SUPPORT_TLS
-  if (regex_STARTTLS == NULL) regex_STARTTLS =
-    regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE);
-  #endif
-
   /* Now make connections to the hosts and do real callouts. The list of hosts
   is passed in as an argument. */
 
@@ -466,6 +462,7 @@ else
 
     deliver_host = host->name;
     deliver_host_address = host->address;
+    deliver_host_port = host->port;
     deliver_domain = addr->domain;
 
     if (!smtp_get_interface(tf->interface, host_af, addr, NULL, &interface,
@@ -505,7 +502,12 @@ else
     tls_retry_connection:
 
     inblock.sock = outblock.sock =
-      smtp_connect(host, host_af, port, interface, callout_connect, TRUE);
+      smtp_connect(host, host_af, port, interface, callout_connect, TRUE, NULL
+#ifdef EXPERIMENTAL_TPDA
+    /*XXX tpda action? NULL for now. */
+                 , NULL
+#endif
+                 );
     /* reconsider DSCP here */
     if (inblock.sock < 0)
       {
@@ -528,9 +530,6 @@ else
       else active_hostname = s;
       }
 
-    deliver_host = deliver_host_address = NULL;
-    deliver_domain = save_deliver_domain;
-
     /* Wait for initial response, and send HELO. The smtp_write_command()
     function leaves its command in big_buffer. This is used in error responses.
     Initialize it in case the connection is rejected. */
@@ -540,12 +539,23 @@ else
     /* Unless ssl-on-connect, wait for the initial greeting */
     smtps_redo_greeting:
 
-    #ifdef SUPPORT_TLS
+#ifdef SUPPORT_TLS
     if (!smtps || (smtps && tls_out.active >= 0))
-    #endif
+#endif
+      {
       if (!(done= smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), '2', callout)))
         goto RESPONSE_FAILED;
-    
+
+#ifdef EXPERIMENTAL_TPDA
+      if (tpda_raise_event(addr->transport->tpda_event_action,
+                           US"smtp:connect", responsebuffer) == DEFER)
+       {
+       /* Logging?  Debug? */
+       goto RESPONSE_FAILED;
+       }
+#endif
+      }
+
     /* Not worth checking greeting line for ESMTP support */
     if (!(esmtp = verify_check_this_host(&(ob->hosts_avoid_esmtp), NULL,
       host->name, host->address, NULL) != OK))
@@ -554,14 +564,14 @@ else
 
     tls_redo_helo:
 
-    #ifdef SUPPORT_TLS
+#ifdef SUPPORT_TLS
     if (smtps  &&  tls_out.active < 0) /* ssl-on-connect, first pass */
       {
       tls_offered = TRUE;
       ob->tls_tempfail_tryclear = FALSE;
       }
-      else                             /* all other cases */
-    #endif
+    else                               /* all other cases */
+#endif
 
       { esmtp_retry:
 
@@ -570,24 +580,31 @@ else
         goto SEND_FAILED;
       if (!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), '2', callout))
         {
-        if (errno != 0 || responsebuffer[0] == 0 || lmtp || !esmtp || tls_out.active >= 0)
-       {
-       done= FALSE;
-          goto RESPONSE_FAILED;
-       }
-        #ifdef SUPPORT_TLS
+       if (errno != 0 || responsebuffer[0] == 0 || lmtp || !esmtp || tls_out.active >= 0)
+         {
+         done= FALSE;
+         goto RESPONSE_FAILED;
+         }
+#ifdef SUPPORT_TLS
         tls_offered = FALSE;
-        #endif
+#endif
         esmtp = FALSE;
         goto esmtp_retry;                      /* fallback to HELO */
         }
 
       /* Set tls_offered if the response to EHLO specifies support for STARTTLS. */
-      #ifdef SUPPORT_TLS
-      tls_offered = esmtp && !suppress_tls &&  tls_out.active < 0  &&
-        pcre_exec(regex_STARTTLS, NULL, CS responsebuffer, Ustrlen(responsebuffer), 0,
-          PCRE_EOPT, NULL, 0) >= 0;
-      #endif
+#ifdef SUPPORT_TLS
+      if (esmtp && !suppress_tls &&  tls_out.active < 0)
+       {
+       if (regex_STARTTLS == NULL) regex_STARTTLS =
+         regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE);
+
+       tls_offered = pcre_exec(regex_STARTTLS, NULL, CS responsebuffer,
+                     Ustrlen(responsebuffer), 0, PCRE_EOPT, NULL, 0) >= 0;
+       }
+      else
+        tls_offered = FALSE;
+#endif
       }
 
     /* If TLS is available on this connection attempt to
@@ -598,10 +615,13 @@ else
     the client not be required to use TLS. If the response is bad, copy the buffer
     for error analysis. */
 
-    #ifdef SUPPORT_TLS
+#ifdef SUPPORT_TLS
     if (tls_offered &&
        verify_check_this_host(&(ob->hosts_avoid_tls), NULL, host->name,
-         host->address, NULL) != OK)
+         host->address, NULL) != OK &&
+       verify_check_this_host(&(ob->hosts_verify_avoid_tls), NULL, host->name,
+         host->address, NULL) != OK
+       )
       {
       uschar buffer2[4096];
       if (  !smtps
@@ -620,45 +640,47 @@ else
         {
         if (errno != 0 || buffer2[0] == 0 ||
                (buffer2[0] == '4' && !ob->tls_tempfail_tryclear))
-       {
-       Ustrncpy(responsebuffer, buffer2, sizeof(responsebuffer));
-       done= FALSE;
-       goto RESPONSE_FAILED;
-       }
+         {
+         Ustrncpy(responsebuffer, buffer2, sizeof(responsebuffer));
+         done= FALSE;
+         goto RESPONSE_FAILED;
+         }
         }
 
        /* STARTTLS accepted or ssl-on-connect: try to negotiate a TLS session. */
       else
         {
-        int rc = tls_client_start(inblock.sock, host, addr,
-        NULL,                    /* No DH param */
-        ob->tls_certificate, ob->tls_privatekey,
-        ob->tls_sni,
-        ob->tls_verify_certificates, ob->tls_crl,
-        ob->tls_require_ciphers,
-        ob->gnutls_require_mac, ob->gnutls_require_kx, ob->gnutls_require_proto,
-        callout);
+       int oldtimeout = ob->command_timeout;
+       int rc;
+
+       ob->command_timeout = callout;
+        rc = tls_client_start(inblock.sock, host, addr, addr->transport);
+       ob->command_timeout = oldtimeout;
 
         /* TLS negotiation failed; give an error.  Try in clear on a new connection,
            if the options permit it for this host. */
         if (rc != OK)
           {
-       if (rc == DEFER && ob->tls_tempfail_tryclear && !smtps &&
-          verify_check_this_host(&(ob->hosts_require_tls), NULL, host->name,
-            host->address, NULL) != OK)
-         {
-            (void)close(inblock.sock);
-         log_write(0, LOG_MAIN, "TLS session failure: delivering unencrypted "
-           "to %s [%s] (not in hosts_require_tls)", host->name, host->address);
-         suppress_tls = TRUE;
-         goto tls_retry_connection;
-         }
-       /*save_errno = ERRNO_TLSFAILURE;*/
-       /*message = US"failure while setting up TLS session";*/
-       send_quit = FALSE;
-       done= FALSE;
-       goto TLS_FAILED;
-       }
+         if (rc == DEFER && ob->tls_tempfail_tryclear && !smtps &&
+            verify_check_this_host(&(ob->hosts_require_tls), NULL, host->name,
+              host->address, NULL) != OK)
+           {
+           (void)close(inblock.sock);
+#ifdef EXPERIMENTAL_TPDA
+           (void) tpda_raise_event(addr->transport->tpda_event_action,
+                                   US"tcp:close", NULL);
+#endif
+           log_write(0, LOG_MAIN, "TLS session failure: delivering unencrypted "
+             "to %s [%s] (not in hosts_require_tls)", host->name, host->address);
+           suppress_tls = TRUE;
+           goto tls_retry_connection;
+           }
+         /*save_errno = ERRNO_TLSFAILURE;*/
+         /*message = US"failure while setting up TLS session";*/
+         send_quit = FALSE;
+         done= FALSE;
+         goto TLS_FAILED;
+         }
 
         /* TLS session is set up.  Copy info for logging. */
         addr->cipher = tls_out.cipher;
@@ -666,7 +688,7 @@ else
 
         /* For SMTPS we need to wait for the initial OK response, then do HELO. */
         if (smtps)
-        goto smtps_redo_greeting;
+         goto smtps_redo_greeting;
 
         /* For STARTTLS we need to redo EHLO */
         goto tls_redo_helo;
@@ -690,13 +712,25 @@ else
 
     done = TRUE; /* so far so good; have response to HELO */
 
-    /*XXX the EHLO response would be analyzed here for IGNOREQUOTA, SIZE, PIPELINING, AUTH */
-    /* If we haven't authenticated, but are required to, give up. */
+    /*XXX the EHLO response would be analyzed here for IGNOREQUOTA, SIZE, PIPELINING */
 
-    /*XXX "filter command specified for this transport" ??? */
-    /* for now, transport_filter by cutthrough-delivery is not supported */
+    /* For now, transport_filter by cutthrough-delivery is not supported */
     /* Need proper integration with the proper transport mechanism. */
-
+    if (cutthrough_delivery)
+      {
+      if (addr->transport->filter_command)
+        {
+        cutthrough_delivery= FALSE;
+        HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of transport filter\n");
+        }
+#ifndef DISABLE_DKIM
+      if (ob->dkim_domain)
+        {
+        cutthrough_delivery= FALSE;
+        HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of DKIM signing\n");
+        }
+#endif
+      }
 
     SEND_FAILED:
     RESPONSE_FAILED:
@@ -704,7 +738,6 @@ else
     ;
     /* Clear down of the TLS, SMTP and TCP layers on error is handled below.  */
 
-
     /* Failure to accept HELO is cached; this blocks the whole domain for all
     senders. I/O errors and defer responses are not cached. */
 
@@ -718,14 +751,33 @@ else
         }
       }
 
+    /* If we haven't authenticated, but are required to, give up. */
+    /* Try to AUTH */
+
+    else done = smtp_auth(responsebuffer, sizeof(responsebuffer),
+       addr, host, ob, esmtp, &inblock, &outblock) == OK  &&
+
+               /* Copy AUTH info for logging */
+      ( (addr->authenticator = client_authenticator),
+        (addr->auth_id = client_authenticated_id),
+
+    /* Build a mail-AUTH string (re-using responsebuffer for convenience */
+        !smtp_mail_auth_str(responsebuffer, sizeof(responsebuffer), addr, ob)
+      )  &&
+
+      ( (addr->auth_sndr = client_authenticated_sender),
+
     /* Send the MAIL command */
+        (smtp_write_command(&outblock, FALSE, "MAIL FROM:<%s>%s\r\n",
+          from_address, responsebuffer) >= 0)
+      )  &&
 
-    else done =
-      smtp_write_command(&outblock, FALSE, "MAIL FROM:<%s>\r\n",
-        from_address) >= 0 &&
       smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer),
         '2', callout);
 
+    deliver_host = deliver_host_address = NULL;
+    deliver_domain = save_deliver_domain;
+
     /* If the host does not accept MAIL FROM:<>, arrange to cache this
     information, but again, don't record anything for an I/O error or a defer. Do
     not cache rejections of MAIL when a non-empty sender has been used, because
@@ -943,6 +995,7 @@ else
       {
       cutthrough_fd= outblock.sock;    /* We assume no buffer in use in the outblock */
       cutthrough_addr = *addr;         /* Save the address_item for later logging */
+      cutthrough_addr.next =     NULL;
       cutthrough_addr.host_used = store_get(sizeof(host_item));
       cutthrough_addr.host_used->name =    host->name;
       cutthrough_addr.host_used->address = host->address;
@@ -957,14 +1010,19 @@ else
       }
     else
       {
+      /* Ensure no cutthrough on multiple address verifies */
       if (options & vopt_callout_recipsender)
-        cancel_cutthrough_connection();        /* Ensure no cutthrough on multiple address verifies */
+        cancel_cutthrough_connection("multiple verify calls");
       if (send_quit) (void)smtp_write_command(&outblock, FALSE, "QUIT\r\n");
 
-      #ifdef SUPPORT_TLS
+#ifdef SUPPORT_TLS
       tls_close(FALSE, TRUE);
-      #endif
+#endif
       (void)close(inblock.sock);
+#ifdef EXPERIMENTAL_TPDA
+      (void) tpda_raise_event(addr->transport->tpda_event_action,
+                             US"tcp:close", NULL);
+#endif
       }
 
     }    /* Loop through all hosts, while !done */
@@ -1129,7 +1187,7 @@ cutthrough_puts(uschar * cp, int n)
 {
 if (cutthrough_fd < 0)       return TRUE;
 if (_cutthrough_puts(cp, n)) return TRUE;
-cancel_cutthrough_connection();
+cancel_cutthrough_connection("transmit failed");
 return FALSE;
 }
 
@@ -1151,7 +1209,7 @@ BOOL
 cutthrough_flush_send( void )
 {
 if (_cutthrough_flush_send()) return TRUE;
-cancel_cutthrough_connection();
+cancel_cutthrough_connection("transmit failed");
 return FALSE;
 }
 
@@ -1178,7 +1236,7 @@ inblock.ptrend = inbuffer;
 inblock.sock = cutthrough_fd;
 /* this relies on (inblock.sock == tls_out.active) */
 if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, CUTTHROUGH_DATA_TIMEOUT))
-  cancel_cutthrough_connection();
+  cancel_cutthrough_connection("target timeout on read");
 
 if(copy != NULL)
   {
@@ -1210,32 +1268,48 @@ return cutthrough_response('3', NULL) == '3';
 }
 
 
+/* fd and use_crlf args only to match write_chunk() */
+static BOOL
+cutthrough_write_chunk(int fd, uschar * s, int len, BOOL use_crlf)
+{
+uschar * s2;
+while(s && (s2 = Ustrchr(s, '\n')))
+ {
+ if(!cutthrough_puts(s, s2-s) || !cutthrough_put_nl())
+  return FALSE;
+ s = s2+1;
+ }
+return TRUE;
+}
+
+
 /* Buffered send of headers.  Return success boolean. */
 /* Expands newlines to wire format (CR,NL).           */
 /* Also sends header-terminating blank line.          */
 BOOL
 cutthrough_headers_send( void )
 {
-header_line * h;
-uschar * cp1, * cp2;
-
 if(cutthrough_fd < 0)
   return FALSE;
 
-for(h= header_list; h != NULL; h= h->next)
-  if(h->type != htype_old  &&  h->text != NULL)
-    for (cp1 = h->text; *cp1 && (cp2 = Ustrchr(cp1, '\n')); cp1 = cp2+1)
-      if(  !cutthrough_puts(cp1, cp2-cp1)
-        || !cutthrough_put_nl())
-        return FALSE;
+/* We share a routine with the mainline transport to handle header add/remove/rewrites,
+   but having a separate buffered-output function (for now)
+*/
+HDEBUG(D_acl) debug_printf("----------- start cutthrough headers send -----------\n");
+
+if (!transport_headers_send(&cutthrough_addr, cutthrough_fd,
+       cutthrough_addr.transport->add_headers, cutthrough_addr.transport->remove_headers,
+       &cutthrough_write_chunk, TRUE,
+       cutthrough_addr.transport->rewrite_rules, cutthrough_addr.transport->rewrite_existflags))
+  return FALSE;
 
-HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP>>(nl)\n");
-return cutthrough_put_nl();
+HDEBUG(D_acl) debug_printf("----------- done cutthrough headers send ------------\n");
+return TRUE;
 }
 
 
 static void
-close_cutthrough_connection( void )
+close_cutthrough_connection( const char * why )
 {
 if(cutthrough_fd >= 0)
   {
@@ -1254,15 +1328,15 @@ if(cutthrough_fd >= 0)
   #endif
   (void)close(cutthrough_fd);
   cutthrough_fd= -1;
-  HDEBUG(D_acl) debug_printf("----------- cutthrough shutdown ------------\n");
+  HDEBUG(D_acl) debug_printf("----------- cutthrough shutdown (%s) ------------\n", why);
   }
 ctblock.ptr = ctbuffer;
 }
 
 void
-cancel_cutthrough_connection( void )
+cancel_cutthrough_connection( const char * why )
 {
-close_cutthrough_connection();
+close_cutthrough_connection(why);
 cutthrough_delivery= FALSE;
 }
 
@@ -1287,7 +1361,7 @@ switch(cutthrough_response('2', &cutthrough_addr.message))
   {
   case '2':
     delivery_log(LOG_MAIN, &cutthrough_addr, (int)'>', NULL);
-    close_cutthrough_connection();
+    close_cutthrough_connection("delivered");
     break;
 
   case '4':
@@ -1518,6 +1592,12 @@ addresses, such rewriting fails. */
 
 if (address[0] == 0) return OK;
 
+/* Flip the legacy TLS-related variables over to the outbound set in case
+they're used in the context of a transport used by verification. Reset them
+at exit from this routine. */
+
+tls_modify_variables(&tls_out);
+
 /* Save a copy of the sender address for re-instating if we change it to <>
 while verifying a sender address (a nice bit of self-reference there). */
 
@@ -1689,8 +1769,20 @@ while (addr_new != NULL)
                   string_is_ip_address(host->name, NULL) != 0)
                 (void)host_find_byname(host, NULL, flags, &canonical_name, TRUE);
               else
+               {
+               uschar * d_request = NULL, * d_require = NULL;
+               if (Ustrcmp(addr->transport->driver_name, "smtp") == 0)
+                 {
+                 smtp_transport_options_block * ob =
+                     (smtp_transport_options_block *)
+                       addr->transport->options_block;
+                 d_request = ob->dnssec_request_domains;
+                 d_require = ob->dnssec_require_domains;
+                 }
+
                 (void)host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
-                  &canonical_name, NULL);
+                 d_request, d_require, &canonical_name, NULL);
+               }
               }
             }
           }
@@ -1710,6 +1802,9 @@ while (addr_new != NULL)
           }
         else
           {
+#ifdef SUPPORT_TLS
+         deliver_set_expansions(addr);
+#endif
           rc = do_callout(addr, host_list, &tf, callout, callout_overall,
             callout_connect, options, se_mailfrom, pm_mailfrom);
           }
@@ -1761,10 +1856,14 @@ while (addr_new != NULL)
         }
       respond_printf(f, "%s\n", cr);
       }
-    cancel_cutthrough_connection();
+    cancel_cutthrough_connection("routing hard fail");
 
-    if (!full_info) return copy_error(vaddr, addr, FAIL);
-      else yield = FAIL;
+    if (!full_info)
+    {
+      yield = copy_error(vaddr, addr, FAIL);
+      goto out;
+    }
+    else yield = FAIL;
     }
 
   /* Soft failure */
@@ -1796,10 +1895,14 @@ while (addr_new != NULL)
         }
       respond_printf(f, "%s\n", cr);
       }
-    cancel_cutthrough_connection();
+    cancel_cutthrough_connection("routing soft fail");
 
-    if (!full_info) return copy_error(vaddr, addr, DEFER);
-      else if (yield == OK) yield = DEFER;
+    if (!full_info)
+      {
+      yield = copy_error(vaddr, addr, DEFER);
+      goto out;
+      }
+    else if (yield == OK) yield = DEFER;
     }
 
   /* If we are handling EXPN, we do not want to continue to route beyond
@@ -1822,7 +1925,8 @@ while (addr_new != NULL)
       if (addr_new == NULL) ok_prefix = US"250 ";
       respond_printf(f, "%s<%s>\r\n", ok_prefix, addr2->address);
       }
-    return OK;
+    yield = OK;
+    goto out;
     }
 
   /* Successful routing other than EXPN. */
@@ -1857,7 +1961,8 @@ while (addr_new != NULL)
       of $address_data to be that of the child */
 
       vaddr->p.address_data = addr->p.address_data;
-      return OK;
+      yield = OK;
+      goto out;
       }
     }
   }     /* Loop for generated addresses */
@@ -1874,7 +1979,7 @@ discarded, usually because of the use of :blackhole: in an alias file. */
 if (allok && addr_local == NULL && addr_remote == NULL)
   {
   fprintf(f, "mail to %s is discarded\n", address);
-  return yield;
+  goto out;
   }
 
 for (addr_list = addr_local, i = 0; i < 2; addr_list = addr_remote, i++)
@@ -1958,9 +2063,12 @@ for (addr_list = addr_local, i = 0; i < 2; addr_list = addr_remote, i++)
     }
   }
 
-/* Will be DEFER or FAIL if any one address has, only for full_info (which is
+/* Yield will be DEFER or FAIL if any one address has, only for full_info (which is
 the -bv or -bt case). */
 
+out:
+tls_modify_variables(&tls_in);
+
 return yield;
 }
 
@@ -2089,6 +2197,41 @@ return yield;
 }
 
 
+/*************************************************
+*      Check header names for 8-bit characters   *
+*************************************************/
+
+/* This function checks for invalid charcters in header names. See
+RFC 5322, 2.2. and RFC 6532, 3.
+
+Arguments:
+  msgptr     where to put an error message
+
+Returns:     OK
+             FAIL
+*/
+
+int
+verify_check_header_names_ascii(uschar **msgptr)
+{
+header_line *h;
+uschar *colon, *s;
+
+for (h = header_list; h != NULL; h = h->next)
+  {
+   colon = Ustrchr(h->text, ':');
+   for(s = h->text; s < colon; s++)
+     {
+        if ((*s < 33) || (*s > 126))
+        {
+                *msgptr = string_sprintf("Invalid character in header \"%.*s\" found",
+                                         colon - h->text, h->text);
+                return FAIL;
+        }
+     }
+  }
+return OK;
+}
 
 /*************************************************
 *          Check for blind recipients            *
@@ -3435,7 +3578,7 @@ revadd[0] = 0;
 
 /* In case this is the first time the DNS resolver is being used. */
 
-dns_init(FALSE, FALSE);
+dns_init(FALSE, FALSE, FALSE); /*XXX dnssec? */
 
 /* Loop through all the domains supplied, until something matches */
 
@@ -3606,4 +3749,6 @@ while ((domain = string_nextinlist(&list, &sep, buffer, sizeof(buffer))) != NULL
 return FAIL;
 }
 
+/* vi: aw ai sw=2
+*/
 /* End of verify.c */