Add new probe_ssl_last_chain_expiry_timestamp_seconds metric (#636)
authorTakuya Kosugiyama <re@itkq.jp>
Wed, 10 Jun 2020 08:20:54 +0000 (17:20 +0900)
committerGitHub <noreply@github.com>
Wed, 10 Jun 2020 08:20:54 +0000 (09:20 +0100)
* Add new probe_ssl_latest_verified_chain_expiry metric

Resolves https://github.com/prometheus/blackbox_exporter/issues/340

Based on the discussion in the issue above, this metric will help determine
when the SSL/TLS certificate expiration error actually happens on clients
like a browser that attempts to verify certificates by building one or
more chains from peer certificates.

Signed-off-by: Takuya Kosugiyama <re@itkq.jp>
prober/http.go
prober/http_test.go
prober/tcp.go
prober/tcp_test.go
prober/tls.go
prober/utils_test.go

index f4abdbe119e3c678151be186a6f2099a5cee13eb..42552cbb0561ff680bc525ed3d7aa22b330ac78f 100644 (file)
@@ -261,6 +261,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
                        Help: "Returns earliest SSL cert expiry in unixtime",
                })
 
+               probeSSLLastChainExpiryTimestampSeconds = prometheus.NewGauge(prometheus.GaugeOpts{
+                       Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
+                       Help: "Returns last SSL chain expiry in timestamp seconds",
+               })
+
                probeTLSVersion = prometheus.NewGaugeVec(
                        prometheus.GaugeOpts{
                                Name: "probe_tls_version_info",
@@ -546,9 +551,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 
        if resp.TLS != nil {
                isSSLGauge.Set(float64(1))
-               registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion)
+               registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
                probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).Unix()))
                probeTLSVersion.WithLabelValues(getTLSVersion(resp.TLS)).Set(1)
+               probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix()))
                if httpConfig.FailIfSSL {
                        level.Error(logger).Log("msg", "Final request was over SSL")
                        success = false
index 78f10ca7c5e47ab3361da4360ce779a7e669e3cc..3d3f63e7e584a96ace0225d1558d9b4f23857dc1 100644 (file)
@@ -16,6 +16,8 @@ package prober
 import (
        "context"
        "crypto/tls"
+       "crypto/x509"
+       "encoding/pem"
        "fmt"
        "io/ioutil"
        "net/http"
@@ -597,7 +599,9 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) {
 func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
        // Create test certificates valid for 1 day.
        certExpiry := time.Now().AddDate(0, 0, 1)
-       testcertPem, testKeyPem := generateTestCertificate(certExpiry, false)
+       testCertTmpl := generateCertificateTemplate(certExpiry, false)
+       testCertTmpl.IsCA = true
+       _, testcertPem, testKey := generateSelfSignedCertificate(testCertTmpl)
 
        // CAFile must be passed via filesystem, use a tempfile.
        tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
@@ -612,6 +616,7 @@ func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
        }
        defer os.Remove(tmpCaFile.Name())
 
+       testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
        testcert, err := tls.X509KeyPair(testcertPem, testKeyPem)
        if err != nil {
                panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
index a450fd28de328af2051f64de75f2a46705633083..b5f5addd8bf8124a860d736e4c558ebff7eb1e99 100644 (file)
@@ -94,6 +94,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
                Name: "probe_ssl_earliest_cert_expiry",
                Help: "Returns earliest SSL cert expiry date",
        })
+       probeSSLLastChainExpiryTimestampSeconds := prometheus.NewGauge(prometheus.GaugeOpts{
+               Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
+               Help: "Returns last SSL chain expiry in unixtime",
+       })
        probeTLSVersion := prometheus.NewGaugeVec(
                prometheus.GaugeOpts{
                        Name: "probe_tls_version_info",
@@ -125,9 +129,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
        }
        if module.TCP.TLS {
                state := conn.(*tls.Conn).ConnectionState()
-               registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion)
+               registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
                probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
                probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
+               probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
        }
        scanner := bufio.NewScanner(conn)
        for i, qr := range module.TCP.QueryResponse {
@@ -194,9 +199,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 
                        // Get certificate expiry.
                        state := tlsConn.ConnectionState()
-                       registry.MustRegister(probeSSLEarliestCertExpiry)
+                       registry.MustRegister(probeSSLEarliestCertExpiry, probeSSLLastChainExpiryTimestampSeconds)
                        probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
                        probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
+                       probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
                }
        }
        return true
index 02401f618282d6c7f5aadcae9cbedc660fe5a690..d05f317454735e30fc538360018cadc92044810c 100644 (file)
 package prober
 
 import (
+       "bytes"
        "context"
+       "crypto/rand"
+       "crypto/rsa"
        "crypto/tls"
+       "crypto/x509"
+       "encoding/pem"
        "fmt"
        "io/ioutil"
        "net"
@@ -84,14 +89,16 @@ func TestTCPConnectionWithTLS(t *testing.T) {
 
        // Create test certificates valid for 1 day.
        certExpiry := time.Now().AddDate(0, 0, 1)
-       testcert_pem, testkey_pem := generateTestCertificate(certExpiry, false)
+       rootCertTmpl := generateCertificateTemplate(certExpiry, false)
+       rootCertTmpl.IsCA = true
+       _, rootCertPem, rootKey := generateSelfSignedCertificate(rootCertTmpl)
 
        // CAFile must be passed via filesystem, use a tempfile.
        tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
        if err != nil {
                t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
        }
-       if _, err := tmpCaFile.Write(testcert_pem); err != nil {
+       if _, err := tmpCaFile.Write(rootCertPem); err != nil {
                t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
        }
        if err := tmpCaFile.Close(); err != nil {
@@ -109,7 +116,8 @@ func TestTCPConnectionWithTLS(t *testing.T) {
                }
                defer conn.Close()
 
-               testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
+               rootKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey)})
+               testcert, err := tls.X509KeyPair(rootCertPem, rootKeyPem)
                if err != nil {
                        panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
                }
@@ -193,6 +201,127 @@ func TestTCPConnectionWithTLS(t *testing.T) {
        checkRegistryResults(expectedResults, mfs, t)
 }
 
+func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
+       if os.Getenv("TRAVIS") == "true" {
+               t.Skip("skipping; travisci is failing on ipv6 dns requests")
+       }
+
+       ln, err := net.Listen("tcp", "127.0.0.1:0")
+       if err != nil {
+               t.Fatalf("Error listening on socket: %s", err)
+       }
+       defer ln.Close()
+       _, listenPort, _ := net.SplitHostPort(ln.Addr().String())
+
+       testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+       defer cancel()
+
+       // From here prepare two certificate chains where one is expired
+
+       rootPrivatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+       if err != nil {
+               panic(fmt.Sprintf("Error creating rsa key: %s", err))
+       }
+
+       rootCertExpiry := time.Now().AddDate(0, 0, 2)
+       rootCertTmpl := generateCertificateTemplate(rootCertExpiry, false)
+       rootCertTmpl.IsCA = true
+       _, rootCertPem := generateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivatekey)
+
+       oldRootCertExpiry := time.Now().AddDate(0, 0, -1)
+       expiredRootCertTmpl := generateCertificateTemplate(oldRootCertExpiry, false)
+       expiredRootCertTmpl.IsCA = true
+       expiredRootCert, expiredRootCertPem := generateSelfSignedCertificateWithPrivateKey(expiredRootCertTmpl, rootPrivatekey)
+
+       serverCertExpiry := time.Now().AddDate(0, 0, 1)
+       serverCertTmpl := generateCertificateTemplate(serverCertExpiry, false)
+       _, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, expiredRootCert, rootPrivatekey)
+
+       // CAFile must be passed via filesystem, use a tempfile.
+       tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
+       if err != nil {
+               t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
+       }
+       if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, expiredRootCertPem}, []byte("\n"))); err != nil {
+               t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
+       }
+       if err := tmpCaFile.Close(); err != nil {
+               t.Fatalf(fmt.Sprintf("Error closing CA tempfile: %s", err))
+       }
+       defer os.Remove(tmpCaFile.Name())
+
+       ch := make(chan (struct{}))
+       logger := log.NewNopLogger()
+       // Handle server side of this test.
+       serverFunc := func() {
+               conn, err := ln.Accept()
+               if err != nil {
+                       panic(fmt.Sprintf("Error accepting on socket: %s", err))
+               }
+               defer conn.Close()
+
+               serverKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)})
+
+               keypair, err := tls.X509KeyPair(serverCertPem, serverKeyPem)
+               if err != nil {
+                       panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
+               }
+
+               // Immediately upgrade to TLS.
+               tlsConfig := &tls.Config{
+                       ServerName:   "localhost",
+                       Certificates: []tls.Certificate{keypair},
+                       MinVersion:   tls.VersionTLS12,
+                       MaxVersion:   tls.VersionTLS12,
+               }
+               tlsConn := tls.Server(conn, tlsConfig)
+               defer tlsConn.Close()
+               if err := tlsConn.Handshake(); err != nil {
+                       level.Error(logger).Log("msg", "Error TLS Handshake (server) failed", "err", err)
+               } else {
+                       // Send some bytes before terminating the connection.
+                       fmt.Fprintf(tlsConn, "Hello World!\n")
+               }
+               ch <- struct{}{}
+       }
+
+       // Expect name-verified TLS connection.
+       module := config.Module{
+               TCP: config.TCPProbe{
+                       IPProtocol:         "ip4",
+                       IPProtocolFallback: true,
+                       TLS:                true,
+                       TLSConfig: pconfig.TLSConfig{
+                               CAFile:             tmpCaFile.Name(),
+                               InsecureSkipVerify: false,
+                       },
+               },
+       }
+
+       registry := prometheus.NewRegistry()
+       go serverFunc()
+       // Test name-verification with name from target.
+       target := net.JoinHostPort("localhost", listenPort)
+       if !ProbeTCP(testCTX, target, module, registry, log.NewNopLogger()) {
+               t.Fatalf("TCP module failed, expected success.")
+       }
+       <-ch
+
+       // Check the resulting metrics.
+       mfs, err := registry.Gather()
+       if err != nil {
+               t.Fatal(err)
+       }
+
+       // Check values
+       expectedResults := map[string]float64{
+               "probe_ssl_earliest_cert_expiry":                float64(serverCertExpiry.Unix()),
+               "probe_ssl_last_chain_expiry_timestamp_seconds": float64(serverCertExpiry.Unix()),
+               "probe_tls_version_info":                        1,
+       }
+       checkRegistryResults(expectedResults, mfs, t)
+}
+
 func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
        ln, err := net.Listen("tcp", "localhost:0")
        if err != nil {
@@ -205,14 +334,16 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
 
        // Create test certificates valid for 1 day.
        certExpiry := time.Now().AddDate(0, 0, 1)
-       testcert_pem, testkey_pem := generateTestCertificate(certExpiry, true)
+       testCertTmpl := generateCertificateTemplate(certExpiry, true)
+       testCertTmpl.IsCA = true
+       _, testCertPem, testKey := generateSelfSignedCertificate(testCertTmpl)
 
        // CAFile must be passed via filesystem, use a tempfile.
        tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
        if err != nil {
                t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
        }
-       if _, err := tmpCaFile.Write(testcert_pem); err != nil {
+       if _, err := tmpCaFile.Write(testCertPem); err != nil {
                t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
        }
        if err := tmpCaFile.Close(); err != nil {
@@ -263,7 +394,8 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
                }
                fmt.Fprintf(conn, "220 2.0.0 Ready to start TLS\n")
 
-               testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
+               testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
+               testcert, err := tls.X509KeyPair(testCertPem, testKeyPem)
                if err != nil {
                        panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
                }
index 758cbf35d44888960557f2d752c9d17df48048f7..a4ef0f9a8a39d2c5ad548c8e61c0c0d91c2bd8a5 100644 (file)
@@ -28,6 +28,23 @@ func getEarliestCertExpiry(state *tls.ConnectionState) time.Time {
        return earliest
 }
 
+func getLastChainExpiry(state *tls.ConnectionState) time.Time {
+       lastChainExpiry := time.Time{}
+       for _, chain := range state.VerifiedChains {
+               earliestCertExpiry := time.Time{}
+               for _, cert := range chain {
+                       if (earliestCertExpiry.IsZero() || cert.NotAfter.Before(earliestCertExpiry)) && !cert.NotAfter.IsZero() {
+                               earliestCertExpiry = cert.NotAfter
+                       }
+               }
+               if lastChainExpiry.IsZero() || lastChainExpiry.After(earliestCertExpiry) {
+                       lastChainExpiry = earliestCertExpiry
+               }
+
+       }
+       return lastChainExpiry
+}
+
 func getTLSVersion(state *tls.ConnectionState) string {
        switch state.Version {
        case tls.VersionTLS10:
index 42dfd9eb8ab3ad7c1cb5a5954402064233a58cbf..2ac38eebe4e5e96dfd825fa04858aac9ab7fee46 100644 (file)
@@ -79,18 +79,8 @@ func checkRegistryLabels(expRes map[string]map[string]string, mfs []*dto.MetricF
        }
 }
 
-// Create test certificate with specified expiry date
-// Certificate will be self-signed and use localhost/127.0.0.1
-// Generated certificate and key are returned in PEM encoding
-func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byte) {
-       privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
-       if err != nil {
-               panic(fmt.Sprintf("Error creating rsa key: %s", err))
-       }
-       publickey := &privatekey.PublicKey
-
-       cert := x509.Certificate{
-               IsCA:                  true,
+func generateCertificateTemplate(expiry time.Time, IPAddressSAN bool) *x509.Certificate {
+       template := &x509.Certificate{
                BasicConstraintsValid: true,
                SubjectKeyId:          []byte{1},
                SerialNumber:          big.NewInt(1),
@@ -102,18 +92,54 @@ func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byt
                ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
                KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
        }
-       cert.DNSNames = append(cert.DNSNames, "localhost")
+
+       template.DNSNames = append(template.DNSNames, "localhost")
        if IPAddressSAN {
-               cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("127.0.0.1"))
-               cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("::1"))
+               template.IPAddresses = append(template.IPAddresses, net.ParseIP("127.0.0.1"))
+               template.IPAddresses = append(template.IPAddresses, net.ParseIP("::1"))
        }
-       derCert, err := x509.CreateCertificate(rand.Reader, &cert, &cert, publickey, privatekey)
+
+       return template
+}
+
+func generateCertificate(template, parent *x509.Certificate, publickey *rsa.PublicKey, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
+       derCert, err := x509.CreateCertificate(rand.Reader, template, template, publickey, privatekey)
        if err != nil {
                panic(fmt.Sprintf("Error signing test-certificate: %s", err))
        }
+       cert, err := x509.ParseCertificate(derCert)
+       if err != nil {
+               panic(fmt.Sprintf("Error parsing test-certificate: %s", err))
+       }
        pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert})
-       pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privatekey)})
-       return pemCert, pemKey
+       return cert, pemCert
+
+}
+
+func generateSignedCertificate(template, parentCert *x509.Certificate, parentKey *rsa.PrivateKey) (*x509.Certificate, []byte, *rsa.PrivateKey) {
+       privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+       if err != nil {
+               panic(fmt.Sprintf("Error creating rsa key: %s", err))
+       }
+       cert, pemCert := generateCertificate(template, parentCert, &privatekey.PublicKey, parentKey)
+       return cert, pemCert, privatekey
+}
+
+func generateSelfSignedCertificate(template *x509.Certificate) (*x509.Certificate, []byte, *rsa.PrivateKey) {
+       privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+       if err != nil {
+               panic(fmt.Sprintf("Error creating rsa key: %s", err))
+       }
+       publickey := &privatekey.PublicKey
+
+       cert, pemCert := generateCertificate(template, template, publickey, privatekey)
+       return cert, pemCert, privatekey
+}
+
+func generateSelfSignedCertificateWithPrivateKey(template *x509.Certificate, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
+       publickey := &privatekey.PublicKey
+       cert, pemCert := generateCertificate(template, template, publickey, privatekey)
+       return cert, pemCert
 }
 
 func TestChooseProtocol(t *testing.T) {