From 725a683985217d871319b2148586048eb48f95a3 Mon Sep 17 00:00:00 2001 From: Takuya Kosugiyama Date: Wed, 10 Jun 2020 17:20:54 +0900 Subject: [PATCH] Add new probe_ssl_last_chain_expiry_timestamp_seconds metric (#636) * Add new probe_ssl_latest_verified_chain_expiry metric Resolves https://github.com/prometheus/blackbox_exporter/issues/340 Based on the discussion in the issue above, this metric will help determine when the SSL/TLS certificate expiration error actually happens on clients like a browser that attempts to verify certificates by building one or more chains from peer certificates. Signed-off-by: Takuya Kosugiyama --- prober/http.go | 8 ++- prober/http_test.go | 7 ++- prober/tcp.go | 10 ++- prober/tcp_test.go | 144 +++++++++++++++++++++++++++++++++++++++++-- prober/tls.go | 17 +++++ prober/utils_test.go | 62 +++++++++++++------ 6 files changed, 220 insertions(+), 28 deletions(-) diff --git a/prober/http.go b/prober/http.go index f4abdbe..42552cb 100644 --- a/prober/http.go +++ b/prober/http.go @@ -261,6 +261,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr Help: "Returns earliest SSL cert expiry in unixtime", }) + probeSSLLastChainExpiryTimestampSeconds = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ssl_last_chain_expiry_timestamp_seconds", + Help: "Returns last SSL chain expiry in timestamp seconds", + }) + probeTLSVersion = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "probe_tls_version_info", @@ -546,9 +551,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr if resp.TLS != nil { isSSLGauge.Set(float64(1)) - registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion) + registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds) probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).Unix())) probeTLSVersion.WithLabelValues(getTLSVersion(resp.TLS)).Set(1) + probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix())) if httpConfig.FailIfSSL { level.Error(logger).Log("msg", "Final request was over SSL") success = false diff --git a/prober/http_test.go b/prober/http_test.go index 78f10ca..3d3f63e 100644 --- a/prober/http_test.go +++ b/prober/http_test.go @@ -16,6 +16,8 @@ package prober import ( "context" "crypto/tls" + "crypto/x509" + "encoding/pem" "fmt" "io/ioutil" "net/http" @@ -597,7 +599,9 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) { func TestHTTPUsesTargetAsTLSServerName(t *testing.T) { // Create test certificates valid for 1 day. certExpiry := time.Now().AddDate(0, 0, 1) - testcertPem, testKeyPem := generateTestCertificate(certExpiry, false) + testCertTmpl := generateCertificateTemplate(certExpiry, false) + testCertTmpl.IsCA = true + _, testcertPem, testKey := generateSelfSignedCertificate(testCertTmpl) // CAFile must be passed via filesystem, use a tempfile. tmpCaFile, err := ioutil.TempFile("", "cafile.pem") @@ -612,6 +616,7 @@ func TestHTTPUsesTargetAsTLSServerName(t *testing.T) { } defer os.Remove(tmpCaFile.Name()) + testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)}) testcert, err := tls.X509KeyPair(testcertPem, testKeyPem) if err != nil { panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err)) diff --git a/prober/tcp.go b/prober/tcp.go index a450fd2..b5f5add 100644 --- a/prober/tcp.go +++ b/prober/tcp.go @@ -94,6 +94,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry Name: "probe_ssl_earliest_cert_expiry", Help: "Returns earliest SSL cert expiry date", }) + probeSSLLastChainExpiryTimestampSeconds := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ssl_last_chain_expiry_timestamp_seconds", + Help: "Returns last SSL chain expiry in unixtime", + }) probeTLSVersion := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "probe_tls_version_info", @@ -125,9 +129,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry } if module.TCP.TLS { state := conn.(*tls.Conn).ConnectionState() - registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion) + registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds) probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix())) probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1) + probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix())) } scanner := bufio.NewScanner(conn) for i, qr := range module.TCP.QueryResponse { @@ -194,9 +199,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry // Get certificate expiry. state := tlsConn.ConnectionState() - registry.MustRegister(probeSSLEarliestCertExpiry) + registry.MustRegister(probeSSLEarliestCertExpiry, probeSSLLastChainExpiryTimestampSeconds) probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix())) probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1) + probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix())) } } return true diff --git a/prober/tcp_test.go b/prober/tcp_test.go index 02401f6..d05f317 100644 --- a/prober/tcp_test.go +++ b/prober/tcp_test.go @@ -14,8 +14,13 @@ package prober import ( + "bytes" "context" + "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "encoding/pem" "fmt" "io/ioutil" "net" @@ -84,14 +89,16 @@ func TestTCPConnectionWithTLS(t *testing.T) { // Create test certificates valid for 1 day. certExpiry := time.Now().AddDate(0, 0, 1) - testcert_pem, testkey_pem := generateTestCertificate(certExpiry, false) + rootCertTmpl := generateCertificateTemplate(certExpiry, false) + rootCertTmpl.IsCA = true + _, rootCertPem, rootKey := generateSelfSignedCertificate(rootCertTmpl) // CAFile must be passed via filesystem, use a tempfile. tmpCaFile, err := ioutil.TempFile("", "cafile.pem") if err != nil { t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err)) } - if _, err := tmpCaFile.Write(testcert_pem); err != nil { + if _, err := tmpCaFile.Write(rootCertPem); err != nil { t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err)) } if err := tmpCaFile.Close(); err != nil { @@ -109,7 +116,8 @@ func TestTCPConnectionWithTLS(t *testing.T) { } defer conn.Close() - testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem) + rootKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey)}) + testcert, err := tls.X509KeyPair(rootCertPem, rootKeyPem) if err != nil { panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err)) } @@ -193,6 +201,127 @@ func TestTCPConnectionWithTLS(t *testing.T) { checkRegistryResults(expectedResults, mfs, t) } +func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) { + if os.Getenv("TRAVIS") == "true" { + t.Skip("skipping; travisci is failing on ipv6 dns requests") + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Error listening on socket: %s", err) + } + defer ln.Close() + _, listenPort, _ := net.SplitHostPort(ln.Addr().String()) + + testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // From here prepare two certificate chains where one is expired + + rootPrivatekey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(fmt.Sprintf("Error creating rsa key: %s", err)) + } + + rootCertExpiry := time.Now().AddDate(0, 0, 2) + rootCertTmpl := generateCertificateTemplate(rootCertExpiry, false) + rootCertTmpl.IsCA = true + _, rootCertPem := generateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivatekey) + + oldRootCertExpiry := time.Now().AddDate(0, 0, -1) + expiredRootCertTmpl := generateCertificateTemplate(oldRootCertExpiry, false) + expiredRootCertTmpl.IsCA = true + expiredRootCert, expiredRootCertPem := generateSelfSignedCertificateWithPrivateKey(expiredRootCertTmpl, rootPrivatekey) + + serverCertExpiry := time.Now().AddDate(0, 0, 1) + serverCertTmpl := generateCertificateTemplate(serverCertExpiry, false) + _, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, expiredRootCert, rootPrivatekey) + + // CAFile must be passed via filesystem, use a tempfile. + tmpCaFile, err := ioutil.TempFile("", "cafile.pem") + if err != nil { + t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err)) + } + if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, expiredRootCertPem}, []byte("\n"))); err != nil { + t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err)) + } + if err := tmpCaFile.Close(); err != nil { + t.Fatalf(fmt.Sprintf("Error closing CA tempfile: %s", err)) + } + defer os.Remove(tmpCaFile.Name()) + + ch := make(chan (struct{})) + logger := log.NewNopLogger() + // Handle server side of this test. + serverFunc := func() { + conn, err := ln.Accept() + if err != nil { + panic(fmt.Sprintf("Error accepting on socket: %s", err)) + } + defer conn.Close() + + serverKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}) + + keypair, err := tls.X509KeyPair(serverCertPem, serverKeyPem) + if err != nil { + panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err)) + } + + // Immediately upgrade to TLS. + tlsConfig := &tls.Config{ + ServerName: "localhost", + Certificates: []tls.Certificate{keypair}, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS12, + } + tlsConn := tls.Server(conn, tlsConfig) + defer tlsConn.Close() + if err := tlsConn.Handshake(); err != nil { + level.Error(logger).Log("msg", "Error TLS Handshake (server) failed", "err", err) + } else { + // Send some bytes before terminating the connection. + fmt.Fprintf(tlsConn, "Hello World!\n") + } + ch <- struct{}{} + } + + // Expect name-verified TLS connection. + module := config.Module{ + TCP: config.TCPProbe{ + IPProtocol: "ip4", + IPProtocolFallback: true, + TLS: true, + TLSConfig: pconfig.TLSConfig{ + CAFile: tmpCaFile.Name(), + InsecureSkipVerify: false, + }, + }, + } + + registry := prometheus.NewRegistry() + go serverFunc() + // Test name-verification with name from target. + target := net.JoinHostPort("localhost", listenPort) + if !ProbeTCP(testCTX, target, module, registry, log.NewNopLogger()) { + t.Fatalf("TCP module failed, expected success.") + } + <-ch + + // Check the resulting metrics. + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + + // Check values + expectedResults := map[string]float64{ + "probe_ssl_earliest_cert_expiry": float64(serverCertExpiry.Unix()), + "probe_ssl_last_chain_expiry_timestamp_seconds": float64(serverCertExpiry.Unix()), + "probe_tls_version_info": 1, + } + checkRegistryResults(expectedResults, mfs, t) +} + func TestTCPConnectionQueryResponseStartTLS(t *testing.T) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -205,14 +334,16 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) { // Create test certificates valid for 1 day. certExpiry := time.Now().AddDate(0, 0, 1) - testcert_pem, testkey_pem := generateTestCertificate(certExpiry, true) + testCertTmpl := generateCertificateTemplate(certExpiry, true) + testCertTmpl.IsCA = true + _, testCertPem, testKey := generateSelfSignedCertificate(testCertTmpl) // CAFile must be passed via filesystem, use a tempfile. tmpCaFile, err := ioutil.TempFile("", "cafile.pem") if err != nil { t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err)) } - if _, err := tmpCaFile.Write(testcert_pem); err != nil { + if _, err := tmpCaFile.Write(testCertPem); err != nil { t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err)) } if err := tmpCaFile.Close(); err != nil { @@ -263,7 +394,8 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) { } fmt.Fprintf(conn, "220 2.0.0 Ready to start TLS\n") - testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem) + testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)}) + testcert, err := tls.X509KeyPair(testCertPem, testKeyPem) if err != nil { panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err)) } diff --git a/prober/tls.go b/prober/tls.go index 758cbf3..a4ef0f9 100644 --- a/prober/tls.go +++ b/prober/tls.go @@ -28,6 +28,23 @@ func getEarliestCertExpiry(state *tls.ConnectionState) time.Time { return earliest } +func getLastChainExpiry(state *tls.ConnectionState) time.Time { + lastChainExpiry := time.Time{} + for _, chain := range state.VerifiedChains { + earliestCertExpiry := time.Time{} + for _, cert := range chain { + if (earliestCertExpiry.IsZero() || cert.NotAfter.Before(earliestCertExpiry)) && !cert.NotAfter.IsZero() { + earliestCertExpiry = cert.NotAfter + } + } + if lastChainExpiry.IsZero() || lastChainExpiry.After(earliestCertExpiry) { + lastChainExpiry = earliestCertExpiry + } + + } + return lastChainExpiry +} + func getTLSVersion(state *tls.ConnectionState) string { switch state.Version { case tls.VersionTLS10: diff --git a/prober/utils_test.go b/prober/utils_test.go index 42dfd9e..2ac38ee 100644 --- a/prober/utils_test.go +++ b/prober/utils_test.go @@ -79,18 +79,8 @@ func checkRegistryLabels(expRes map[string]map[string]string, mfs []*dto.MetricF } } -// Create test certificate with specified expiry date -// Certificate will be self-signed and use localhost/127.0.0.1 -// Generated certificate and key are returned in PEM encoding -func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byte) { - privatekey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - panic(fmt.Sprintf("Error creating rsa key: %s", err)) - } - publickey := &privatekey.PublicKey - - cert := x509.Certificate{ - IsCA: true, +func generateCertificateTemplate(expiry time.Time, IPAddressSAN bool) *x509.Certificate { + template := &x509.Certificate{ BasicConstraintsValid: true, SubjectKeyId: []byte{1}, SerialNumber: big.NewInt(1), @@ -102,18 +92,54 @@ func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byt ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, } - cert.DNSNames = append(cert.DNSNames, "localhost") + + template.DNSNames = append(template.DNSNames, "localhost") if IPAddressSAN { - cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("127.0.0.1")) - cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("::1")) + template.IPAddresses = append(template.IPAddresses, net.ParseIP("127.0.0.1")) + template.IPAddresses = append(template.IPAddresses, net.ParseIP("::1")) } - derCert, err := x509.CreateCertificate(rand.Reader, &cert, &cert, publickey, privatekey) + + return template +} + +func generateCertificate(template, parent *x509.Certificate, publickey *rsa.PublicKey, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) { + derCert, err := x509.CreateCertificate(rand.Reader, template, template, publickey, privatekey) if err != nil { panic(fmt.Sprintf("Error signing test-certificate: %s", err)) } + cert, err := x509.ParseCertificate(derCert) + if err != nil { + panic(fmt.Sprintf("Error parsing test-certificate: %s", err)) + } pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert}) - pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privatekey)}) - return pemCert, pemKey + return cert, pemCert + +} + +func generateSignedCertificate(template, parentCert *x509.Certificate, parentKey *rsa.PrivateKey) (*x509.Certificate, []byte, *rsa.PrivateKey) { + privatekey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(fmt.Sprintf("Error creating rsa key: %s", err)) + } + cert, pemCert := generateCertificate(template, parentCert, &privatekey.PublicKey, parentKey) + return cert, pemCert, privatekey +} + +func generateSelfSignedCertificate(template *x509.Certificate) (*x509.Certificate, []byte, *rsa.PrivateKey) { + privatekey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(fmt.Sprintf("Error creating rsa key: %s", err)) + } + publickey := &privatekey.PublicKey + + cert, pemCert := generateCertificate(template, template, publickey, privatekey) + return cert, pemCert, privatekey +} + +func generateSelfSignedCertificateWithPrivateKey(template *x509.Certificate, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) { + publickey := &privatekey.PublicKey + cert, pemCert := generateCertificate(template, template, publickey, privatekey) + return cert, pemCert } func TestChooseProtocol(t *testing.T) { -- 2.25.1