From: conorbroderick Date: Mon, 15 May 2017 13:10:46 +0000 (+0100) Subject: Switch from handling output by hand to using a custom registry X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=7abdf0570d44cc8dbd258987e9d3353af1d6126e;p=blackbox_exporter.git Switch from handling output by hand to using a custom registry --- diff --git a/dns.go b/dns.go index 90c8982..6c8f56f 100644 --- a/dns.go +++ b/dns.go @@ -14,12 +14,12 @@ package main import ( - "fmt" "net" "net/http" "regexp" "github.com/miekg/dns" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" ) @@ -81,15 +81,36 @@ func validRcode(rcode int, valid []string) bool { return false } -func probeDNS(target string, w http.ResponseWriter, module Module) bool { +func probeDNS(target string, w http.ResponseWriter, module Module, registry *prometheus.Registry) bool { var numAnswer, numAuthority, numAdditional int var dialProtocol, fallbackProtocol string + probeIPProtocolGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ip_protocol", + Help: "Specifies whether probe ip protocl is IP4 or IP6", + }) + probeDNSAnswerRRSGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_dns_answer_rrs", + Help: "Returns number of entries in the answer resource record list", + }) + probeDNSAuthorityRRSGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_dns_authority_rrs", + Help: "Returns number of entries in the authority resource record list", + }) + probeDNSAdditionalRRSGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_dns_additional_rrs", + Help: "Returns number of entries in the additional resource record list", + }) + registry.MustRegister(probeIPProtocolGauge) + registry.MustRegister(probeDNSAnswerRRSGauge) + registry.MustRegister(probeDNSAuthorityRRSGauge) + registry.MustRegister(probeDNSAdditionalRRSGauge) + defer func() { // These metrics can be used to build additional alerting based on the number of replies. // They should be returned even in case of errors. - fmt.Fprintf(w, "probe_dns_answer_rrs %d\n", numAnswer) - fmt.Fprintf(w, "probe_dns_authority_rrs %d\n", numAuthority) - fmt.Fprintf(w, "probe_dns_additional_rrs %d\n", numAdditional) + probeDNSAnswerRRSGauge.Set(float64(numAnswer)) + probeDNSAuthorityRRSGauge.Set(float64(numAuthority)) + probeDNSAdditionalRRSGauge.Set(float64(numAdditional)) }() if module.DNS.Protocol == "" { @@ -124,9 +145,9 @@ func probeDNS(target string, w http.ResponseWriter, module Module) bool { } if dialProtocol[len(dialProtocol)-1] == '6' { - fmt.Fprintln(w, "probe_ip_protocol 6") + probeIPProtocolGauge.Set(6) } else { - fmt.Fprintln(w, "probe_ip_protocol 4") + probeIPProtocolGauge.Set(4) } client := new(dns.Client) diff --git a/dns_test.go b/dns_test.go index 5468f24..9736434 100644 --- a/dns_test.go +++ b/dns_test.go @@ -14,14 +14,17 @@ package main import ( + "bytes" "net" "net/http/httptest" + "regexp" "runtime" - "strings" "testing" "time" "github.com/miekg/dns" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" ) var PROTOCOLS = [...]string{"udp", "tcp"} @@ -110,11 +113,6 @@ func TestRecursiveDNSResponse(t *testing.T) { }, false, }, } - expectedOutput := []string{ - "probe_dns_answer_rrs 2\n", - "probe_dns_authority_rrs 0\n", - "probe_dns_additional_rrs 0\n", - } for _, protocol := range PROTOCOLS { server, addr := startDNSServer(protocol, recursiveDNSHandler) @@ -123,14 +121,31 @@ func TestRecursiveDNSResponse(t *testing.T) { for i, test := range tests { test.Probe.Protocol = protocol recorder := httptest.NewRecorder() - result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}) + registry := prometheus.NewPedanticRegistry() + registry.Gather() + result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}, registry) if result != test.ShouldSucceed { t.Fatalf("Test %d had unexpected result: %v", i, result) } - body := recorder.Body.String() - for _, line := range expectedOutput { - if !strings.Contains(body, line) { - t.Fatalf("Did not find expected output in test %d: %q", i, line) + + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + + for _, re := range []*regexp.Regexp{ + regexp.MustCompile("probe_dns_answer_rrs 2"), + regexp.MustCompile("probe_dns_authority_rrs 0"), + regexp.MustCompile("probe_dns_additional_rrs 0"), + } { + if !re.Match(buf.Bytes()) { + t.Errorf("Did not find expected output in test %d: %q", i, re) } } } @@ -236,11 +251,6 @@ func TestAuthoritativeDNSResponse(t *testing.T) { }, false, }, } - expectedOutput := []string{ - "probe_dns_answer_rrs 1\n", - "probe_dns_authority_rrs 2\n", - "probe_dns_additional_rrs 3\n", - } for _, protocol := range PROTOCOLS { server, addr := startDNSServer(protocol, authoritativeDNSHandler) @@ -249,14 +259,30 @@ func TestAuthoritativeDNSResponse(t *testing.T) { for i, test := range tests { test.Probe.Protocol = protocol recorder := httptest.NewRecorder() - result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}) + registry := prometheus.NewRegistry() + result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}, registry) if result != test.ShouldSucceed { t.Fatalf("Test %d had unexpected result: %v", i, result) } - body := recorder.Body.String() - for _, line := range expectedOutput { - if !strings.Contains(body, line) { - t.Fatalf("Did not find expected output in test %d: %q", i, line) + + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + + for _, re := range []*regexp.Regexp{ + regexp.MustCompile("probe_dns_answer_rrs 1"), + regexp.MustCompile("probe_dns_authority_rrs 2"), + regexp.MustCompile("probe_dns_additional_rrs 3"), + } { + if !re.Match(buf.Bytes()) { + t.Errorf("Did not find expected output in test %d: %q", i, re) } } } @@ -292,11 +318,6 @@ func TestServfailDNSResponse(t *testing.T) { }, false, }, } - expectedOutput := []string{ - "probe_dns_answer_rrs 0\n", - "probe_dns_authority_rrs 0\n", - "probe_dns_additional_rrs 0\n", - } for _, protocol := range PROTOCOLS { // dns.HandleFailed returns SERVFAIL on everything @@ -306,14 +327,29 @@ func TestServfailDNSResponse(t *testing.T) { for i, test := range tests { test.Probe.Protocol = protocol recorder := httptest.NewRecorder() - result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}) + registry := prometheus.NewRegistry() + result := probeDNS(addr.String(), recorder, Module{Timeout: time.Second, DNS: test.Probe}, registry) if result != test.ShouldSucceed { t.Fatalf("Test %d had unexpected result: %v", i, result) } - body := recorder.Body.String() - for _, line := range expectedOutput { - if !strings.Contains(body, line) { - t.Fatalf("Did not find expected output in test %d: %q", i, line) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + + for _, re := range []*regexp.Regexp{ + regexp.MustCompile("probe_dns_answer_rrs 0"), + regexp.MustCompile("probe_dns_authority_rrs 0"), + regexp.MustCompile("probe_dns_additional_rrs 0"), + } { + if !re.Match(buf.Bytes()) { + t.Errorf("Did not find expected output in test %d: %q", i, re) } } } @@ -321,7 +357,7 @@ func TestServfailDNSResponse(t *testing.T) { } func TestDNSProtocol(t *testing.T) { - // This test assumes that listening "tcp" listens both IPv6 and IPv4 traffic and + // This test assumes that listening TCP listens both IPv6 and IPv4 traffic and // localhost resolves to both 127.0.0.1 and ::1. we must skip the test if either // of these isn't true. This should be true for modern Linux systems. if runtime.GOOS == "dragonfly" || runtime.GOOS == "openbsd" { @@ -347,13 +383,24 @@ func TestDNSProtocol(t *testing.T) { }, } recorder := httptest.NewRecorder() - result := probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body := recorder.Body.String() + registry := prometheus.NewRegistry() + result := probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("DNS protocol: \"%v4\" connection test failed, expected success.", protocol) } - if !strings.Contains(body, "probe_ip_protocol 4\n") { - t.Fatalf("Expected IPv4, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re := regexp.MustCompile("probe_ip_protocol 4") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv4, got %s", buf.String()) } // Force IPv6 @@ -365,13 +412,23 @@ func TestDNSProtocol(t *testing.T) { }, } recorder = httptest.NewRecorder() - result = probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("DNS protocol: \"%v6\" connection test failed, expected success.", protocol) } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // Prefer IPv6 @@ -384,13 +441,23 @@ func TestDNSProtocol(t *testing.T) { }, } recorder = httptest.NewRecorder() - result = probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("DNS protocol: \"%v\", preferred \"ip6\" connection test failed, expected success.", protocol) } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // Prefer IPv4 @@ -403,13 +470,23 @@ func TestDNSProtocol(t *testing.T) { }, } recorder = httptest.NewRecorder() - result = probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("DNS protocol: \"%v\", preferred \"ip4\" connection test failed, expected success.", protocol) } - if !strings.Contains(body, "probe_ip_protocol 4\n") { - t.Fatalf("Expected IPv4, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 4") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv4, got %s", buf.String()) } // Prefer none @@ -421,13 +498,23 @@ func TestDNSProtocol(t *testing.T) { }, } recorder = httptest.NewRecorder() - result = probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("DNS protocol: \"%v\" connection test failed, expected success.", protocol) } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // No protocol @@ -438,8 +525,8 @@ func TestDNSProtocol(t *testing.T) { }, } recorder = httptest.NewRecorder() - result = probeDNS(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeDNS(net.JoinHostPort("localhost", port), recorder, module, registry) if protocol == "udp" { if !result { t.Fatalf("DNS test connection with protocol %s failed, expected success.", protocol) @@ -449,8 +536,19 @@ func TestDNSProtocol(t *testing.T) { t.Fatalf("DNS test connection with protocol %s succeeded, expected failure.", protocol) } } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) + } + } } diff --git a/http.go b/http.go index d57b455..d3fa554 100644 --- a/http.go +++ b/http.go @@ -15,7 +15,6 @@ package main import ( "errors" - "fmt" "io" "io/ioutil" "net" @@ -24,6 +23,7 @@ import ( "regexp" "strings" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" ) @@ -56,10 +56,48 @@ func matchRegularExpressions(reader io.Reader, config HTTPProbe) bool { return true } -func probeHTTP(target string, w http.ResponseWriter, module Module) (success bool) { - var isSSL, redirects int +func probeHTTP(target string, w http.ResponseWriter, module Module, registry *prometheus.Registry) (success bool) { + var redirects int var dialProtocol, fallbackProtocol string + var ( + contentLengthGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "content_length", + Help: "Length of http content response", + }) + + redirectsGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_http_redirects", + Help: "The number of redirects", + }) + + isSSLGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_http_ssl", + Help: "Indicates if SSL was used for the final redirect", + }) + + statusCodeGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_http_status_code", + Help: "Response HTTP status code", + }) + + probeIPProtocolGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ip_protocol", + Help: "Specifies whether probe ip protocl is IP4 or IP6", + }) + + probeSSLEarliestCertExpiryGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ssl_earliest_cert_expiry", + Help: "Returns earliest SSL cert expiry in unixtime", + }) + ) + + registry.MustRegister(contentLengthGauge) + registry.MustRegister(redirectsGauge) + registry.MustRegister(isSSLGauge) + registry.MustRegister(statusCodeGauge) + registry.MustRegister(probeIPProtocolGauge) + config := module.HTTP if module.HTTP.Protocol == "" { @@ -105,9 +143,9 @@ func probeHTTP(target string, w http.ResponseWriter, module Module) (success boo } if dialProtocol == "tcp6" { - fmt.Fprintln(w, "probe_ip_protocol 6") + probeIPProtocolGauge.Set(6) } else { - fmt.Fprintln(w, "probe_ip_protocol 4") + probeIPProtocolGauge.Set(4) } client := &http.Client{ @@ -161,6 +199,7 @@ func probeHTTP(target string, w http.ResponseWriter, module Module) (success boo } resp, err := client.Do(request) + // Err won't be nil if redirects were turned off. See https://github.com/golang/go/issues/3795 if err != nil && resp == nil { log.Warnf("Error for HTTP request to %s: %s", target, err) @@ -187,18 +226,18 @@ func probeHTTP(target string, w http.ResponseWriter, module Module) (success boo } if resp.TLS != nil { - isSSL = 1 - fmt.Fprintf(w, "probe_ssl_earliest_cert_expiry %f\n", - float64(getEarliestCertExpiry(resp.TLS).UnixNano())/1e9) + isSSLGauge.Set(float64(1)) + probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).UnixNano() / 1e9)) if config.FailIfSSL { success = false } } else if config.FailIfNotSSL { success = false } - fmt.Fprintf(w, "probe_http_status_code %d\n", resp.StatusCode) - fmt.Fprintf(w, "probe_http_content_length %d\n", resp.ContentLength) - fmt.Fprintf(w, "probe_http_redirects %d\n", redirects) - fmt.Fprintf(w, "probe_http_ssl %d\n", isSSL) + + statusCodeGauge.Set(float64(resp.StatusCode)) + contentLengthGauge.Set(float64(resp.ContentLength)) + redirectsGauge.Set(float64(redirects)) + return } diff --git a/http_test.go b/http_test.go index 85877b5..57f4df3 100644 --- a/http_test.go +++ b/http_test.go @@ -14,13 +14,18 @@ package main import ( + "bytes" "fmt" - "github.com/prometheus/common/config" "net/http" "net/http/httptest" + "regexp" "strings" "testing" "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/expfmt" ) func TestHTTPStatusCodes(t *testing.T) { @@ -40,15 +45,15 @@ func TestHTTPStatusCodes(t *testing.T) { {404, []int{404}, true}, {200, []int{404}, false}, } - for i, test := range tests { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(test.StatusCode) })) defer ts.Close() + registry := prometheus.NewRegistry() recorder := httptest.NewRecorder() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{ValidStatusCodes: test.ValidStatusCodes}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{ValidStatusCodes: test.ValidStatusCodes}}, registry) body := recorder.Body.String() if result != test.ShouldSucceed { t.Fatalf("Test %d had unexpected result: %s", i, body) @@ -66,13 +71,25 @@ func TestRedirectFollowed(t *testing.T) { // Follow redirect, should succeed with 200. recorder := httptest.NewRecorder() - result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{}}) + registry := prometheus.NewRegistry() + result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{}}, registry) body := recorder.Body.String() if !result { t.Fatalf("Redirect test failed unexpectedly, got %s", body) } - if !strings.Contains(body, "probe_http_redirects 1\n") { - t.Fatalf("Expected one redirect, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re := regexp.MustCompile("probe_http_redirects 1") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected one redirect, got %s", body) } } @@ -85,8 +102,9 @@ func TestRedirectNotFollowed(t *testing.T) { // Follow redirect, should succeed with 200. recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{NoFollowRedirects: true, ValidStatusCodes: []int{302}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{NoFollowRedirects: true, ValidStatusCodes: []int{302}}}, registry) body := recorder.Body.String() if !result { t.Fatalf("Redirect test failed unexpectedly, got %s", body) @@ -103,8 +121,9 @@ func TestPost(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{Method: "POST"}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{Method: "POST"}}, registry) body := recorder.Body.String() if !result { t.Fatalf("Post test failed unexpectedly, got %s", body) @@ -117,15 +136,28 @@ func TestFailIfNotSSL(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotSSL: true}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotSSL: true}}, registry) body := recorder.Body.String() if result { t.Fatalf("Fail if not SSL test suceeded unexpectedly, got %s", body) } - if !strings.Contains(body, "probe_http_ssl 0\n") { - t.Fatalf("Expected HTTP without SSL, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } } + re := regexp.MustCompile("probe_http_ssl 0") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected HTTP without SSL, got %s", body) + } + } func TestFailIfMatchesRegexp(t *testing.T) { @@ -135,8 +167,9 @@ func TestFailIfMatchesRegexp(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database"}}}, registry) body := recorder.Body.String() if result { t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) @@ -148,8 +181,9 @@ func TestFailIfMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database"}}}, registry) body = recorder.Body.String() if !result { t.Fatalf("Regexp test failed unexpectedly, got %s", body) @@ -163,8 +197,9 @@ func TestFailIfMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database", "internal error"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database", "internal error"}}}, registry) body = recorder.Body.String() if result { t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) @@ -176,8 +211,9 @@ func TestFailIfMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database", "internal error"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfMatchesRegexp: []string{"could not connect to database", "internal error"}}}, registry) body = recorder.Body.String() if !result { t.Fatalf("Regexp test failed unexpectedly, got %s", body) @@ -191,8 +227,9 @@ func TestFailIfNotMatchesRegexp(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here"}}}, registry) body := recorder.Body.String() if result { t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) @@ -204,8 +241,9 @@ func TestFailIfNotMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here"}}}, registry) body = recorder.Body.String() if !result { t.Fatalf("Regexp test failed unexpectedly, got %s", body) @@ -219,8 +257,9 @@ func TestFailIfNotMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here", "Copyright 2015"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here", "Copyright 2015"}}}, registry) body = recorder.Body.String() if result { t.Fatalf("Regexp test succeeded unexpectedly, got %s", body) @@ -232,8 +271,9 @@ func TestFailIfNotMatchesRegexp(t *testing.T) { defer ts.Close() recorder = httptest.NewRecorder() + registry = prometheus.NewRegistry() result = probeHTTP(ts.URL, recorder, - Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here", "Copyright 2015"}}}) + Module{Timeout: time.Second, HTTP: HTTPProbe{FailIfNotMatchesRegexp: []string{"Download the latest version here", "Copyright 2015"}}}, registry) body = recorder.Body.String() if !result { t.Fatalf("Regexp test failed unexpectedly, got %s", body) @@ -262,9 +302,10 @@ func TestHTTPHeaders(t *testing.T) { })) defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{ Headers: headers, - }}) + }}, registry) if !result { t.Fatalf("Probe failed unexpectedly.") } @@ -276,16 +317,28 @@ func TestFailIfSelfSignedCA(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{ TLSConfig: config.TLSConfig{InsecureSkipVerify: false}, - }}) + }}, registry) body := recorder.Body.String() if result { t.Fatalf("Fail if selfsigned CA test suceeded unexpectedly, got %s", body) } - if !strings.Contains(body, "probe_http_ssl 0\n") { - t.Fatalf("Expected HTTP without SSL because of CA failure, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re := regexp.MustCompile("probe_http_ssl 0") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected HTTP without SSL because of CA failure, got %s", body) } } @@ -295,16 +348,28 @@ func TestSucceedIfSelfSignedCA(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{ TLSConfig: config.TLSConfig{InsecureSkipVerify: true}, - }}) + }}, registry) body := recorder.Body.String() if !result { t.Fatalf("Fail if (not strict) selfsigned CA test fails unexpectedly, got %s", body) } - if !strings.Contains(body, "probe_http_ssl 1\n") { - t.Fatalf("Expected HTTP with SSL, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re := regexp.MustCompile("probe_http_ssl 1") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected HTTP with SSL, got %s", body) } } @@ -314,15 +379,28 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) { defer ts.Close() recorder := httptest.NewRecorder() + registry := prometheus.NewRegistry() result := probeHTTP(ts.URL, recorder, Module{Timeout: time.Second, HTTP: HTTPProbe{ TLSConfig: config.TLSConfig{InsecureSkipVerify: false}, - }}) + }}, registry) body := recorder.Body.String() if !result { t.Fatalf("Fail if InsecureSkipVerify affects simple http fails unexpectedly, got %s", body) } - if !strings.Contains(body, "probe_http_ssl 0\n") { - t.Fatalf("Expected HTTP without SSL, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } } + re := regexp.MustCompile("probe_http_ssl 0") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected HTTP without SSL, got %s", body) + } + } diff --git a/icmp.go b/icmp.go index 60ae926..b8f501d 100644 --- a/icmp.go +++ b/icmp.go @@ -15,7 +15,6 @@ package main import ( "bytes" - "fmt" "net" "net/http" "os" @@ -26,6 +25,7 @@ import ( "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" ) @@ -41,14 +41,25 @@ func getICMPSequence() uint16 { return icmpSequence } -func probeICMP(target string, w http.ResponseWriter, module Module) (success bool) { +func probeICMP(target string, w http.ResponseWriter, module Module, registry *prometheus.Registry) (success bool) { var ( - socket *icmp.PacketConn - requestType icmp.Type - replyType icmp.Type - fallbackProtocol string + socket *icmp.PacketConn + requestType icmp.Type + replyType icmp.Type + fallbackProtocol string + probeIPProtocolGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ip_protocol", + Help: "Specifies whether probe ip protocl is IP4 or IP6", + }) + probeDNSLookupTimeSeconds = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_dns_lookup_time_seconds", + Help: "Returns the time taken for probe dns lookup in seconds", + }) ) + registry.MustRegister(probeIPProtocolGauge) + registry.MustRegister(probeDNSLookupTimeSeconds) + deadline := time.Now().Add(module.Timeout) // Defaults to IPv4 to be compatible with older versions @@ -78,7 +89,7 @@ func probeICMP(target string, w http.ResponseWriter, module Module) (success boo if err != nil && fallbackProtocol != "" { ip, err = net.ResolveIPAddr(fallbackProtocol, target) } - fmt.Fprintf(w, "probe_dns_lookup_time_seconds %f\n", time.Since(resolveStart).Seconds()) + probeDNSLookupTimeSeconds.Add(time.Since(resolveStart).Seconds()) if err != nil { log.Warnf("Error resolving address %s: %s", target, err) @@ -89,12 +100,12 @@ func probeICMP(target string, w http.ResponseWriter, module Module) (success boo requestType = ipv6.ICMPTypeEchoRequest replyType = ipv6.ICMPTypeEchoReply socket, err = icmp.ListenPacket("ip6:ipv6-icmp", "::") - fmt.Fprintln(w, "probe_ip_protocol 6") + probeIPProtocolGauge.Set(6) } else { requestType = ipv4.ICMPTypeEcho replyType = ipv4.ICMPTypeEchoReply socket, err = icmp.ListenPacket("ip4:icmp", "0.0.0.0") - fmt.Fprintln(w, "probe_ip_protocol 4") + probeIPProtocolGauge.Set(4) } if err != nil { diff --git a/main.go b/main.go index 51bf2a3..c67e280 100644 --- a/main.go +++ b/main.go @@ -23,10 +23,12 @@ import ( "syscall" "time" - "gopkg.in/yaml.v2" "sync" + "gopkg.in/yaml.v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/config" "github.com/prometheus/common/log" "github.com/prometheus/common/version" @@ -100,7 +102,7 @@ type DNSRRValidator struct { FailIfNotMatchesRegexp []string `yaml:"fail_if_not_matches_regexp"` } -var Probers = map[string]func(string, http.ResponseWriter, Module) bool{ +var Probers = map[string]func(string, http.ResponseWriter, Module, *prometheus.Registry) bool{ "http": probeHTTP, "tcp": probeTCP, "icmp": probeICMP, @@ -130,6 +132,14 @@ func (sc *SafeConfig) reloadConfig(confFile string) (err error) { } func probeHandler(w http.ResponseWriter, r *http.Request, conf *Config) { + probeSuccessGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_success", + Help: "Displays whether or not the probe was a success", + }) + probeDurationGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_duration_seconds", + Help: "Returns how long the probe took to complete in seconds", + }) params := r.URL.Query() target := params.Get("target") if target == "" { @@ -153,13 +163,16 @@ func probeHandler(w http.ResponseWriter, r *http.Request, conf *Config) { } start := time.Now() - success := prober(target, w, module) - fmt.Fprintf(w, "probe_duration_seconds %f\n", time.Since(start).Seconds()) + registry := prometheus.NewRegistry() + registry.MustRegister(probeSuccessGauge) + registry.MustRegister(probeDurationGauge) + success := prober(target, w, module, registry) + probeDurationGauge.Set(time.Since(start).Seconds()) if success { - fmt.Fprintln(w, "probe_success 1") - } else { - fmt.Fprintln(w, "probe_success 0") + probeSuccessGauge.Set(1) } + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) } func init() { @@ -167,6 +180,7 @@ func init() { } func main() { + var ( configFile = flag.String("config.file", "blackbox.yml", "Blackbox exporter configuration file.") listenAddress = flag.String("web.listen-address", ":9115", "The address to listen on for HTTP requests.") diff --git a/tcp.go b/tcp.go index ae27254..135e8eb 100644 --- a/tcp.go +++ b/tcp.go @@ -22,10 +22,11 @@ import ( "regexp" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" ) -func dialTCP(target string, w http.ResponseWriter, module Module) (net.Conn, error) { +func dialTCP(target string, w http.ResponseWriter, module Module, protocolProbeGauge prometheus.Gauge) (net.Conn, error) { var dialProtocol, fallbackProtocol string dialer := &net.Dialer{Timeout: module.Timeout} @@ -60,9 +61,9 @@ func dialTCP(target string, w http.ResponseWriter, module Module) (net.Conn, err } if dialProtocol == "tcp6" { - fmt.Fprintln(w, "probe_ip_protocol 6") + protocolProbeGauge.Set(6) } else { - fmt.Fprintln(w, "probe_ip_protocol 4") + protocolProbeGauge.Set(4) } if !module.TCP.TLS { @@ -75,9 +76,19 @@ func dialTCP(target string, w http.ResponseWriter, module Module) (net.Conn, err return tls.DialWithDialer(dialer, dialProtocol, target, config) } -func probeTCP(target string, w http.ResponseWriter, module Module) bool { +func probeTCP(target string, w http.ResponseWriter, module Module, registry *prometheus.Registry) bool { + probeIPProtocolGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ip_protocol", + Help: "Specifies whether probe ip protocol is IP4 or IP6", + }) + probeSSLEarliestCertExpiry := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ssl_earliest_cert_expiry", + Help: "Returns earliest SSL cert expiry date", + }) + registry.MustRegister(probeIPProtocolGauge) + registry.MustRegister(probeSSLEarliestCertExpiry) deadline := time.Now().Add(module.Timeout) - conn, err := dialTCP(target, w, module) + conn, err := dialTCP(target, w, module, probeIPProtocolGauge) if err != nil { return false } @@ -91,8 +102,7 @@ func probeTCP(target string, w http.ResponseWriter, module Module) bool { } if module.TCP.TLS { state := conn.(*tls.Conn).ConnectionState() - fmt.Fprintf(w, "probe_ssl_earliest_cert_expiry %f\n", - float64(getEarliestCertExpiry(&state).UnixNano())/1e9) + probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).UnixNano()) / 1e9) } scanner := bufio.NewScanner(conn) for _, qr := range module.TCP.QueryResponse { diff --git a/tcp_test.go b/tcp_test.go index 7307393..d697d44 100644 --- a/tcp_test.go +++ b/tcp_test.go @@ -14,13 +14,17 @@ package main import ( + "bytes" "fmt" "net" "net/http/httptest" + "regexp" "runtime" - "strings" "testing" "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" ) func TestTCPConnection(t *testing.T) { @@ -40,7 +44,8 @@ func TestTCPConnection(t *testing.T) { ch <- struct{}{} }() recorder := httptest.NewRecorder() - if !probeTCP(ln.Addr().String(), recorder, Module{Timeout: time.Second}) { + registry := prometheus.NewRegistry() + if !probeTCP(ln.Addr().String(), recorder, Module{Timeout: time.Second}, registry) { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -49,7 +54,8 @@ func TestTCPConnection(t *testing.T) { func TestTCPConnectionFails(t *testing.T) { // Invalid port number. recorder := httptest.NewRecorder() - if probeTCP(":0", recorder, Module{Timeout: time.Second}) { + registry := prometheus.NewRegistry() + if probeTCP(":0", recorder, Module{Timeout: time.Second}, registry) { t.Fatalf("TCP module suceeded, expected failure.") } } @@ -87,7 +93,8 @@ func TestTCPConnectionQueryResponseIRC(t *testing.T) { ch <- struct{}{} }() recorder := httptest.NewRecorder() - if !probeTCP(ln.Addr().String(), recorder, module) { + registry := prometheus.NewRegistry() + if !probeTCP(ln.Addr().String(), recorder, module, registry) { t.Fatalf("TCP module failed, expected success.") } <-ch @@ -105,7 +112,8 @@ func TestTCPConnectionQueryResponseIRC(t *testing.T) { conn.Close() ch <- struct{}{} }() - if probeTCP(ln.Addr().String(), recorder, module) { + registry = prometheus.NewRegistry() + if probeTCP(ln.Addr().String(), recorder, module, registry) { t.Fatalf("TCP module succeeded, expected failure.") } <-ch @@ -144,7 +152,8 @@ func TestTCPConnectionQueryResponseMatching(t *testing.T) { ch <- version }() recorder := httptest.NewRecorder() - if !probeTCP(ln.Addr().String(), recorder, module) { + registry := prometheus.NewRegistry() + if !probeTCP(ln.Addr().String(), recorder, module, registry) { t.Fatalf("TCP module failed, expected success.") } if got, want := <-ch, "OpenSSH_6.9p1"; got != want { @@ -153,7 +162,7 @@ func TestTCPConnectionQueryResponseMatching(t *testing.T) { } func TestTCPConnectionProtocol(t *testing.T) { - // This test assumes that listening "tcp" listens both IPv6 and IPv4 traffic and + // This test assumes that listening TCP listens both IPv6 and IPv4 traffic and // localhost resolves to both 127.0.0.1 and ::1. we must skip the test if either // of these isn't true. This should be true for modern Linux systems. if runtime.GOOS == "dragonfly" || runtime.GOOS == "openbsd" { @@ -181,13 +190,24 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder := httptest.NewRecorder() - result := probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body := recorder.Body.String() + registry := prometheus.NewRegistry() + result := probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP protocol: \"tcp4\" connection test failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 4\n") { - t.Fatalf("Expected IPv4, got %s", body) + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re := regexp.MustCompile("probe_ip_protocol 4") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv4, got %s", buf.String()) } // Force IPv6 @@ -199,13 +219,23 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder = httptest.NewRecorder() - result = probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP protocol: \"tcp6\" connection test failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // Prefer IPv4 @@ -218,13 +248,23 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder = httptest.NewRecorder() - result = probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP protocol: \"tcp\", prefer: \"ip4\" connection test failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 4\n") { - t.Fatalf("Expected IPv4, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 4") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv4, got %s", buf.String()) } // Prefer IPv6 @@ -237,13 +277,23 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder = httptest.NewRecorder() - result = probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP protocol: \"tcp\", prefer: \"ip6\" connection test failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // Prefer nothing @@ -255,13 +305,23 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder = httptest.NewRecorder() - result = probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP protocol: \"tcp\" connection test failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } // No protocol @@ -271,12 +331,22 @@ func TestTCPConnectionProtocol(t *testing.T) { } recorder = httptest.NewRecorder() - result = probeTCP(net.JoinHostPort("localhost", port), recorder, module) - body = recorder.Body.String() + registry = prometheus.NewRegistry() + result = probeTCP(net.JoinHostPort("localhost", port), recorder, module, registry) if !result { t.Fatalf("TCP connection test with protocol unspecified failed, expected success.") } - if !strings.Contains(body, "probe_ip_protocol 6\n") { - t.Fatalf("Expected IPv6, got %s", body) + mfs, err = registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, mf := range mfs { + if _, err = expfmt.MetricFamilyToText(&buf, mf); err != nil { + t.Fatal(err) + } + } + re = regexp.MustCompile("probe_ip_protocol 6") + if !re.Match(buf.Bytes()) { + t.Errorf("Expected IPv6, got %s", buf.String()) } }