From 729b90d3702d985c39cec9845b0503365c97fd34 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Thu, 14 Sep 2017 15:45:28 +0100 Subject: [PATCH] Add last 100 probes to the front page, with logs --- history.go | 75 ++++++++++++++++++++++++++++++++++++++++++ main.go | 93 +++++++++++++++++++++++++++++++++++++--------------- main_test.go | 19 +++++++++-- 3 files changed, 158 insertions(+), 29 deletions(-) create mode 100644 history.go diff --git a/history.go b/history.go new file mode 100644 index 0000000..5560690 --- /dev/null +++ b/history.go @@ -0,0 +1,75 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "sync" +) + +type result struct { + id int64 + moduleName string + target string + debugOutput *bytes.Buffer + success bool +} + +type resultHistory struct { + mu sync.Mutex + nextId int64 + results []*result +} + +// Add a result to the history. +func (rh *resultHistory) Add(moduleName, target string, debugOutput *bytes.Buffer, success bool) { + rh.mu.Lock() + defer rh.mu.Unlock() + + result := &result{ + id: rh.nextId, + moduleName: moduleName, + target: target, + debugOutput: debugOutput, + success: success, + } + rh.nextId++ + + rh.results = append(rh.results, result) + if len(rh.results) > 100 { + copy(rh.results, rh.results[1:]) + } +} + +// Return a list of all results. +func (rh *resultHistory) List() []*result { + rh.mu.Lock() + defer rh.mu.Unlock() + + return rh.results[:] +} + +// Return a given result. +func (rh *resultHistory) Get(id int64) *result { + rh.mu.Lock() + defer rh.mu.Unlock() + + for _, r := range rh.results { + if r.id == id { + return r + } + } + + return nil +} diff --git a/main.go b/main.go index 9adf6ac..f98c7c1 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "bytes" "context" "fmt" + "html" "net/http" "os" "os/signal" @@ -26,15 +27,14 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "gopkg.in/alecthomas/kingpin.v2" - "gopkg.in/yaml.v2" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/promlog" "github.com/prometheus/common/promlog/flag" "github.com/prometheus/common/version" + "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/yaml.v2" "github.com/prometheus/blackbox_exporter/config" "github.com/prometheus/blackbox_exporter/prober" @@ -57,7 +57,7 @@ var ( } ) -func probeHandler(w http.ResponseWriter, r *http.Request, c *config.Config, logger log.Logger) { +func probeHandler(w http.ResponseWriter, r *http.Request, c *config.Config, logger log.Logger, rh *resultHistory) { moduleName := r.URL.Query().Get("module") if moduleName == "" { moduleName = "http_2xx" @@ -128,29 +128,12 @@ func probeHandler(w http.ResponseWriter, r *http.Request, c *config.Config, logg level.Error(sl).Log("msg", "Probe failed", "duration_seconds", duration) } - debug := false - if r.URL.Query().Get("debug") == "true" { - debug = true - } + debugOutput := DebugOutput(&module, &sl.buffer, registry) + rh.Add(moduleName, target, debugOutput, success) - if debug { + if r.URL.Query().Get("debug") == "true" { w.Header().Set("Content-Type", "text/plain") - fmt.Fprintf(w, "Logs for the probe:\n") - sl.buffer.WriteTo(w) - fmt.Fprintf(w, "\n\n\nMetrics that would have been returned:\n") - mfs, err := registry.Gather() - if err != nil { - fmt.Fprintf(w, "Error gathering metrics: %s\n", err) - } - for _, mf := range mfs { - expfmt.MetricFamilyToText(w, mf) - } - fmt.Fprintf(w, "\n\n\nModule configuration:\n") - c, err := yaml.Marshal(module) - if err != nil { - fmt.Fprintf(w, "Error marshalling config: %s\n", err) - } - w.Write(c) + debugOutput.WriteTo(w) return } @@ -190,6 +173,29 @@ func (sl scrapeLogger) Log(keyvals ...interface{}) error { return sl.next.Log(kvs...) } +// Returns plaintext debug output for a probe. +func DebugOutput(module *config.Module, logBuffer *bytes.Buffer, registry *prometheus.Registry) *bytes.Buffer { + buf := &bytes.Buffer{} + fmt.Fprintf(buf, "Logs for the probe:\n") + logBuffer.WriteTo(buf) + fmt.Fprintf(buf, "\n\n\nMetrics that would have been returned:\n") + mfs, err := registry.Gather() + if err != nil { + fmt.Fprintf(buf, "Error gathering metrics: %s\n", err) + } + for _, mf := range mfs { + expfmt.MetricFamilyToText(buf, mf) + } + fmt.Fprintf(buf, "\n\n\nModule configuration:\n") + c, err := yaml.Marshal(module) + if err != nil { + fmt.Fprintf(buf, "Error marshalling config: %s\n", err) + } + buf.Write(c) + + return buf +} + func init() { prometheus.MustRegister(version.NewCollector("blackbox_exporter")) } @@ -201,6 +207,7 @@ func main() { kingpin.HelpFlag.Short('h') kingpin.Parse() logger := promlog.New(allowedLevel) + rh := &resultHistory{} level.Info(logger).Log("msg", "Starting blackbox_exporter", "version", version.Info()) level.Info(logger).Log("msg", "Build context", version.BuildContext()) @@ -254,9 +261,10 @@ func main() { sc.Lock() conf := sc.C sc.Unlock() - probeHandler(w, r, conf, logger) + probeHandler(w, r, conf, logger, rh) }) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") w.Write([]byte(` Blackbox Exporter @@ -265,10 +273,40 @@ func main() {

Debug probe prometheus.io for http_2xx

Metrics

Configuration

- +

Recent Probes

+ `)) + + results := rh.List() + + for i := len(results) - 1; i > 0; i-- { + r := results[i] + success := "Success" + if !r.success { + success = "Failure" + } + fmt.Fprintf(w, "", + html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) + } + + w.Write([]byte(`
ModuleTargetResultDebug
%s%s%sLogs
`)) }) + http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.URL.Query().Get("id"), 10, 64) + if err != nil { + http.Error(w, "Invalid probe id", 500) + return + } + result := rh.Get(id) + if result == nil { + http.Error(w, "Probe id not found", 404) + return + } + w.Header().Set("Content-Type", "text/plain") + result.debugOutput.WriteTo(w) + }) + http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { sc.RLock() c, err := yaml.Marshal(sc.C) @@ -278,6 +316,7 @@ func main() { http.Error(w, err.Error(), 500) return } + w.Header().Set("Content-Type", "text/plain") w.Write(c) }) diff --git a/main_test.go b/main_test.go index 7d7488f..7bd66e6 100644 --- a/main_test.go +++ b/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "net/http" "net/http/httptest" "strings" @@ -8,6 +9,7 @@ import ( "time" "github.com/go-kit/kit/log" + "github.com/prometheus/client_golang/prometheus" pconfig "github.com/prometheus/common/config" "github.com/prometheus/blackbox_exporter/config" @@ -41,7 +43,7 @@ func TestPrometheusTimeoutHTTP(t *testing.T) { rr := httptest.NewRecorder() handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - probeHandler(w, r, c, log.NewNopLogger()) + probeHandler(w, r, c, log.NewNopLogger(), &resultHistory{}) }) handler.ServeHTTP(rr, req) @@ -63,7 +65,7 @@ func TestPrometheusConfigSecretsHidden(t *testing.T) { } rr := httptest.NewRecorder() handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - probeHandler(w, r, c, log.NewNopLogger()) + probeHandler(w, r, c, log.NewNopLogger(), &resultHistory{}) }) handler.ServeHTTP(rr, req) @@ -75,3 +77,16 @@ func TestPrometheusConfigSecretsHidden(t *testing.T) { t.Errorf("Hidden secret missing from debug config output: %v", body) } } + +func TestDebugOutputSecretsHidden(t *testing.T) { + module := c.Modules["http_2xx"] + buf := DebugOutput(&module, &bytes.Buffer{}, prometheus.NewRegistry()) + + out := buf.String() + if strings.Contains(out, "mysecret") { + t.Errorf("Secret exposed in debug output: %v", out) + } + if !strings.Contains(out, "") { + t.Errorf("Hidden secret missing from debug output: %v", out) + } +} -- 2.25.1