Add separate history for expired failed probe results (#517)

Signed-off-by: Jake Utley <jutley@hiya.com>

diff --git a/history.go b/history.go
index 52e02c01d0387f7a097d72b2fe3c21d724f794d0..95b686deb4f87ab9bd016a729bfd0dfdee4ecce4 100644 (file)
--- a/history.go
+++ b/history.go
@@ -25,11 +25,15 @@ type result struct {
        success     bool
 }
 
+// resultHistory contains two history slices: `results` contains most recent `maxResults` results.
+// After they expire out of `results`, failures will be saved in `preservedFailedResults`. This
+// ensures that we are always able to see debug information about recent failures.
 type resultHistory struct {
-       mu         sync.Mutex
-       nextId     int64
-       results    []*result
-       maxResults uint
+       mu                     sync.Mutex
+       nextId                 int64
+       results                []*result
+       preservedFailedResults []*result
+       maxResults             uint
 }
 
 // Add a result to the history.
@@ -48,6 +52,16 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo
 
        rh.results = append(rh.results, r)
        if uint(len(rh.results)) > rh.maxResults {
+               // If we are about to remove a failure, add it to the failed result history, then
+               // remove the oldest failed result, if needed.
+               if !rh.results[0].success {
+                       rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0])
+                       if uint(len(rh.preservedFailedResults)) > rh.maxResults {
+                               preservedFailedResults := make([]*result, len(rh.preservedFailedResults)-1)
+                               copy(preservedFailedResults, rh.preservedFailedResults[1:])
+                               rh.preservedFailedResults = preservedFailedResults
+                       }
+               }
                results := make([]*result, len(rh.results)-1)
                copy(results, rh.results[1:])
                rh.results = results
@@ -59,7 +73,8 @@ func (rh *resultHistory) List() []*result {
        rh.mu.Lock()
        defer rh.mu.Unlock()
 
-       return rh.results[:]
+       // Results in each slice are disjoint. We can simply concatenate the results.
+       return append(rh.preservedFailedResults[:], rh.results...)
 }
 
 // Get returns a given result.
@@ -67,6 +82,11 @@ func (rh *resultHistory) Get(id int64) *result {
        rh.mu.Lock()
        defer rh.mu.Unlock()
 
+       for _, r := range rh.preservedFailedResults {
+               if r.id == id {
+                       return r
+               }
+       }
        for _, r := range rh.results {
                if r.id == id {
                        return r

diff --git a/history_test.go b/history_test.go
new file mode 100644 (file)
index 0000000..3fb3c49
--- /dev/null
+++ b/history_test.go
@@ -0,0 +1,81 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+       "fmt"
+       "testing"
+)
+
+func TestHistoryKeepsLatestResults(t *testing.T) {
+       history := &resultHistory{maxResults: 3}
+       for i := 0; i < 4; i++ {
+               history.Add("module", "target", fmt.Sprintf("result %d", i), true)
+       }
+
+       savedResults := history.List()
+       for i := 0; i < len(savedResults); i++ {
+               if savedResults[i].debugOutput != fmt.Sprintf("result %d", i+1) {
+                       t.Errorf("History contained the wrong result at index %d", i)
+               }
+       }
+}
+
+func FillHistoryWithMaxSuccesses(h *resultHistory) {
+       for i := uint(0); i < h.maxResults; i++ {
+               h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), true)
+       }
+}
+
+func FillHistoryWithMaxPreservedFailures(h *resultHistory) {
+       for i := uint(0); i < h.maxResults; i++ {
+               h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), false)
+       }
+}
+
+func TestHistoryPreservesExpiredFailedResults(t *testing.T) {
+       history := &resultHistory{maxResults: 3}
+
+       // Success are expired, no failues are expired
+       FillHistoryWithMaxSuccesses(history)
+       FillHistoryWithMaxPreservedFailures(history)
+       savedResults := history.List()
+       for i := uint(0); i < uint(len(savedResults)); i++ {
+               expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults)
+               if savedResults[i].debugOutput != expectedDebugOutput {
+                       t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+               }
+       }
+
+       // Failures are expired, should all be preserved
+       FillHistoryWithMaxPreservedFailures(history)
+       savedResults = history.List()
+       for i := uint(0); i < uint(len(savedResults)); i++ {
+               expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults)
+               if savedResults[i].debugOutput != expectedDebugOutput {
+                       t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+               }
+       }
+
+       // New expired failures are preserved, new success are not expired
+       FillHistoryWithMaxPreservedFailures(history)
+       FillHistoryWithMaxSuccesses(history)
+       savedResults = history.List()
+       for i := uint(0); i < uint(len(savedResults)); i++ {
+               expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults*3)
+               if savedResults[i].debugOutput != expectedDebugOutput {
+                       t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+               }
+       }
+}