From d0c9b46d3d3628f76e859cac33fecdf486e629f0 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Tue, 10 Sep 2019 04:48:50 -0700 Subject: [PATCH] Add separate history for expired failed probe results (#517) Signed-off-by: Jake Utley --- history.go | 30 +++++++++++++++--- history_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 history_test.go diff --git a/history.go b/history.go index 52e02c0..95b686d 100644 --- a/history.go +++ b/history.go @@ -25,11 +25,15 @@ type result struct { success bool } +// resultHistory contains two history slices: `results` contains most recent `maxResults` results. +// After they expire out of `results`, failures will be saved in `preservedFailedResults`. This +// ensures that we are always able to see debug information about recent failures. type resultHistory struct { - mu sync.Mutex - nextId int64 - results []*result - maxResults uint + mu sync.Mutex + nextId int64 + results []*result + preservedFailedResults []*result + maxResults uint } // Add a result to the history. @@ -48,6 +52,16 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo rh.results = append(rh.results, r) if uint(len(rh.results)) > rh.maxResults { + // If we are about to remove a failure, add it to the failed result history, then + // remove the oldest failed result, if needed. + if !rh.results[0].success { + rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0]) + if uint(len(rh.preservedFailedResults)) > rh.maxResults { + preservedFailedResults := make([]*result, len(rh.preservedFailedResults)-1) + copy(preservedFailedResults, rh.preservedFailedResults[1:]) + rh.preservedFailedResults = preservedFailedResults + } + } results := make([]*result, len(rh.results)-1) copy(results, rh.results[1:]) rh.results = results @@ -59,7 +73,8 @@ func (rh *resultHistory) List() []*result { rh.mu.Lock() defer rh.mu.Unlock() - return rh.results[:] + // Results in each slice are disjoint. We can simply concatenate the results. + return append(rh.preservedFailedResults[:], rh.results...) } // Get returns a given result. @@ -67,6 +82,11 @@ func (rh *resultHistory) Get(id int64) *result { rh.mu.Lock() defer rh.mu.Unlock() + for _, r := range rh.preservedFailedResults { + if r.id == id { + return r + } + } for _, r := range rh.results { if r.id == id { return r diff --git a/history_test.go b/history_test.go new file mode 100644 index 0000000..3fb3c49 --- /dev/null +++ b/history_test.go @@ -0,0 +1,81 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" +) + +func TestHistoryKeepsLatestResults(t *testing.T) { + history := &resultHistory{maxResults: 3} + for i := 0; i < 4; i++ { + history.Add("module", "target", fmt.Sprintf("result %d", i), true) + } + + savedResults := history.List() + for i := 0; i < len(savedResults); i++ { + if savedResults[i].debugOutput != fmt.Sprintf("result %d", i+1) { + t.Errorf("History contained the wrong result at index %d", i) + } + } +} + +func FillHistoryWithMaxSuccesses(h *resultHistory) { + for i := uint(0); i < h.maxResults; i++ { + h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), true) + } +} + +func FillHistoryWithMaxPreservedFailures(h *resultHistory) { + for i := uint(0); i < h.maxResults; i++ { + h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), false) + } +} + +func TestHistoryPreservesExpiredFailedResults(t *testing.T) { + history := &resultHistory{maxResults: 3} + + // Success are expired, no failues are expired + FillHistoryWithMaxSuccesses(history) + FillHistoryWithMaxPreservedFailures(history) + savedResults := history.List() + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + + // Failures are expired, should all be preserved + FillHistoryWithMaxPreservedFailures(history) + savedResults = history.List() + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + + // New expired failures are preserved, new success are not expired + FillHistoryWithMaxPreservedFailures(history) + FillHistoryWithMaxSuccesses(history) + savedResults = history.List() + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults*3) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } +} -- 2.25.1