diff --git a/grafana-alertcheck/.changeset/v0.1.1.md b/grafana-alertcheck/.changeset/v0.1.1.md new file mode 100644 index 000000000..9fd255891 --- /dev/null +++ b/grafana-alertcheck/.changeset/v0.1.1.md @@ -0,0 +1 @@ +- Shorter summary \ No newline at end of file diff --git a/grafana-alertcheck/cmd/table.go b/grafana-alertcheck/cmd/table.go index 39ec75cbc..26a8944de 100644 --- a/grafana-alertcheck/cmd/table.go +++ b/grafana-alertcheck/cmd/table.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "sort" + "strconv" "text/tabwriter" "time" @@ -19,12 +20,7 @@ import ( // // 1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not // with the largest gap; -// 2. VIOLATIONS, one line per Violation (only when any): a rule's worst-of -// outcome does not carry the State/Health of the instance that actually -// caused it — Violation does — so this is also where those two columns -// appear, sorted after the result table rather than folded into it, and it -// is the only place an operator running WITHOUT --output json sees the -// --allow-paused hint that Violation.Note already carries (classify.go); +// 2. VIOLATIONS, one line per distinct violation. // 3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped // rule's maxGap/healthGrace/evalStaleAfter, followed by the global // transitionGrace and drainTimeout, and the largest measured clock skew @@ -59,9 +55,9 @@ func renderTable(w io.Writer, res gate.Result) error { if len(res.Violations) > 0 { fmt.Fprintln(w, "\nVIOLATIONS") vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tNOTE") - for _, v := range sortedViolations(res.Violations) { - fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note) + fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tINSTANCE COUNT\tNOTE") + for _, g := range groupedViolations(res.Violations) { + fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\t%s\n", alertLabel(g.v, alertOf), g.v.Outcome, g.v.State, g.v.Health, instanceCount(g), g.v.Note) } if err := vtw.Flush(); err != nil { return fmt.Errorf("render table: %w", err) @@ -157,12 +153,42 @@ func sortedVerdicts(in []gate.RuleVerdict) []gate.RuleVerdict { return out } -func sortedViolations(in []gate.Violation) []gate.Violation { - out := append([]gate.Violation(nil), in...) - sort.SliceStable(out, func(i, j int) bool { return out[i].Alert < out[j].Alert }) +type violationGroup struct { + v gate.Violation + n int +} + +func groupedViolations(in []gate.Violation) []violationGroup { + sorted := append([]gate.Violation(nil), in...) + sort.SliceStable(sorted, func(i, j int) bool { + return violationSignature(sorted[i]) < violationSignature(sorted[j]) + }) + var out []violationGroup + for _, v := range sorted { + if n := len(out); n > 0 && sameRendered(out[n-1].v, v) { + out[n-1].n++ + } else { + out = append(out, violationGroup{v: v, n: 1}) + } + } return out } +func violationSignature(v gate.Violation) string { + return v.Alert + "\x00" + v.RuleUID + "\x00" + string(v.Outcome) + "\x00" + string(v.State) + "\x00" + v.Health + "\x00" + v.Note +} + +func sameRendered(a, b gate.Violation) bool { + return violationSignature(a) == violationSignature(b) +} + +func instanceCount(g violationGroup) string { + if g.v.Outcome == gate.OutcomeSkipped { + return "-" + } + return strconv.Itoa(g.n) +} + func sortedThresholdUIDs(thresholds map[string]gate.RuleThresholds, alertOf map[string]string) []string { uids := make([]string, 0, len(thresholds)) for uid := range thresholds { diff --git a/grafana-alertcheck/cmd/table_test.go b/grafana-alertcheck/cmd/table_test.go index 680829e12..b050f0d30 100644 --- a/grafana-alertcheck/cmd/table_test.go +++ b/grafana-alertcheck/cmd/table_test.go @@ -65,6 +65,7 @@ func TestRenderTable(t *testing.T) { require.Contains(t, out, "--allow-paused") require.Contains(t, out, "STATE") require.Contains(t, out, "HEALTH") + require.Contains(t, out, "INSTANCE COUNT") require.Contains(t, out, string(gate.StateFiring)) require.Contains(t, out, "error") @@ -87,3 +88,27 @@ func TestRenderTable(t *testing.T) { func TestProvedLabel_Skipped(t *testing.T) { require.Equal(t, "-", provedLabel(gate.CoverageResult{})) } + +// groupedViolations collapses a rule's many firing instances into one row per +// rendered signature, each with a count. +func TestGroupedViolations(t *testing.T) { + in := []gate.Violation{ + {Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"}, + {Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"}, + {Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"}, + {Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "error"}, + {Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"}, + {Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"}, + } + + got := groupedViolations(in) + + require.Len(t, got, 3) + counts := map[string]int{} + for _, g := range got { + counts[g.v.Health+"|"+string(g.v.Outcome)] = g.n + } + require.Equal(t, 3, counts["ok|"+string(gate.OutcomePersistentlyBad)]) + require.Equal(t, 1, counts["error|"+string(gate.OutcomePersistentlyBad)]) + require.Equal(t, 2, counts["ok|"+string(gate.OutcomeNewlyBad)]) +} diff --git a/grafana-alertcheck/docs/reference/cli.md b/grafana-alertcheck/docs/reference/cli.md index da58f019d..52f1a86d6 100644 --- a/grafana-alertcheck/docs/reference/cli.md +++ b/grafana-alertcheck/docs/reference/cli.md @@ -82,7 +82,7 @@ Datasource-managed and recording rules are refused with a specific error. A name ## Output and exit codes -The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per violation), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout. +The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per distinct rule/outcome/state/health/note signature, with a `COUNT` of the instances it stands for — instance identity is only in the JSON), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout. | Code | Meaning | | ---- | ------- |