Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions grafana-alertcheck/.changeset/v0.1.1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Shorter summary
50 changes: 38 additions & 12 deletions grafana-alertcheck/cmd/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"
"sort"
"strconv"
"text/tabwriter"
"time"

Expand All @@ -19,12 +20,7 @@ import (
//
// 1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not
// with the largest gap;
// 2. VIOLATIONS, one line per Violation (only when any): a rule's worst-of
// outcome does not carry the State/Health of the instance that actually
// caused it — Violation does — so this is also where those two columns
// appear, sorted after the result table rather than folded into it, and it
// is the only place an operator running WITHOUT --output json sees the
// --allow-paused hint that Violation.Note already carries (classify.go);
// 2. VIOLATIONS, one line per distinct violation.
// 3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped
// rule's maxGap/healthGrace/evalStaleAfter, followed by the global
// transitionGrace and drainTimeout, and the largest measured clock skew
Expand Down Expand Up @@ -59,9 +55,9 @@ func renderTable(w io.Writer, res gate.Result) error {
if len(res.Violations) > 0 {
fmt.Fprintln(w, "\nVIOLATIONS")
vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tNOTE")
for _, v := range sortedViolations(res.Violations) {
fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note)
fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tINSTANCE COUNT\tNOTE")
Comment thread
Tofel marked this conversation as resolved.
for _, g := range groupedViolations(res.Violations) {
fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\t%s\n", alertLabel(g.v, alertOf), g.v.Outcome, g.v.State, g.v.Health, instanceCount(g), g.v.Note)
}
if err := vtw.Flush(); err != nil {
return fmt.Errorf("render table: %w", err)
Expand Down Expand Up @@ -157,12 +153,42 @@ func sortedVerdicts(in []gate.RuleVerdict) []gate.RuleVerdict {
return out
}

func sortedViolations(in []gate.Violation) []gate.Violation {
out := append([]gate.Violation(nil), in...)
sort.SliceStable(out, func(i, j int) bool { return out[i].Alert < out[j].Alert })
type violationGroup struct {
v gate.Violation
n int
}

func groupedViolations(in []gate.Violation) []violationGroup {
sorted := append([]gate.Violation(nil), in...)
sort.SliceStable(sorted, func(i, j int) bool {
return violationSignature(sorted[i]) < violationSignature(sorted[j])
})
Comment thread
Tofel marked this conversation as resolved.
var out []violationGroup
for _, v := range sorted {
if n := len(out); n > 0 && sameRendered(out[n-1].v, v) {
out[n-1].n++
} else {
out = append(out, violationGroup{v: v, n: 1})
}
}
return out
}

func violationSignature(v gate.Violation) string {
return v.Alert + "\x00" + v.RuleUID + "\x00" + string(v.Outcome) + "\x00" + string(v.State) + "\x00" + v.Health + "\x00" + v.Note
}

func sameRendered(a, b gate.Violation) bool {
return violationSignature(a) == violationSignature(b)
}

func instanceCount(g violationGroup) string {
if g.v.Outcome == gate.OutcomeSkipped {
return "-"
}
return strconv.Itoa(g.n)
}

func sortedThresholdUIDs(thresholds map[string]gate.RuleThresholds, alertOf map[string]string) []string {
uids := make([]string, 0, len(thresholds))
for uid := range thresholds {
Expand Down
25 changes: 25 additions & 0 deletions grafana-alertcheck/cmd/table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ func TestRenderTable(t *testing.T) {
require.Contains(t, out, "--allow-paused")
require.Contains(t, out, "STATE")
require.Contains(t, out, "HEALTH")
require.Contains(t, out, "INSTANCE COUNT")
require.Contains(t, out, string(gate.StateFiring))
require.Contains(t, out, "error")

Expand All @@ -87,3 +88,27 @@ func TestRenderTable(t *testing.T) {
func TestProvedLabel_Skipped(t *testing.T) {
require.Equal(t, "-", provedLabel(gate.CoverageResult{}))
}

// groupedViolations collapses a rule's many firing instances into one row per
// rendered signature, each with a count.
func TestGroupedViolations(t *testing.T) {
in := []gate.Violation{
{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "error"},
{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"},
{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"},
}

got := groupedViolations(in)

require.Len(t, got, 3)
counts := map[string]int{}
for _, g := range got {
counts[g.v.Health+"|"+string(g.v.Outcome)] = g.n
}
require.Equal(t, 3, counts["ok|"+string(gate.OutcomePersistentlyBad)])
require.Equal(t, 1, counts["error|"+string(gate.OutcomePersistentlyBad)])
require.Equal(t, 2, counts["ok|"+string(gate.OutcomeNewlyBad)])
}
2 changes: 1 addition & 1 deletion grafana-alertcheck/docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ Datasource-managed and recording rules are refused with a specific error. A name

## Output and exit codes

The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per violation), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout.
The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per distinct rule/outcome/state/health/note signature, with a `COUNT` of the instances it stands for — instance identity is only in the JSON), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout.

| Code | Meaning |
| ---- | ------- |
Expand Down
Loading