feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution

キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状
(レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで
recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が
alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた
MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が
同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
seon committed 2026-10-08 19:20:35 +09:00
1 parent 13797e5741
commit cd8fbdddab
4 files changed
+109 -7

No files matched your search

+30 -1
View File
@@ -9,6 +9,7 @@ import (
"io"
"log"
"net/http"
"sort"
"sync"
"time"
)
@@ -22,10 +23,37 @@ type AlertRouter struct {
app *App
mu sync.Mutex
timers map[string]*time.Timer // fingerprint -> escalation timer
firing map[string]string // fingerprint -> alertname while Grafana reports it firing (judge: no recovery with an open alert)
}
func NewAlertRouter(a *App) *AlertRouter {
return &AlertRouter{app: a, timers: map[string]*time.Timer{}}
return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}}
}
// Firing lists the alert names currently firing (deduplicated, sorted).
func (r *AlertRouter) Firing() []string {
r.mu.Lock()
defer r.mu.Unlock()
seen := map[string]bool{}
var out []string
for _, n := range r.firing {
if !seen[n] {
seen[n] = true
out = append(out, n)
}
}
sort.Strings(out)
return out
}
func (r *AlertRouter) setFiring(fp, name string, on bool) {
r.mu.Lock()
defer r.mu.Unlock()
if on {
r.firing[fp] = name
} else {
delete(r.firing, fp)
}
}
// Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/)
@@ -80,6 +108,7 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) {
"startsAt": al.StartsAt, "fingerprint": al.Fingerprint, "layer": al.Labels["layer"], "signature": sig}
r.app.store.Event(bg, runID, sid, "alert", "grafana", payload)
handled++
r.setFiring(al.Fingerprint, name, al.Status == "firing")
if al.Status != "firing" {
r.cancelTimer(al.Fingerprint)
continue