feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution
キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状 (レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が 同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
1 parent
13797e5741
commit
cd8fbdddab
4 files changed
+109
-7
No files matched your search
+30
-1
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -22,10 +23,37 @@ type AlertRouter struct {
|
||||
app *App
|
||||
mu sync.Mutex
|
||||
timers map[string]*time.Timer // fingerprint -> escalation timer
|
||||
firing map[string]string // fingerprint -> alertname while Grafana reports it firing (judge: no recovery with an open alert)
|
||||
}
|
||||
|
||||
func NewAlertRouter(a *App) *AlertRouter {
|
||||
return &AlertRouter{app: a, timers: map[string]*time.Timer{}}
|
||||
return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}}
|
||||
}
|
||||
|
||||
// Firing lists the alert names currently firing (deduplicated, sorted).
|
||||
func (r *AlertRouter) Firing() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, n := range r.firing {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *AlertRouter) setFiring(fp, name string, on bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if on {
|
||||
r.firing[fp] = name
|
||||
} else {
|
||||
delete(r.firing, fp)
|
||||
}
|
||||
}
|
||||
|
||||
// Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/)
|
||||
@@ -80,6 +108,7 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) {
|
||||
"startsAt": al.StartsAt, "fingerprint": al.Fingerprint, "layer": al.Labels["layer"], "signature": sig}
|
||||
r.app.store.Event(bg, runID, sid, "alert", "grafana", payload)
|
||||
handled++
|
||||
r.setFiring(al.Fingerprint, name, al.Status == "firing")
|
||||
if al.Status != "firing" {
|
||||
r.cancelTimer(al.Fingerprint)
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user