feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition
第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
/diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
route フィルタを付け、判定と同じ式にそろえる。
This commit is contained in:
1 parent
8836b37927
commit
f8e0aa9122
9 files changed
+229
-9
No files matched your search
+48
-1
@@ -24,10 +24,14 @@ type AlertRouter struct {
|
||||
mu sync.Mutex
|
||||
timers map[string]*time.Timer // fingerprint -> escalation timer
|
||||
firing map[string]string // fingerprint -> alertname while Grafana reports it firing (judge: no recovery with an open alert)
|
||||
// alertname+scenario -> when the last layer-2 diagnosis was dispatched. Grafana emits one alert per label set
|
||||
// (ApiMemoryPressure: one per pod) and resends the group 30 s later; campaign 4 (2026-10-08) turned one fault into
|
||||
// four diagnoses and four proposals. Inside cfg.AlertDedupeS the later alerts are recorded and skipped.
|
||||
lastDiag map[string]time.Time
|
||||
}
|
||||
|
||||
func NewAlertRouter(a *App) *AlertRouter {
|
||||
return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}}
|
||||
return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}, lastDiag: map[string]time.Time{}}
|
||||
}
|
||||
|
||||
// Firing lists the alert names currently firing (deduplicated, sorted).
|
||||
@@ -113,6 +117,7 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) {
|
||||
r.setFiring(al.Fingerprint, name, al.Status == "firing" && sig == "")
|
||||
if al.Status != "firing" {
|
||||
r.cancelTimer(al.Fingerprint)
|
||||
r.clearDedupe(name, sid)
|
||||
continue
|
||||
}
|
||||
if sig != "" {
|
||||
@@ -129,11 +134,53 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) {
|
||||
r.startEscalation(al.Fingerprint, payload, runID, sid)
|
||||
continue
|
||||
}
|
||||
if since, dup := r.dedupe(name, sid); dup {
|
||||
r.app.store.Event(bg, runID, sid, "note", "runner", map[string]any{"phase": "dedupe", "alertname": name, "fingerprint": al.Fingerprint,
|
||||
"since_last_diagnosis_s": since.Seconds(), "note": "same alert name already under diagnosis; no second /diagnose"})
|
||||
continue
|
||||
}
|
||||
go r.diagnose(payload, runID, sid)
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"received": handled, "run_id": runID, "scenario_id": sid})
|
||||
}
|
||||
|
||||
// dedupe reports whether a layer-2 diagnosis for this alert name (within the scenario) was dispatched less than
|
||||
// cfg.AlertDedupeS ago; when not, it records now as the dispatch time.
|
||||
func (r *AlertRouter) dedupe(name string, sid *string) (time.Duration, bool) {
|
||||
if r.app.cfg.AlertDedupeS <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
key := name
|
||||
if sid != nil {
|
||||
key += "|" + *sid
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
now := time.Now()
|
||||
if t, ok := r.lastDiag[key]; ok && now.Sub(t) < time.Duration(r.app.cfg.AlertDedupeS)*time.Second {
|
||||
return now.Sub(t), true
|
||||
}
|
||||
r.lastDiag[key] = now
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// clearDedupe forgets the dispatch time once no alert of that name is firing any more, so the next firing starts a
|
||||
// new diagnosis even inside the window.
|
||||
func (r *AlertRouter) clearDedupe(name string, sid *string) {
|
||||
key := name
|
||||
if sid != nil {
|
||||
key += "|" + *sid
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, n := range r.firing {
|
||||
if n == name {
|
||||
return
|
||||
}
|
||||
}
|
||||
delete(r.lastDiag, key)
|
||||
}
|
||||
|
||||
func targetFromLabels(l map[string]string) map[string]any {
|
||||
t := map[string]any{}
|
||||
for _, k := range []string{"namespace", "pod", "deployment", "node", "instance"} {
|
||||
|
||||
Reference in new issue
Block a user