feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition
第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
/diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
route フィルタを付け、判定と同じ式にそろえる。
This commit is contained in:
1 parent
8836b37927
commit
f8e0aa9122
9 files changed
+229
-9
No files matched your search
@@ -78,15 +78,15 @@ func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string
|
||||
rec := &ShadowRecord{Gate: "triage"}
|
||||
rec.state = map[string]any{
|
||||
"alert": compactAlert(req.Alert),
|
||||
"now": a.nowSummary(ctx),
|
||||
"now": a.nowSummary(ctx, req.Alert),
|
||||
"samples_last_10m": prefetched,
|
||||
"recent_error_logs": a.recentErrorLogs(ctx),
|
||||
"slo": "availability: probe_success = 1; latency: api p95 < 0.5 s; errors: 5xx ratio < 1%",
|
||||
}
|
||||
rec.questions = map[string]jev.Question{
|
||||
"label": {Type: "choice", Instructions: "Which root cause label best explains `alert`, `now`, `samples_last_10m` and `recent_error_logs`?", Criteria: labels},
|
||||
"recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.slo_now_ok` true, meaning the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?",
|
||||
Criteria: map[string]string{"true": "now.slo_now_ok is true: the symptom is gone in the newest samples", "false": "now.slo_now_ok is false, or the values are missing"}},
|
||||
"recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.recovered_now` true, meaning the alert's own condition no longer holds (`now.alert_condition_clear`) and the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?",
|
||||
Criteria: map[string]string{"true": "now.recovered_now is true: the alert condition is clear and the SLO values are inside", "false": "now.recovered_now is false, or the values are missing"}},
|
||||
"needs_investigation": {Type: "noul", Instructions: "To choose a safe remediation action, would an operator need evidence that is not in this state (more logs, Kubernetes objects, VM status, database replication status)?",
|
||||
Criteria: map[string]string{"true": "more evidence is needed before any action", "false": "this state alone is enough to decide the action"}},
|
||||
"irreversible_needed": {Type: "noul", Instructions: "Would fixing the root cause shown in this state require an irreversible or human-only operation: promoting the database replica, repairing a database schema or data, or starting or recreating a stopped VM?",
|
||||
@@ -98,7 +98,7 @@ func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string
|
||||
|
||||
// nowSummary computes the newest values and the SLO verdict in code (Jev reads text literally and should not compare
|
||||
// numbers or read the tail of a series: jev-1.13 jaggedness notes).
|
||||
func (a *Agent) nowSummary(ctx context.Context) map[string]any {
|
||||
func (a *Agent) nowSummary(ctx context.Context, alert map[string]any) map[string]any {
|
||||
q := func(promql string) any {
|
||||
v, err := a.tools.Instant(ctx, promql)
|
||||
if err != nil || v != v { // NaN
|
||||
@@ -110,8 +110,18 @@ func (a *Agent) nowSummary(ctx context.Context) map[string]any {
|
||||
probeDB := q(`probe_success{target="db-primary"}`)
|
||||
probeMin1m := q(`min_over_time(probe_success{target="alb"}[1m])`)
|
||||
probeMin3m := q(`min_over_time(probe_success{target="alb"}[3m])`)
|
||||
p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[1m])))`)
|
||||
errRatio := q(`sum(rate(demo_http_requests_total{status=~"5.."}[1m])) / sum(rate(demo_http_requests_total[1m]))`)
|
||||
p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m])))`)
|
||||
errRatio := q(`sum(rate(demo_http_requests_total{route=~"orders|price",status=~"5.."}[1m])) / sum(rate(demo_http_requests_total{route=~"orders|price"}[1m]))`)
|
||||
// the alert's own condition (Grafana annotation `condition`, the rule expression in bool form). A fault that does
|
||||
// not touch the SLO (replay paused, memory pressure) is "recovering" only when this clears (campaign 3/4: 0.98).
|
||||
var condNow any
|
||||
condClear := true
|
||||
if cond := alertCondition(alert); cond != "" {
|
||||
condNow = q("max(" + cond + ")")
|
||||
if v, ok := condNow.(float64); ok {
|
||||
condClear = v == 0
|
||||
}
|
||||
}
|
||||
restarts := q(`sum(increase(kube_pod_container_status_restarts_total{namespace="demo"}[10m]))`)
|
||||
legacyUp := q(`count(up{job="legacy-api"} == 1)`)
|
||||
f := func(v any) (float64, bool) {
|
||||
@@ -127,10 +137,22 @@ func (a *Agent) nowSummary(ctx context.Context) map[string]any {
|
||||
return map[string]any{
|
||||
"probe_alb_now": probeALB, "probe_db_primary_now": probeDB, "probe_alb_min_1m": probeMin1m, "probe_alb_min_3m": probeMin3m,
|
||||
"api_p95_1m_s": p95, "api_error_ratio_1m": errRatio, "api_restarts_10m": restarts, "legacy_instances_up": legacyUp,
|
||||
"alert_condition_now": condNow, "alert_condition_clear": condClear,
|
||||
"probe_ok_now": probeOK, "latency_ok_now": latOK, "errors_ok_now": errOK, "slo_now_ok": probeOK && latOK && errOK,
|
||||
"recovered_now": probeOK && latOK && errOK && condClear,
|
||||
}
|
||||
}
|
||||
|
||||
// alertCondition returns the rule expression carried in the alert annotations (empty when absent).
|
||||
func alertCondition(alert map[string]any) string {
|
||||
ann, _ := alert["annotations"].(map[string]any)
|
||||
if ann == nil {
|
||||
return ""
|
||||
}
|
||||
c, _ := ann["condition"].(string)
|
||||
return c
|
||||
}
|
||||
|
||||
// recentErrorLogs returns a short, newest-first excerpt of api error lines (5 minutes, 8 lines) for the triage state.
|
||||
func (a *Agent) recentErrorLogs(ctx context.Context) string {
|
||||
out := a.tools.Call(ctx, "query_logs", `{"logql":"{namespace=\"demo\", app=\"api\"} |~ \"(?i)error|timeout|refused|SQLSTATE\"","range_minutes":5,"limit":8}`)
|
||||
|
||||
Reference in new issue
Block a user