feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition

第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
  /diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
  出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
  全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
  メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
  直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
  recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
  障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
  route フィルタを付け、判定と同じ式にそろえる。
This commit is contained in:
seon committed 2026-10-09 08:33:41 +09:00
1 parent 8836b37927
commit f8e0aa9122
9 files changed
+229 -9

No files matched your search

+1
View File
@@ -59,6 +59,7 @@ func main() {
}
connected = true
bot.RearmPending(ctx)
go bot.SyncLoop(ctx, 60*time.Second)
}
mux := http.NewServeMux()
+28 -6
View File
@@ -78,15 +78,15 @@ func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string
rec := &ShadowRecord{Gate: "triage"}
rec.state = map[string]any{
"alert": compactAlert(req.Alert),
"now": a.nowSummary(ctx),
"now": a.nowSummary(ctx, req.Alert),
"samples_last_10m": prefetched,
"recent_error_logs": a.recentErrorLogs(ctx),
"slo": "availability: probe_success = 1; latency: api p95 < 0.5 s; errors: 5xx ratio < 1%",
}
rec.questions = map[string]jev.Question{
"label": {Type: "choice", Instructions: "Which root cause label best explains `alert`, `now`, `samples_last_10m` and `recent_error_logs`?", Criteria: labels},
"recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.slo_now_ok` true, meaning the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?",
Criteria: map[string]string{"true": "now.slo_now_ok is true: the symptom is gone in the newest samples", "false": "now.slo_now_ok is false, or the values are missing"}},
"recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.recovered_now` true, meaning the alert's own condition no longer holds (`now.alert_condition_clear`) and the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?",
Criteria: map[string]string{"true": "now.recovered_now is true: the alert condition is clear and the SLO values are inside", "false": "now.recovered_now is false, or the values are missing"}},
"needs_investigation": {Type: "noul", Instructions: "To choose a safe remediation action, would an operator need evidence that is not in this state (more logs, Kubernetes objects, VM status, database replication status)?",
Criteria: map[string]string{"true": "more evidence is needed before any action", "false": "this state alone is enough to decide the action"}},
"irreversible_needed": {Type: "noul", Instructions: "Would fixing the root cause shown in this state require an irreversible or human-only operation: promoting the database replica, repairing a database schema or data, or starting or recreating a stopped VM?",
@@ -98,7 +98,7 @@ func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string
// nowSummary computes the newest values and the SLO verdict in code (Jev reads text literally and should not compare
// numbers or read the tail of a series: jev-1.13 jaggedness notes).
func (a *Agent) nowSummary(ctx context.Context) map[string]any {
func (a *Agent) nowSummary(ctx context.Context, alert map[string]any) map[string]any {
q := func(promql string) any {
v, err := a.tools.Instant(ctx, promql)
if err != nil || v != v { // NaN
@@ -110,8 +110,18 @@ func (a *Agent) nowSummary(ctx context.Context) map[string]any {
probeDB := q(`probe_success{target="db-primary"}`)
probeMin1m := q(`min_over_time(probe_success{target="alb"}[1m])`)
probeMin3m := q(`min_over_time(probe_success{target="alb"}[3m])`)
p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[1m])))`)
errRatio := q(`sum(rate(demo_http_requests_total{status=~"5.."}[1m])) / sum(rate(demo_http_requests_total[1m]))`)
p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m])))`)
errRatio := q(`sum(rate(demo_http_requests_total{route=~"orders|price",status=~"5.."}[1m])) / sum(rate(demo_http_requests_total{route=~"orders|price"}[1m]))`)
// the alert's own condition (Grafana annotation `condition`, the rule expression in bool form). A fault that does
// not touch the SLO (replay paused, memory pressure) is "recovering" only when this clears (campaign 3/4: 0.98).
var condNow any
condClear := true
if cond := alertCondition(alert); cond != "" {
condNow = q("max(" + cond + ")")
if v, ok := condNow.(float64); ok {
condClear = v == 0
}
}
restarts := q(`sum(increase(kube_pod_container_status_restarts_total{namespace="demo"}[10m]))`)
legacyUp := q(`count(up{job="legacy-api"} == 1)`)
f := func(v any) (float64, bool) {
@@ -127,10 +137,22 @@ func (a *Agent) nowSummary(ctx context.Context) map[string]any {
return map[string]any{
"probe_alb_now": probeALB, "probe_db_primary_now": probeDB, "probe_alb_min_1m": probeMin1m, "probe_alb_min_3m": probeMin3m,
"api_p95_1m_s": p95, "api_error_ratio_1m": errRatio, "api_restarts_10m": restarts, "legacy_instances_up": legacyUp,
"alert_condition_now": condNow, "alert_condition_clear": condClear,
"probe_ok_now": probeOK, "latency_ok_now": latOK, "errors_ok_now": errOK, "slo_now_ok": probeOK && latOK && errOK,
"recovered_now": probeOK && latOK && errOK && condClear,
}
}
// alertCondition returns the rule expression carried in the alert annotations (empty when absent).
func alertCondition(alert map[string]any) string {
ann, _ := alert["annotations"].(map[string]any)
if ann == nil {
return ""
}
c, _ := ann["condition"].(string)
return c
}
// recentErrorLogs returns a short, newest-first excerpt of api error lines (5 minutes, 8 lines) for the triage state.
func (a *Agent) recentErrorLogs(ctx context.Context) string {
out := a.tools.Call(ctx, "query_logs", `{"logql":"{namespace=\"demo\", app=\"api\"} |~ \"(?i)error|timeout|refused|SQLSTATE\"","range_minutes":5,"limit":8}`)
+21
View File
@@ -57,6 +57,27 @@ type Proposal struct {
Approved bool `json:"approved"`
}
// ProposalState is GET /proposals/{token}: the decision as the harness knows it (approval may come through the API).
type ProposalState struct {
Token string `json:"token"`
ActionID string `json:"action_id"`
Approved bool `json:"approved"`
ApprovedAt *string `json:"approved_at"`
ApprovedBy *string `json:"approved_by"`
Executed bool `json:"executed"`
ExecutedAt *string `json:"executed_at"`
ExpiresAt string `json:"expires_at"`
}
// GetProposal reads the harness-side state of a proposal.
func (c *Client) GetProposal(ctx context.Context, token string) (*ProposalState, error) {
var out ProposalState
if _, err := c.call(ctx, "GET", "/proposals/"+token, nil, &out); err != nil {
return nil, err
}
return &out, nil
}
// Propose registers a layer-2 proposal (POST /proposals); the harness records the proposal event and returns the token.
func (c *Client) Propose(ctx context.Context, runID, scenarioID *string, actionID string, target, diagnosis map[string]any, ttl time.Duration) (*Proposal, error) {
body := map[string]any{"run_id": runID, "scenario_id": scenarioID, "action_id": actionID, "target": target, "diagnosis": diagnosis}
+45
View File
@@ -230,12 +230,57 @@ func (b *Bot) edit(m *rag.HitlMessage, status string, color int, result map[stri
return b.dc.Edit(m.ChannelID, m.MessageID, em, b.buttons(m.Token, true))
}
// syncOne mirrors an approval recorded by the harness (API path, not the button) into the message. Returns true when
// the message was decided here.
func (b *Bot) syncOne(ctx context.Context, m *rag.HitlMessage) bool {
st, err := b.harness.GetProposal(ctx, m.Token)
if err != nil || !st.Approved {
return false
}
by := "unknown"
if st.ApprovedBy != nil && *st.ApprovedBy != "" {
by = *st.ApprovedBy
}
first, err := b.store.Decide(ctx, m.Token, "approved", by, map[string]any{"via": "harness-api", "executed": st.Executed})
if err != nil || !first {
return false
}
b.disarm(m.Token)
_ = b.edit(m, b.t.StatusApprovedAPI(by, st.Executed), colorApproved, nil)
log.Printf("proposal %s approved outside Discord by %s (executed=%v)", m.Token[len(m.Token)-6:], by, st.Executed)
return true
}
// SyncLoop polls the harness for pending proposals decided outside Discord (campaign 3/4 2026-10-08: approvals through
// POST /proposals/{token}/approve left the message to expire by timer).
func (b *Bot) SyncLoop(ctx context.Context, every time.Duration) {
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
list, err := b.store.Pending(ctx)
if err != nil {
continue
}
for i := range list {
b.syncOne(ctx, &list[i])
}
}
}
}
func (b *Bot) expire(token string) {
ctx := context.Background()
m, err := b.store.GetHitl(ctx, token)
if err != nil {
return
}
if b.syncOne(ctx, m) {
return
}
first, err := b.store.Decide(ctx, token, "expired", "timer", nil)
if err != nil || !first {
return
+15 -2
View File
@@ -25,6 +25,7 @@ type Strings struct {
StatusExecFailed func(by, err string) string
StatusExpiredBefore string
StatusNoDecision string
StatusApprovedAPI func(by string, executed bool) string
ExpiryMessage func(actionID, target, runID, scenarioID string) string
EscalationTitle func(label string) string
EscalationStatus string
@@ -48,7 +49,13 @@ var langs = map[string]Strings{
StatusApproveFailed: func(e string) string { return "approval failed: " + e },
StatusExecFailed: func(by, e string) string { return "approved by " + by + "; execution failed: " + e },
StatusExpiredBefore: "expired before approval",
StatusNoDecision: "no decision within the window; moved to layer 3",
StatusApprovedAPI: func(by string, executed bool) string {
if executed {
return "approved through the harness API by " + by + "; executed"
}
return "approved through the harness API by " + by
},
StatusNoDecision: "no decision within the window; moved to layer 3",
ExpiryMessage: func(a, t, r, s string) string {
return fmt.Sprintf("Proposal `%s` for `%s` (run %s, scenario %s) received no decision within the approval window. Layer 3: follow the runbook or call the harness `/actions/execute` with the operator token.", a, t, r, s)
},
@@ -72,7 +79,13 @@ var langs = map[string]Strings{
StatusApproveFailed: func(e string) string { return "承認の記録に失敗: " + e },
StatusExecFailed: func(by, e string) string { return by + " が承認したが実行に失敗: " + e },
StatusExpiredBefore: "承認前に期限切れ",
StatusNoDecision: "期限内に判断がなかったため第3層へ引き継ぎ",
StatusApprovedAPI: func(by string, executed bool) string {
if executed {
return by + " がハーネス API で承認、実行済み"
}
return by + " がハーネス API で承認"
},
StatusNoDecision: "期限内に判断がなかったため第3層へ引き継ぎ",
ExpiryMessage: func(a, t, r, s string) string {
return fmt.Sprintf("提案 `%s`(対象 `%s`、run %s、シナリオ %s)は承認期限内に判断されませんでした。第3層の対応です。ランブックに従って手動で対処するか、オペレーター用トークンでハーネスの `/actions/execute` を呼び出してください。", a, t, r, s)
},