feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition
第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
/diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
route フィルタを付け、判定と同じ式にそろえる。
This commit is contained in:
1 parent
8836b37927
commit
f8e0aa9122
9 files changed
+229
-9
No files matched your search
@@ -230,12 +230,57 @@ func (b *Bot) edit(m *rag.HitlMessage, status string, color int, result map[stri
|
||||
return b.dc.Edit(m.ChannelID, m.MessageID, em, b.buttons(m.Token, true))
|
||||
}
|
||||
|
||||
// syncOne mirrors an approval recorded by the harness (API path, not the button) into the message. Returns true when
|
||||
// the message was decided here.
|
||||
func (b *Bot) syncOne(ctx context.Context, m *rag.HitlMessage) bool {
|
||||
st, err := b.harness.GetProposal(ctx, m.Token)
|
||||
if err != nil || !st.Approved {
|
||||
return false
|
||||
}
|
||||
by := "unknown"
|
||||
if st.ApprovedBy != nil && *st.ApprovedBy != "" {
|
||||
by = *st.ApprovedBy
|
||||
}
|
||||
first, err := b.store.Decide(ctx, m.Token, "approved", by, map[string]any{"via": "harness-api", "executed": st.Executed})
|
||||
if err != nil || !first {
|
||||
return false
|
||||
}
|
||||
b.disarm(m.Token)
|
||||
_ = b.edit(m, b.t.StatusApprovedAPI(by, st.Executed), colorApproved, nil)
|
||||
log.Printf("proposal %s approved outside Discord by %s (executed=%v)", m.Token[len(m.Token)-6:], by, st.Executed)
|
||||
return true
|
||||
}
|
||||
|
||||
// SyncLoop polls the harness for pending proposals decided outside Discord (campaign 3/4 2026-10-08: approvals through
|
||||
// POST /proposals/{token}/approve left the message to expire by timer).
|
||||
func (b *Bot) SyncLoop(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
list, err := b.store.Pending(ctx)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for i := range list {
|
||||
b.syncOne(ctx, &list[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bot) expire(token string) {
|
||||
ctx := context.Background()
|
||||
m, err := b.store.GetHitl(ctx, token)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if b.syncOne(ctx, m) {
|
||||
return
|
||||
}
|
||||
first, err := b.store.Decide(ctx, token, "expired", "timer", nil)
|
||||
if err != nil || !first {
|
||||
return
|
||||
|
||||
@@ -25,6 +25,7 @@ type Strings struct {
|
||||
StatusExecFailed func(by, err string) string
|
||||
StatusExpiredBefore string
|
||||
StatusNoDecision string
|
||||
StatusApprovedAPI func(by string, executed bool) string
|
||||
ExpiryMessage func(actionID, target, runID, scenarioID string) string
|
||||
EscalationTitle func(label string) string
|
||||
EscalationStatus string
|
||||
@@ -48,7 +49,13 @@ var langs = map[string]Strings{
|
||||
StatusApproveFailed: func(e string) string { return "approval failed: " + e },
|
||||
StatusExecFailed: func(by, e string) string { return "approved by " + by + "; execution failed: " + e },
|
||||
StatusExpiredBefore: "expired before approval",
|
||||
StatusNoDecision: "no decision within the window; moved to layer 3",
|
||||
StatusApprovedAPI: func(by string, executed bool) string {
|
||||
if executed {
|
||||
return "approved through the harness API by " + by + "; executed"
|
||||
}
|
||||
return "approved through the harness API by " + by
|
||||
},
|
||||
StatusNoDecision: "no decision within the window; moved to layer 3",
|
||||
ExpiryMessage: func(a, t, r, s string) string {
|
||||
return fmt.Sprintf("Proposal `%s` for `%s` (run %s, scenario %s) received no decision within the approval window. Layer 3: follow the runbook or call the harness `/actions/execute` with the operator token.", a, t, r, s)
|
||||
},
|
||||
@@ -72,7 +79,13 @@ var langs = map[string]Strings{
|
||||
StatusApproveFailed: func(e string) string { return "承認の記録に失敗: " + e },
|
||||
StatusExecFailed: func(by, e string) string { return by + " が承認したが実行に失敗: " + e },
|
||||
StatusExpiredBefore: "承認前に期限切れ",
|
||||
StatusNoDecision: "期限内に判断がなかったため第3層へ引き継ぎ",
|
||||
StatusApprovedAPI: func(by string, executed bool) string {
|
||||
if executed {
|
||||
return by + " がハーネス API で承認、実行済み"
|
||||
}
|
||||
return by + " がハーネス API で承認"
|
||||
},
|
||||
StatusNoDecision: "期限内に判断がなかったため第3層へ引き継ぎ",
|
||||
ExpiryMessage: func(a, t, r, s string) string {
|
||||
return fmt.Sprintf("提案 `%s`(対象 `%s`、run %s、シナリオ %s)は承認期限内に判断されませんでした。第3層の対応です。ランブックに従って手動で対処するか、オペレーター用トークンでハーネスの `/actions/execute` を呼び出してください。", a, t, r, s)
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user