Files
selfheal-lab-apps/harness/proposals.go
T
seon f8e0aa9122 feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition
第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
  /diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
  出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
  全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
  メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
  直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
  recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
  障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
  route フィルタを付け、判定と同じ式にそろえる。
2026-10-09 08:33:41 +09:00

125 lines
4.9 KiB
Go

package main
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"time"
)
// POST /proposals: the AI VM registers a layer-2 proposal (design 6.5). Response carries the approval token that the
// Discord approval hands back; approved=true in the body records the approval directly (used by tests and by the AI VM
// after the button round-trip). Execution then goes through /actions/execute with that token.
type ProposalRequest struct {
RunID *string `json:"run_id"`
ScenarioID *string `json:"scenario_id"`
ActionID string `json:"action_id"`
Target map[string]any `json:"target"`
Diagnosis map[string]any `json:"diagnosis"`
Approved bool `json:"approved"`
ApprovedBy string `json:"approved_by"`
TTLSeconds int `json:"ttl_seconds"`
}
func (a *App) handleProposal(w http.ResponseWriter, r *http.Request) {
var req ProposalRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, 400, err)
return
}
if catalogEntry(req.ActionID) == nil {
writeErr(w, 400, errors.New("action_id not in the catalogue"))
return
}
if cur := a.runs.current(); cur != nil && req.RunID == nil {
req.RunID = &cur.RunID
if cur.ScenarioID != "" {
req.ScenarioID = &cur.ScenarioID
}
}
ttl := 15 * time.Minute // design 2.1: no approval within 15 minutes -> layer 3
if req.TTLSeconds > 0 {
ttl = time.Duration(req.TTLSeconds) * time.Second
}
b := make([]byte, 24)
_, _ = rand.Read(b)
token := "apr-" + hex.EncodeToString(b)
if err := a.store.InsertProposal(r.Context(), token, req.RunID, req.ScenarioID, req.ActionID, req.Target, ttl); err != nil {
writeErr(w, 500, err)
return
}
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "proposal", "ai", map[string]any{"action_id": req.ActionID, "target": req.Target, "diagnosis": req.Diagnosis, "token_suffix": token[len(token)-6:]})
if req.Approved {
by := req.ApprovedBy
if by == "" {
by = "unspecified"
}
_ = a.store.ApproveProposal(r.Context(), token, by)
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]})
}
writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved})
}
// GET /proposals/{token}: the HITL bot reads the decision state (approval may arrive through this API instead of the
// Discord button; the bot mirrors it into its message instead of letting the message expire, campaign 3/4 2026-10-08).
func (a *App) handleProposalGet(w http.ResponseWriter, r *http.Request) {
p, err := a.store.GetProposal(r.Context(), r.PathValue("token"))
if err != nil {
writeErr(w, 404, errors.New("unknown proposal"))
return
}
writeJSON(w, 200, map[string]any{"token": p.Token, "run_id": p.RunID, "scenario_id": p.ScenarioID, "action_id": p.ActionID,
"approved": p.ApprovedAt != nil, "approved_at": p.ApprovedAt, "approved_by": p.ApprovedBy, "executed": p.ExecutedAt != nil, "executed_at": p.ExecutedAt, "expires_at": p.ExpiresAt})
}
// POST /proposals/{token}/approve: the HITL bot records the human decision (design 6.5 step 4). The proposal was
// registered at diagnosis time, so MTTA = approval - proposal (design 5.4) measures the real waiting time.
func (a *App) handleProposalApprove(w http.ResponseWriter, r *http.Request) {
token := r.PathValue("token")
var req struct {
ApprovedBy string `json:"approved_by"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
if req.ApprovedBy == "" {
req.ApprovedBy = "unspecified"
}
p, err := a.store.GetProposal(r.Context(), token)
if err != nil {
writeErr(w, 404, errors.New("unknown proposal"))
return
}
switch {
case p.ApprovedAt != nil:
writeErr(w, 409, errors.New("proposal already approved"))
return
case time.Now().After(p.ExpiresAt):
writeErr(w, 410, errors.New("proposal expired"))
return
}
if err := a.store.ApproveProposal(r.Context(), token, req.ApprovedBy); err != nil {
writeErr(w, 500, err)
return
}
a.store.Event(r.Context(), p.RunID, p.ScenarioID, "approval", req.ApprovedBy, map[string]any{"action_id": p.ActionID, "token_suffix": token[len(token)-6:]})
writeJSON(w, 200, map[string]any{"token": token, "approved": true, "approved_by": req.ApprovedBy})
}
// POST /proposals/{token}/reject: recorded as a note event (kind list of design 5.3); the token stays unapproved.
func (a *App) handleProposalReject(w http.ResponseWriter, r *http.Request) {
token := r.PathValue("token")
var req struct {
RejectedBy string `json:"rejected_by"`
Reason string `json:"reason"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
p, err := a.store.GetProposal(r.Context(), token)
if err != nil {
writeErr(w, 404, errors.New("unknown proposal"))
return
}
a.store.Event(r.Context(), p.RunID, p.ScenarioID, "note", req.RejectedBy, map[string]any{"rejected": true, "action_id": p.ActionID, "reason": req.Reason, "token_suffix": token[len(token)-6:]})
writeJSON(w, 200, map[string]any{"token": token, "rejected": true})
}