第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
/diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
route フィルタを付け、判定と同じ式にそろえる。
125 lines
4.9 KiB
Go
125 lines
4.9 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// POST /proposals: the AI VM registers a layer-2 proposal (design 6.5). Response carries the approval token that the
|
|
// Discord approval hands back; approved=true in the body records the approval directly (used by tests and by the AI VM
|
|
// after the button round-trip). Execution then goes through /actions/execute with that token.
|
|
type ProposalRequest struct {
|
|
RunID *string `json:"run_id"`
|
|
ScenarioID *string `json:"scenario_id"`
|
|
ActionID string `json:"action_id"`
|
|
Target map[string]any `json:"target"`
|
|
Diagnosis map[string]any `json:"diagnosis"`
|
|
Approved bool `json:"approved"`
|
|
ApprovedBy string `json:"approved_by"`
|
|
TTLSeconds int `json:"ttl_seconds"`
|
|
}
|
|
|
|
func (a *App) handleProposal(w http.ResponseWriter, r *http.Request) {
|
|
var req ProposalRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeErr(w, 400, err)
|
|
return
|
|
}
|
|
if catalogEntry(req.ActionID) == nil {
|
|
writeErr(w, 400, errors.New("action_id not in the catalogue"))
|
|
return
|
|
}
|
|
if cur := a.runs.current(); cur != nil && req.RunID == nil {
|
|
req.RunID = &cur.RunID
|
|
if cur.ScenarioID != "" {
|
|
req.ScenarioID = &cur.ScenarioID
|
|
}
|
|
}
|
|
ttl := 15 * time.Minute // design 2.1: no approval within 15 minutes -> layer 3
|
|
if req.TTLSeconds > 0 {
|
|
ttl = time.Duration(req.TTLSeconds) * time.Second
|
|
}
|
|
b := make([]byte, 24)
|
|
_, _ = rand.Read(b)
|
|
token := "apr-" + hex.EncodeToString(b)
|
|
if err := a.store.InsertProposal(r.Context(), token, req.RunID, req.ScenarioID, req.ActionID, req.Target, ttl); err != nil {
|
|
writeErr(w, 500, err)
|
|
return
|
|
}
|
|
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "proposal", "ai", map[string]any{"action_id": req.ActionID, "target": req.Target, "diagnosis": req.Diagnosis, "token_suffix": token[len(token)-6:]})
|
|
if req.Approved {
|
|
by := req.ApprovedBy
|
|
if by == "" {
|
|
by = "unspecified"
|
|
}
|
|
_ = a.store.ApproveProposal(r.Context(), token, by)
|
|
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]})
|
|
}
|
|
writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved})
|
|
}
|
|
|
|
// GET /proposals/{token}: the HITL bot reads the decision state (approval may arrive through this API instead of the
|
|
// Discord button; the bot mirrors it into its message instead of letting the message expire, campaign 3/4 2026-10-08).
|
|
func (a *App) handleProposalGet(w http.ResponseWriter, r *http.Request) {
|
|
p, err := a.store.GetProposal(r.Context(), r.PathValue("token"))
|
|
if err != nil {
|
|
writeErr(w, 404, errors.New("unknown proposal"))
|
|
return
|
|
}
|
|
writeJSON(w, 200, map[string]any{"token": p.Token, "run_id": p.RunID, "scenario_id": p.ScenarioID, "action_id": p.ActionID,
|
|
"approved": p.ApprovedAt != nil, "approved_at": p.ApprovedAt, "approved_by": p.ApprovedBy, "executed": p.ExecutedAt != nil, "executed_at": p.ExecutedAt, "expires_at": p.ExpiresAt})
|
|
}
|
|
|
|
// POST /proposals/{token}/approve: the HITL bot records the human decision (design 6.5 step 4). The proposal was
|
|
// registered at diagnosis time, so MTTA = approval - proposal (design 5.4) measures the real waiting time.
|
|
func (a *App) handleProposalApprove(w http.ResponseWriter, r *http.Request) {
|
|
token := r.PathValue("token")
|
|
var req struct {
|
|
ApprovedBy string `json:"approved_by"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
|
if req.ApprovedBy == "" {
|
|
req.ApprovedBy = "unspecified"
|
|
}
|
|
p, err := a.store.GetProposal(r.Context(), token)
|
|
if err != nil {
|
|
writeErr(w, 404, errors.New("unknown proposal"))
|
|
return
|
|
}
|
|
switch {
|
|
case p.ApprovedAt != nil:
|
|
writeErr(w, 409, errors.New("proposal already approved"))
|
|
return
|
|
case time.Now().After(p.ExpiresAt):
|
|
writeErr(w, 410, errors.New("proposal expired"))
|
|
return
|
|
}
|
|
if err := a.store.ApproveProposal(r.Context(), token, req.ApprovedBy); err != nil {
|
|
writeErr(w, 500, err)
|
|
return
|
|
}
|
|
a.store.Event(r.Context(), p.RunID, p.ScenarioID, "approval", req.ApprovedBy, map[string]any{"action_id": p.ActionID, "token_suffix": token[len(token)-6:]})
|
|
writeJSON(w, 200, map[string]any{"token": token, "approved": true, "approved_by": req.ApprovedBy})
|
|
}
|
|
|
|
// POST /proposals/{token}/reject: recorded as a note event (kind list of design 5.3); the token stays unapproved.
|
|
func (a *App) handleProposalReject(w http.ResponseWriter, r *http.Request) {
|
|
token := r.PathValue("token")
|
|
var req struct {
|
|
RejectedBy string `json:"rejected_by"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
|
p, err := a.store.GetProposal(r.Context(), token)
|
|
if err != nil {
|
|
writeErr(w, 404, errors.New("unknown proposal"))
|
|
return
|
|
}
|
|
a.store.Event(r.Context(), p.RunID, p.ScenarioID, "note", req.RejectedBy, map[string]any{"rejected": true, "action_id": p.ActionID, "reason": req.Reason, "token_suffix": token[len(token)-6:]})
|
|
writeJSON(w, 200, map[string]any{"token": token, "rejected": true})
|
|
}
|