feat(harness): per-run auto-approval of layer-2 proposals for model measurement

런 옵션 auto_approve_after_s(runs 열·POST /runs)를 추가한다. 값이 양수면 하네스가 제안
등록 뒤 그 지연만큼 기다렸다가 스스로 승인(actor auto)하고 HITL 버튼 경로와 같은 방식으로
조치를 실행(actor ai-approved, MarkProposalExecuted)한다. 사람 없이 모델 비교 캠페인을
반복하기 위한 측정 모드로, MTTA는 지연값으로 고정되고 판정 로직은 그대로다(2층 판정은
ai-approved 조치에서 나온다). 이미 결정·만료된 제안과 기록 전용 조치는 API 경로와 같이
처리한다.
This commit is contained in:
seon committed 2026-10-15 23:39:39 +09:00
1 parent f671e2419f
commit ca933a646e
4 files changed
+63 -7

No files matched your search

+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"time"
)
// autoApprove is the measurement-mode stand-in for the Discord approver (run option auto_approve_after_s, 2026-10-15):
// after the configured delay the harness approves the layer-2 proposal (actor "auto") and executes it exactly as the
// HITL bot does after a button press (actor "ai-approved", MarkProposalExecuted). MTTA therefore equals the delay;
// the verdict logic is unchanged (layer 2 from the ai-approved action). A proposal already decided, expired or
// recorded-only is handled the same way as the API path.
func (a *App) autoApprove(token string, req ProposalRequest, after time.Duration) {
time.Sleep(after)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
p, err := a.store.GetProposal(ctx, token)
if err != nil || p.ApprovedAt != nil || time.Now().After(p.ExpiresAt) {
return
}
if err := a.store.ApproveProposal(ctx, token, "auto"); err != nil {
return
}
suffix := token[len(token)-6:]
a.store.Event(ctx, p.RunID, p.ScenarioID, "approval", "auto", map[string]any{"action_id": p.ActionID, "token_suffix": suffix, "auto_approve_after_s": after.Seconds()})
ce := catalogEntry(p.ActionID)
if ce == nil {
return
}
note := "auto-approved by the harness (measurement mode)"
if !ce.Exec {
a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "recorded_only": true, "note": note, "layer": ce.Layer})
mActs.WithLabelValues(p.ActionID, "ai-approved", "recorded").Inc()
_ = a.store.MarkProposalExecuted(ctx, token)
return
}
res, err := a.executeAction(ctx, p.ActionID, req.Target, "ai-approved", p.RunID, p.ScenarioID)
result := "ok"
if err != nil {
result = "error"
}
mActs.WithLabelValues(p.ActionID, "ai-approved", result).Inc()
a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "result": res, "error": errStr(err), "note": note})
_ = a.store.MarkProposalExecuted(ctx, token)
}
+3
View File
@@ -59,6 +59,9 @@ func (a *App) handleProposal(w http.ResponseWriter, r *http.Request) {
_ = a.store.ApproveProposal(r.Context(), token, by) _ = a.store.ApproveProposal(r.Context(), token, by)
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]}) a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]})
} }
if cur := a.runs.current(); !req.Approved && cur != nil && cur.AutoApproveAfter > 0 && req.RunID != nil && *req.RunID == cur.RunID {
go a.autoApprove(token, req, cur.AutoApproveAfter) // measurement mode (run option auto_approve_after_s)
}
writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved}) writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved})
} }
+9 -3
View File
@@ -27,6 +27,7 @@ type activeRun struct {
RunID string RunID string
Campaign string Campaign string
Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose
AutoApproveAfter time.Duration // >0: measurement mode, the harness approves and executes layer-2 proposals after this delay
ScenarioID string // "" between scenarios ScenarioID string // "" between scenarios
Scenario *Scenario Scenario *Scenario
TInject time.Time TInject time.Time
@@ -45,6 +46,7 @@ type RunRequest struct {
GitSHA string `json:"git_sha"` GitSHA string `json:"git_sha"`
Notes string `json:"notes"` Notes string `json:"notes"`
SkipRevert bool `json:"skip_revert"` SkipRevert bool `json:"skip_revert"`
AutoApproveAfterS *int `json:"auto_approve_after_s"` // measurement mode (model comparison without a human in the loop): nil/0 = off
} }
// modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI. // modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI.
@@ -98,9 +100,13 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
} }
runID := newID("run-") runID := newID("run-")
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, cancel: cancel} var autoAfter time.Duration
if req.AutoApproveAfterS != nil && *req.AutoApproveAfterS > 0 {
autoAfter = time.Duration(*req.AutoApproveAfterS) * time.Second
}
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, AutoApproveAfter: autoAfter, cancel: cancel}
m.mu.Unlock() m.mu.Unlock()
if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model); err != nil { if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model, req.AutoApproveAfterS); err != nil {
m.mu.Lock() m.mu.Lock()
m.active = nil m.active = nil
m.mu.Unlock() m.mu.Unlock()
@@ -110,7 +116,7 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
} }
mRuns.Set(1) mRuns.Set(1)
go m.execute(ctx, runID, req) go m.execute(ctx, runID, req)
writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model}) writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model, "auto_approve_after_s": req.AutoApproveAfterS})
} }
func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) { func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) {
+6 -4
View File
@@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS runs (
notes text notes text
); );
ALTER TABLE runs ADD COLUMN IF NOT EXISTS model text; -- LLM requested for the run's diagnoses (campaign-level model selection, 2026-10-14) ALTER TABLE runs ADD COLUMN IF NOT EXISTS model text; -- LLM requested for the run's diagnoses (campaign-level model selection, 2026-10-14)
ALTER TABLE runs ADD COLUMN IF NOT EXISTS auto_approve_after_s int; -- measurement mode: the harness approves + executes layer-2 proposals after this delay (2026-10-15)
CREATE TABLE IF NOT EXISTS scenarios ( CREATE TABLE IF NOT EXISTS scenarios (
scenario_id text PRIMARY KEY, scenario_id text PRIMARY KEY,
path text NOT NULL, path text NOT NULL,
@@ -120,8 +121,8 @@ func (s *Store) UpsertScenarios(ctx context.Context, m map[string]*Scenario) err
return nil return nil
} }
func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string) error { func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string, autoApproveAfterS *int) error {
_, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model) VALUES ($1,$2,$3,$4,$5,$6)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model)) _, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model,auto_approve_after_s) VALUES ($1,$2,$3,$4,$5,$6,$7)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model), autoApproveAfterS)
return err return err
} }
@@ -224,10 +225,11 @@ type RunRow struct {
Status string `json:"status"` Status string `json:"status"`
Notes *string `json:"notes"` Notes *string `json:"notes"`
Model *string `json:"model"` // nil for runs before campaign-level model selection Model *string `json:"model"` // nil for runs before campaign-level model selection
AutoApprove *int `json:"auto_approve_after_s"` // nil/0 = human approval (Discord); >0 = harness auto-approved layer-2 proposals after this delay
} }
func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) { func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID) rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model,auto_approve_after_s FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -235,7 +237,7 @@ func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
var out []RunRow var out []RunRow
for rows.Next() { for rows.Next() {
var r RunRow var r RunRow
if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model); err != nil { if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model, &r.AutoApprove); err != nil {
return nil, err return nil, err
} }
out = append(out, r) out = append(out, r)