From ca933a646e9087745283df1a64719e5f37ccf68d Mon Sep 17 00:00:00 2001 From: seonNoh Date: Thu, 15 Oct 2026 23:39:39 +0900 Subject: [PATCH] feat(harness): per-run auto-approval of layer-2 proposals for model measurement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 런 옵션 auto_approve_after_s(runs 열·POST /runs)를 추가한다. 값이 양수면 하네스가 제안 등록 뒤 그 지연만큼 기다렸다가 스스로 승인(actor auto)하고 HITL 버튼 경로와 같은 방식으로 조치를 실행(actor ai-approved, MarkProposalExecuted)한다. 사람 없이 모델 비교 캠페인을 반복하기 위한 측정 모드로, MTTA는 지연값으로 고정되고 판정 로직은 그대로다(2층 판정은 ai-approved 조치에서 나온다). 이미 결정·만료된 제안과 기록 전용 조치는 API 경로와 같이 처리한다. --- harness/autoapprove.go | 45 +++++++++++++++++++++++++++++++++++++++ harness/proposals.go | 3 +++ harness/run.go | 48 ++++++++++++++++++++++++------------------ harness/store.go | 12 ++++++----- 4 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 harness/autoapprove.go diff --git a/harness/autoapprove.go b/harness/autoapprove.go new file mode 100644 index 0000000..9ca84ac --- /dev/null +++ b/harness/autoapprove.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "time" +) + +// autoApprove is the measurement-mode stand-in for the Discord approver (run option auto_approve_after_s, 2026-10-15): +// after the configured delay the harness approves the layer-2 proposal (actor "auto") and executes it exactly as the +// HITL bot does after a button press (actor "ai-approved", MarkProposalExecuted). MTTA therefore equals the delay; +// the verdict logic is unchanged (layer 2 from the ai-approved action). A proposal already decided, expired or +// recorded-only is handled the same way as the API path. +func (a *App) autoApprove(token string, req ProposalRequest, after time.Duration) { + time.Sleep(after) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + p, err := a.store.GetProposal(ctx, token) + if err != nil || p.ApprovedAt != nil || time.Now().After(p.ExpiresAt) { + return + } + if err := a.store.ApproveProposal(ctx, token, "auto"); err != nil { + return + } + suffix := token[len(token)-6:] + a.store.Event(ctx, p.RunID, p.ScenarioID, "approval", "auto", map[string]any{"action_id": p.ActionID, "token_suffix": suffix, "auto_approve_after_s": after.Seconds()}) + ce := catalogEntry(p.ActionID) + if ce == nil { + return + } + note := "auto-approved by the harness (measurement mode)" + if !ce.Exec { + a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "recorded_only": true, "note": note, "layer": ce.Layer}) + mActs.WithLabelValues(p.ActionID, "ai-approved", "recorded").Inc() + _ = a.store.MarkProposalExecuted(ctx, token) + return + } + res, err := a.executeAction(ctx, p.ActionID, req.Target, "ai-approved", p.RunID, p.ScenarioID) + result := "ok" + if err != nil { + result = "error" + } + mActs.WithLabelValues(p.ActionID, "ai-approved", result).Inc() + a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "result": res, "error": errStr(err), "note": note}) + _ = a.store.MarkProposalExecuted(ctx, token) +} diff --git a/harness/proposals.go b/harness/proposals.go index 16ae96a..4810272 100644 --- a/harness/proposals.go +++ b/harness/proposals.go @@ -59,6 +59,9 @@ func (a *App) handleProposal(w http.ResponseWriter, r *http.Request) { _ = a.store.ApproveProposal(r.Context(), token, by) a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]}) } + if cur := a.runs.current(); !req.Approved && cur != nil && cur.AutoApproveAfter > 0 && req.RunID != nil && *req.RunID == cur.RunID { + go a.autoApprove(token, req, cur.AutoApproveAfter) // measurement mode (run option auto_approve_after_s) + } writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved}) } diff --git a/harness/run.go b/harness/run.go index 5e863ac..d9a8e9e 100644 --- a/harness/run.go +++ b/harness/run.go @@ -24,27 +24,29 @@ type RunManager struct { } type activeRun struct { - RunID string - Campaign string - Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose - ScenarioID string // "" between scenarios - Scenario *Scenario - TInject time.Time - cancel context.CancelFunc + RunID string + Campaign string + Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose + AutoApproveAfter time.Duration // >0: measurement mode, the harness approves and executes layer-2 proposals after this delay + ScenarioID string // "" between scenarios + Scenario *Scenario + TInject time.Time + cancel context.CancelFunc } type RunRequest struct { - Campaign string `json:"campaign"` - Model string `json:"model"` // optional: LLM id for the diagnoses (e.g. gpt-6-luna, claude-sonnet-5-5); "" = AI VM default - ScenarioIDs []string `json:"scenario_ids"` - BaselineS *int `json:"baseline_s"` // default 300 (design 8) - CooldownS *int `json:"cooldown_s"` // default 300 - ObserveTimeoutS *int `json:"observe_timeout_s"` // default 1800 - MinObserveS *int `json:"min_observe_s"` // default 180: earliest t_recover (12 probe successes after inject) - DriftBefore *int `json:"drift_before"` - GitSHA string `json:"git_sha"` - Notes string `json:"notes"` - SkipRevert bool `json:"skip_revert"` + Campaign string `json:"campaign"` + Model string `json:"model"` // optional: LLM id for the diagnoses (e.g. gpt-6-luna, claude-sonnet-5-5); "" = AI VM default + ScenarioIDs []string `json:"scenario_ids"` + BaselineS *int `json:"baseline_s"` // default 300 (design 8) + CooldownS *int `json:"cooldown_s"` // default 300 + ObserveTimeoutS *int `json:"observe_timeout_s"` // default 1800 + MinObserveS *int `json:"min_observe_s"` // default 180: earliest t_recover (12 probe successes after inject) + DriftBefore *int `json:"drift_before"` + GitSHA string `json:"git_sha"` + Notes string `json:"notes"` + SkipRevert bool `json:"skip_revert"` + AutoApproveAfterS *int `json:"auto_approve_after_s"` // measurement mode (model comparison without a human in the loop): nil/0 = off } // modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI. @@ -98,9 +100,13 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) { } runID := newID("run-") ctx, cancel := context.WithCancel(context.Background()) - m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, cancel: cancel} + var autoAfter time.Duration + if req.AutoApproveAfterS != nil && *req.AutoApproveAfterS > 0 { + autoAfter = time.Duration(*req.AutoApproveAfterS) * time.Second + } + m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, AutoApproveAfter: autoAfter, cancel: cancel} m.mu.Unlock() - if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model); err != nil { + if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model, req.AutoApproveAfterS); err != nil { m.mu.Lock() m.active = nil m.mu.Unlock() @@ -110,7 +116,7 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) { } mRuns.Set(1) go m.execute(ctx, runID, req) - writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model}) + writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model, "auto_approve_after_s": req.AutoApproveAfterS}) } func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) { diff --git a/harness/store.go b/harness/store.go index fede06b..f9e230d 100644 --- a/harness/store.go +++ b/harness/store.go @@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS runs ( notes text ); ALTER TABLE runs ADD COLUMN IF NOT EXISTS model text; -- LLM requested for the run's diagnoses (campaign-level model selection, 2026-10-14) +ALTER TABLE runs ADD COLUMN IF NOT EXISTS auto_approve_after_s int; -- measurement mode: the harness approves + executes layer-2 proposals after this delay (2026-10-15) CREATE TABLE IF NOT EXISTS scenarios ( scenario_id text PRIMARY KEY, path text NOT NULL, @@ -120,8 +121,8 @@ func (s *Store) UpsertScenarios(ctx context.Context, m map[string]*Scenario) err return nil } -func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string) error { - _, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model) VALUES ($1,$2,$3,$4,$5,$6)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model)) +func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string, autoApproveAfterS *int) error { + _, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model,auto_approve_after_s) VALUES ($1,$2,$3,$4,$5,$6,$7)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model), autoApproveAfterS) return err } @@ -223,11 +224,12 @@ type RunRow struct { GitSHA *string `json:"git_sha"` Status string `json:"status"` Notes *string `json:"notes"` - Model *string `json:"model"` // nil for runs before campaign-level model selection + Model *string `json:"model"` // nil for runs before campaign-level model selection + AutoApprove *int `json:"auto_approve_after_s"` // nil/0 = human approval (Discord); >0 = harness auto-approved layer-2 proposals after this delay } func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) { - rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID) + rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model,auto_approve_after_s FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID) if err != nil { return nil, err } @@ -235,7 +237,7 @@ func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) { var out []RunRow for rows.Next() { var r RunRow - if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model); err != nil { + if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model, &r.AutoApprove); err != nil { return nil, err } out = append(out, r)