feat(harness): per-run auto-approval of layer-2 proposals for model measurement
런 옵션 auto_approve_after_s(runs 열·POST /runs)를 추가한다. 값이 양수면 하네스가 제안 등록 뒤 그 지연만큼 기다렸다가 스스로 승인(actor auto)하고 HITL 버튼 경로와 같은 방식으로 조치를 실행(actor ai-approved, MarkProposalExecuted)한다. 사람 없이 모델 비교 캠페인을 반복하기 위한 측정 모드로, MTTA는 지연값으로 고정되고 판정 로직은 그대로다(2층 판정은 ai-approved 조치에서 나온다). 이미 결정·만료된 제안과 기록 전용 조치는 API 경로와 같이 처리한다.
This commit is contained in:
1 parent
f671e2419f
commit
ca933a646e
4 files changed
+63
-7
No files matched your search
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// autoApprove is the measurement-mode stand-in for the Discord approver (run option auto_approve_after_s, 2026-10-15):
|
||||
// after the configured delay the harness approves the layer-2 proposal (actor "auto") and executes it exactly as the
|
||||
// HITL bot does after a button press (actor "ai-approved", MarkProposalExecuted). MTTA therefore equals the delay;
|
||||
// the verdict logic is unchanged (layer 2 from the ai-approved action). A proposal already decided, expired or
|
||||
// recorded-only is handled the same way as the API path.
|
||||
func (a *App) autoApprove(token string, req ProposalRequest, after time.Duration) {
|
||||
time.Sleep(after)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
p, err := a.store.GetProposal(ctx, token)
|
||||
if err != nil || p.ApprovedAt != nil || time.Now().After(p.ExpiresAt) {
|
||||
return
|
||||
}
|
||||
if err := a.store.ApproveProposal(ctx, token, "auto"); err != nil {
|
||||
return
|
||||
}
|
||||
suffix := token[len(token)-6:]
|
||||
a.store.Event(ctx, p.RunID, p.ScenarioID, "approval", "auto", map[string]any{"action_id": p.ActionID, "token_suffix": suffix, "auto_approve_after_s": after.Seconds()})
|
||||
ce := catalogEntry(p.ActionID)
|
||||
if ce == nil {
|
||||
return
|
||||
}
|
||||
note := "auto-approved by the harness (measurement mode)"
|
||||
if !ce.Exec {
|
||||
a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "recorded_only": true, "note": note, "layer": ce.Layer})
|
||||
mActs.WithLabelValues(p.ActionID, "ai-approved", "recorded").Inc()
|
||||
_ = a.store.MarkProposalExecuted(ctx, token)
|
||||
return
|
||||
}
|
||||
res, err := a.executeAction(ctx, p.ActionID, req.Target, "ai-approved", p.RunID, p.ScenarioID)
|
||||
result := "ok"
|
||||
if err != nil {
|
||||
result = "error"
|
||||
}
|
||||
mActs.WithLabelValues(p.ActionID, "ai-approved", result).Inc()
|
||||
a.store.Event(ctx, p.RunID, p.ScenarioID, "action", "ai-approved", map[string]any{"action_id": p.ActionID, "target": req.Target, "result": res, "error": errStr(err), "note": note})
|
||||
_ = a.store.MarkProposalExecuted(ctx, token)
|
||||
}
|
||||
@@ -59,6 +59,9 @@ func (a *App) handleProposal(w http.ResponseWriter, r *http.Request) {
|
||||
_ = a.store.ApproveProposal(r.Context(), token, by)
|
||||
a.store.Event(r.Context(), req.RunID, req.ScenarioID, "approval", by, map[string]any{"action_id": req.ActionID, "token_suffix": token[len(token)-6:]})
|
||||
}
|
||||
if cur := a.runs.current(); !req.Approved && cur != nil && cur.AutoApproveAfter > 0 && req.RunID != nil && *req.RunID == cur.RunID {
|
||||
go a.autoApprove(token, req, cur.AutoApproveAfter) // measurement mode (run option auto_approve_after_s)
|
||||
}
|
||||
writeJSON(w, 201, map[string]any{"token": token, "expires_in_s": ttl.Seconds(), "approved": req.Approved})
|
||||
}
|
||||
|
||||
|
||||
+9
-3
@@ -27,6 +27,7 @@ type activeRun struct {
|
||||
RunID string
|
||||
Campaign string
|
||||
Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose
|
||||
AutoApproveAfter time.Duration // >0: measurement mode, the harness approves and executes layer-2 proposals after this delay
|
||||
ScenarioID string // "" between scenarios
|
||||
Scenario *Scenario
|
||||
TInject time.Time
|
||||
@@ -45,6 +46,7 @@ type RunRequest struct {
|
||||
GitSHA string `json:"git_sha"`
|
||||
Notes string `json:"notes"`
|
||||
SkipRevert bool `json:"skip_revert"`
|
||||
AutoApproveAfterS *int `json:"auto_approve_after_s"` // measurement mode (model comparison without a human in the loop): nil/0 = off
|
||||
}
|
||||
|
||||
// modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI.
|
||||
@@ -98,9 +100,13 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
runID := newID("run-")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, cancel: cancel}
|
||||
var autoAfter time.Duration
|
||||
if req.AutoApproveAfterS != nil && *req.AutoApproveAfterS > 0 {
|
||||
autoAfter = time.Duration(*req.AutoApproveAfterS) * time.Second
|
||||
}
|
||||
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, AutoApproveAfter: autoAfter, cancel: cancel}
|
||||
m.mu.Unlock()
|
||||
if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model); err != nil {
|
||||
if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model, req.AutoApproveAfterS); err != nil {
|
||||
m.mu.Lock()
|
||||
m.active = nil
|
||||
m.mu.Unlock()
|
||||
@@ -110,7 +116,7 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
mRuns.Set(1)
|
||||
go m.execute(ctx, runID, req)
|
||||
writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model})
|
||||
writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model, "auto_approve_after_s": req.AutoApproveAfterS})
|
||||
}
|
||||
|
||||
func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
+6
-4
@@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
||||
notes text
|
||||
);
|
||||
ALTER TABLE runs ADD COLUMN IF NOT EXISTS model text; -- LLM requested for the run's diagnoses (campaign-level model selection, 2026-10-14)
|
||||
ALTER TABLE runs ADD COLUMN IF NOT EXISTS auto_approve_after_s int; -- measurement mode: the harness approves + executes layer-2 proposals after this delay (2026-10-15)
|
||||
CREATE TABLE IF NOT EXISTS scenarios (
|
||||
scenario_id text PRIMARY KEY,
|
||||
path text NOT NULL,
|
||||
@@ -120,8 +121,8 @@ func (s *Store) UpsertScenarios(ctx context.Context, m map[string]*Scenario) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model) VALUES ($1,$2,$3,$4,$5,$6)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model))
|
||||
func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string, autoApproveAfterS *int) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model,auto_approve_after_s) VALUES ($1,$2,$3,$4,$5,$6,$7)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model), autoApproveAfterS)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -224,10 +225,11 @@ type RunRow struct {
|
||||
Status string `json:"status"`
|
||||
Notes *string `json:"notes"`
|
||||
Model *string `json:"model"` // nil for runs before campaign-level model selection
|
||||
AutoApprove *int `json:"auto_approve_after_s"` // nil/0 = human approval (Discord); >0 = harness auto-approved layer-2 proposals after this delay
|
||||
}
|
||||
|
||||
func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID)
|
||||
rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model,auto_approve_after_s FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -235,7 +237,7 @@ func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
|
||||
var out []RunRow
|
||||
for rows.Next() {
|
||||
var r RunRow
|
||||
if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model); err != nil {
|
||||
if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model, &r.AutoApprove); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
|
||||
Reference in new issue
Block a user