feat: per-run model selection, usage in diagnosis results and a Claude Messages API provider
캠페인마다 진단 모델을 고를 수 있게 한다. 하네스는 runs.model 열을 추가해 POST /runs의 model을 저장하고, 활성 런의 model을 /diagnose 요청에 실어 보낸다. 에이전트는 요청의 model을 우선 사용하고(없으면 OPENAI_MODEL 기본값), 모델 id가 claude-로 시작하면 internal/anthropic(Claude Messages API 어댑터)로 라우팅한다. 어댑터는 Responses API 항목(message/function_call/function_call_output)을 system·tools(input_schema)·tool_use/ tool_result로 변환하고 previous_response_id 체인을 클라이언트 쪽에서 재현하며, Claude 5.5가 강제 tool_choice를 거부하므로 강제 호출은 auto + 지시문으로 바꾼다. 키는 ANTHROPIC_KEY_FILE(/run/secrets/anthropic-api-key)에서 읽고 비어 있으면 명확한 오류를 낸다. Result에 usage(입력·출력·합계 토큰)를 추가해 하네스 diagnosis 이벤트에 비용 근거를 남긴다.
This commit is contained in:
seon committed
1 parent
fa9c08c778
commit
53ff243183
8 files changed
+492
-12
No files matched your search
@@ -231,6 +231,9 @@ func (r *AlertRouter) diagnose(alert map[string]any, runID, sid *string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
reqBody := map[string]any{"alert": alert, "run_id": runID, "scenario_id": sid}
|
||||
if cur := r.app.runs.current(); cur != nil && runID != nil && cur.RunID == *runID && cur.Model != "" {
|
||||
reqBody["model"] = cur.Model // campaign-level model selection (the AI VM falls back to its default when absent)
|
||||
}
|
||||
b, _ := json.Marshal(reqBody)
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", r.app.cfg.AIDiagnoseURL, bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
+13
-3
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -25,6 +26,7 @@ type RunManager struct {
|
||||
type activeRun struct {
|
||||
RunID string
|
||||
Campaign string
|
||||
Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose
|
||||
ScenarioID string // "" between scenarios
|
||||
Scenario *Scenario
|
||||
TInject time.Time
|
||||
@@ -33,6 +35,7 @@ type activeRun struct {
|
||||
|
||||
type RunRequest struct {
|
||||
Campaign string `json:"campaign"`
|
||||
Model string `json:"model"` // optional: LLM id for the diagnoses (e.g. gpt-6-luna, claude-sonnet-5-5); "" = AI VM default
|
||||
ScenarioIDs []string `json:"scenario_ids"`
|
||||
BaselineS *int `json:"baseline_s"` // default 300 (design 8)
|
||||
CooldownS *int `json:"cooldown_s"` // default 300
|
||||
@@ -44,6 +47,9 @@ type RunRequest struct {
|
||||
SkipRevert bool `json:"skip_revert"`
|
||||
}
|
||||
|
||||
// modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI.
|
||||
var modelRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{0,64}$`)
|
||||
|
||||
func NewRunManager(a *App) *RunManager { return &RunManager{app: a} }
|
||||
|
||||
func (m *RunManager) current() *activeRun {
|
||||
@@ -78,6 +84,10 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if !modelRe.MatchString(req.Model) {
|
||||
writeErr(w, 400, errors.New("model must match [A-Za-z0-9._:-]{1,64}"))
|
||||
return
|
||||
}
|
||||
m := a.runs
|
||||
m.mu.Lock()
|
||||
if m.active != nil {
|
||||
@@ -88,9 +98,9 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
runID := newID("run-")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, cancel: cancel}
|
||||
m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, cancel: cancel}
|
||||
m.mu.Unlock()
|
||||
if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes); err != nil {
|
||||
if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model); err != nil {
|
||||
m.mu.Lock()
|
||||
m.active = nil
|
||||
m.mu.Unlock()
|
||||
@@ -100,7 +110,7 @@ func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
mRuns.Set(1)
|
||||
go m.execute(ctx, runID, req)
|
||||
writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs})
|
||||
writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model})
|
||||
}
|
||||
|
||||
func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
+6
-4
@@ -24,6 +24,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
||||
status text NOT NULL DEFAULT 'running',
|
||||
notes text
|
||||
);
|
||||
ALTER TABLE runs ADD COLUMN IF NOT EXISTS model text; -- LLM requested for the run's diagnoses (campaign-level model selection, 2026-10-14)
|
||||
CREATE TABLE IF NOT EXISTS scenarios (
|
||||
scenario_id text PRIMARY KEY,
|
||||
path text NOT NULL,
|
||||
@@ -119,8 +120,8 @@ func (s *Store) UpsertScenarios(ctx context.Context, m map[string]*Scenario) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes) VALUES ($1,$2,$3,$4,$5)`, id, campaign, driftBefore, nz(gitSHA), nz(notes))
|
||||
func (s *Store) CreateRun(ctx context.Context, id, campaign string, driftBefore *int, gitSHA, notes, model string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO runs (run_id,campaign,drift_before,git_sha,notes,model) VALUES ($1,$2,$3,$4,$5,$6)`, id, campaign, driftBefore, nz(gitSHA), nz(notes), nz(model))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -222,10 +223,11 @@ type RunRow struct {
|
||||
GitSHA *string `json:"git_sha"`
|
||||
Status string `json:"status"`
|
||||
Notes *string `json:"notes"`
|
||||
Model *string `json:"model"` // nil for runs before campaign-level model selection
|
||||
}
|
||||
|
||||
func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID)
|
||||
rows, err := s.pool.Query(ctx, `SELECT run_id,campaign,started_at,finished_at,drift_before,drift_after,git_sha,status,notes,model FROM runs WHERE ($1='' OR run_id=$1) ORDER BY started_at DESC LIMIT 100`, runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -233,7 +235,7 @@ func (s *Store) Runs(ctx context.Context, runID string) ([]RunRow, error) {
|
||||
var out []RunRow
|
||||
for rows.Next() {
|
||||
var r RunRow
|
||||
if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes); err != nil {
|
||||
if err := rows.Scan(&r.RunID, &r.Campaign, &r.StartedAt, &r.FinishedAt, &r.DriftBefore, &r.DriftAfter, &r.GitSHA, &r.Status, &r.Notes, &r.Model); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
|
||||
Reference in new issue
Block a user