package main import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" "log" "net/http" "regexp" "strings" "sync" "time" ) // RunManager executes one run at a time: for each scenario baseline -> inject -> observe (judge every 30s, // alerts/diagnoses/actions arrive through the HTTP handlers) -> revert -> verdict -> cooldown (design 5.2, 8). type RunManager struct { app *App mu sync.Mutex active *activeRun } type activeRun struct { RunID string Campaign string Model string // LLM for this run's diagnoses ("" = AI VM default); passed to the AI VM on every /diagnose AutoApproveAfter time.Duration // >0: measurement mode, the harness approves and executes layer-2 proposals after this delay ScenarioID string // "" between scenarios Scenario *Scenario TInject time.Time cancel context.CancelFunc } type RunRequest struct { Campaign string `json:"campaign"` Model string `json:"model"` // optional: LLM id for the diagnoses (e.g. gpt-6-luna, claude-sonnet-5-5); "" = AI VM default ScenarioIDs []string `json:"scenario_ids"` BaselineS *int `json:"baseline_s"` // default 300 (design 8) CooldownS *int `json:"cooldown_s"` // default 300 ObserveTimeoutS *int `json:"observe_timeout_s"` // default 1800 MinObserveS *int `json:"min_observe_s"` // default 180: earliest t_recover (12 probe successes after inject) DriftBefore *int `json:"drift_before"` GitSHA string `json:"git_sha"` Notes string `json:"notes"` SkipRevert bool `json:"skip_revert"` AutoApproveAfterS *int `json:"auto_approve_after_s"` // measurement mode (model comparison without a human in the loop): nil/0 = off } // modelRe bounds the model id (empty = AI VM default). The AI VM routes by prefix: claude-* -> Anthropic, else OpenAI. var modelRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{0,64}$`) func NewRunManager(a *App) *RunManager { return &RunManager{app: a} } func (m *RunManager) current() *activeRun { m.mu.Lock() defer m.mu.Unlock() if m.active == nil { return nil } c := *m.active return &c } func newID(prefix string) string { b := make([]byte, 4) _, _ = rand.Read(b) return prefix + time.Now().UTC().Format("20060102-150405") + "-" + hex.EncodeToString(b) } func (a *App) handleRunCreate(w http.ResponseWriter, r *http.Request) { var req RunRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeErr(w, 400, err) return } if req.Campaign == "" || len(req.ScenarioIDs) == 0 { writeErr(w, 400, errors.New("campaign and scenario_ids are required")) return } for _, id := range req.ScenarioIDs { if _, ok := a.scen[id]; !ok { writeErr(w, 400, fmt.Errorf("unknown scenario %q", id)) return } } if !modelRe.MatchString(req.Model) { writeErr(w, 400, errors.New("model must match [A-Za-z0-9._:-]{1,64}")) return } m := a.runs m.mu.Lock() if m.active != nil { id := m.active.RunID m.mu.Unlock() writeJSON(w, 409, map[string]string{"error": "a run is active", "run_id": id}) return } runID := newID("run-") ctx, cancel := context.WithCancel(context.Background()) var autoAfter time.Duration if req.AutoApproveAfterS != nil && *req.AutoApproveAfterS > 0 { autoAfter = time.Duration(*req.AutoApproveAfterS) * time.Second } m.active = &activeRun{RunID: runID, Campaign: req.Campaign, Model: req.Model, AutoApproveAfter: autoAfter, cancel: cancel} m.mu.Unlock() if err := a.store.CreateRun(r.Context(), runID, req.Campaign, req.DriftBefore, req.GitSHA, req.Notes, req.Model, req.AutoApproveAfterS); err != nil { m.mu.Lock() m.active = nil m.mu.Unlock() cancel() writeErr(w, 500, err) return } mRuns.Set(1) go m.execute(ctx, runID, req) writeJSON(w, 202, map[string]any{"run_id": runID, "scenario_ids": req.ScenarioIDs, "model": req.Model, "auto_approve_after_s": req.AutoApproveAfterS}) } func (a *App) handleRunAbort(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") m := a.runs m.mu.Lock() if m.active == nil || m.active.RunID != id { m.mu.Unlock() writeErr(w, 404, errors.New("run not active")) return } m.active.cancel() m.mu.Unlock() writeJSON(w, 202, map[string]string{"run_id": id, "status": "aborting"}) } func (a *App) handleRunFinish(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") var body struct { DriftAfter *int `json:"drift_after"` } _ = json.NewDecoder(r.Body).Decode(&body) if err := a.store.FinishRun(r.Context(), id, "finished", body.DriftAfter); err != nil { writeErr(w, 500, err) return } writeJSON(w, 200, map[string]any{"run_id": id, "drift_after": body.DriftAfter}) } func (a *App) handleRunList(w http.ResponseWriter, r *http.Request) { runs, err := a.store.Runs(r.Context(), "") if err != nil { writeErr(w, 500, err) return } writeJSON(w, 200, map[string]any{"active": a.runs.current(), "runs": runs}) } func (a *App) handleRunGet(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") runs, err := a.store.Runs(r.Context(), id) if err != nil || len(runs) == 0 { writeErr(w, 404, errors.New("run not found")) return } inj, _ := a.store.Injections(r.Context(), id) ev, _ := a.store.Events(r.Context(), id, "") vd, _ := a.store.Verdicts(r.Context(), id) writeJSON(w, 200, map[string]any{"run": runs[0], "injections": inj, "events": ev, "verdicts": vd}) } func dur(p *int, def time.Duration) time.Duration { if p == nil { return def } return time.Duration(*p) * time.Second } func (m *RunManager) execute(ctx context.Context, runID string, req RunRequest) { a := m.app status := "finished" defer func() { m.mu.Lock() m.active = nil m.mu.Unlock() mRuns.Set(0) _ = a.store.FinishRun(context.Background(), runID, status, nil) log.Printf("run %s %s", runID, status) }() baseline, cooldown := dur(req.BaselineS, 5*time.Minute), dur(req.CooldownS, 5*time.Minute) observeMax, minObserve := dur(req.ObserveTimeoutS, a.cfg.ObserveMax), dur(req.MinObserveS, 3*time.Minute) for i, sid := range req.ScenarioIDs { sc := a.scen[sid] if err := m.runScenario(ctx, runID, sc, baseline, observeMax, minObserve, req.SkipRevert); err != nil { if errors.Is(err, context.Canceled) { status = "aborted" return } log.Printf("run %s scenario %s: %v", runID, sid, err) } if i < len(req.ScenarioIDs)-1 && !sleepCtx(ctx, cooldown) { status = "aborted" return } } } func sleepCtx(ctx context.Context, d time.Duration) bool { if d <= 0 { return true } select { case <-time.After(d): return true case <-ctx.Done(): return false } } func (m *RunManager) setScenario(sc *Scenario, tInject time.Time) { m.mu.Lock() defer m.mu.Unlock() if m.active == nil { return } if sc == nil { m.active.ScenarioID, m.active.Scenario, m.active.TInject = "", nil, time.Time{} return } m.active.ScenarioID, m.active.Scenario, m.active.TInject = sc.ID, sc, tInject } func (m *RunManager) runScenario(ctx context.Context, runID string, sc *Scenario, baseline, observeMax, minObserve time.Duration, skipRevert bool) error { a := m.app sid := sc.ID bg := context.Background() a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "baseline", "seconds": baseline.Seconds()}) if !sleepCtx(ctx, baseline) { return ctx.Err() } // inject m.setScenario(sc, time.Now().UTC()) defer m.setScenario(nil, time.Time{}) injCtx, cancelInj := context.WithTimeout(ctx, 5*time.Minute) params, err := a.runStep(injCtx, sc, sc.Inject, runID, nil) cancelInj() tInject := time.Now().UTC() injID, dbErr := a.store.InsertInjection(bg, runID, sid, sc.Inject.Method, params) if dbErr != nil { log.Printf("injection insert: %v", dbErr) } if err != nil { a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "inject", "error": err.Error()}) _ = a.store.InsertVerdict(bg, Verdict{RunID: runID, ScenarioID: sid, Recovered: false, Notes: "injection failed: " + err.Error()}) return err } m.setScenario(sc, tInject) a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "inject", "t_inject": tInject, "params": params}) // observe: judge every 30s until recovered (after minObserve) or timeout var tRecover *time.Time var last SLOState deadline := tInject.Add(observeMax) for { if !sleepCtx(ctx, a.cfg.JudgeEvery) { break } last = a.evaluateSLO(ctx) if last.Recovered && time.Since(tInject) >= minObserve { t := time.Now().UTC() tRecover = &t a.store.Event(bg, &runID, &sid, "recovered", "judge", last) break } if time.Now().After(deadline) { a.store.Event(bg, &runID, &sid, "timeout", "judge", last) break } } aborted := ctx.Err() != nil // revert (always, even on abort, unless the scenario is irreversible by design or skip_revert) if !skipRevert && sc.Revert.Method != "none" { rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) rres, rerr := a.runStep(rvCtx, sc, sc.Revert, runID, params) cancelRv() if rerr != nil { a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres}) } else { a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "result": rres}) } } if injID != 0 { _ = a.store.ClearInjection(bg, injID) } // verdict v := a.buildVerdict(bg, runID, sc, tInject, tRecover, last) if aborted { v.Notes += " aborted;" } if err := a.store.InsertVerdict(bg, v); err != nil { log.Printf("verdict insert: %v", err) } if aborted { return ctx.Err() } return nil } // buildVerdict derives design 5.4 metrics from the events of this scenario. func (a *App) buildVerdict(ctx context.Context, runID string, sc *Scenario, tInject time.Time, tRecover *time.Time, last SLOState) Verdict { v := Verdict{RunID: runID, ScenarioID: sc.ID, Recovered: tRecover != nil} if tRecover != nil { s := tRecover.Sub(tInject).Seconds() v.MTTRs = &s } evs, _ := a.store.Events(ctx, runID, sc.ID) var tAlert, tProposal, tApproval *time.Time layer := 1 misjudge := 0 var diagScore *float64 var actionMatch *bool for _, e := range evs { var p map[string]any _ = json.Unmarshal(e.Payload, &p) switch e.Kind { case "alert": if tAlert == nil && e.TS.After(tInject) && p["status"] == "firing" { t := e.TS tAlert = &t } case "diagnosis": if e, _ := p["error"].(string); e != "" { // AI VM unreachable / malformed: not a diagnosis, not a misjudge continue } if layer < 2 { layer = 2 } label, _ := p["root_cause_label"].(string) s := scoreDiagnosis(label, sc.GroundTruthLabel) diagScore = &s if s == 0 { misjudge++ } if pa, ok := p["proposed_action"].(string); ok { b := pa == sc.GroundTruthAction actionMatch = &b if sc.GroundTruthAction == "escalate" && pa != "escalate" { misjudge++ } } case "proposal": if tProposal == nil { t := e.TS tProposal = &t } case "approval": if tApproval == nil { t := e.TS tApproval = &t } case "action": switch e.Actor { case "human": layer = 3 case "ai-approved": if layer < 2 { layer = 2 } } case "misjudge": misjudge++ } } if tAlert != nil { s := tAlert.Sub(tInject).Seconds() v.MTTDs = &s } if tProposal != nil && tApproval != nil { s := tApproval.Sub(*tProposal).Seconds() v.MTTAs = &s } v.LayerActual = &layer v.DiagnosisScore = diagScore v.ActionMatch = actionMatch v.MisjudgeCount = misjudge b, _ := json.Marshal(last) v.Notes = "last_slo=" + string(b) return v } // design 9: exact label 1.0, same top category (network/compute/db/config) 0.5, else 0 func scoreDiagnosis(got, want string) float64 { if got == "" || got == "unknown" { return 0 } if got == want { return 1 } if category(got) != "" && category(got) == category(want) { return 0.5 } return 0 } func category(label string) string { switch { case contains(label, "network", "latency", "partition", "loss"): return "network" case contains(label, "pod", "process", "instance", "memory", "cpu"): return "compute" case contains(label, "db", "replication", "schema", "primary"): return "db" case contains(label, "drift", "config", "gitops"): return "config" } return "" } func contains(s string, subs ...string) bool { for _, x := range subs { if strings.Contains(s, x) { return true } } return false }