From de076a79e4b6966b8011ffd5308f7daf9401012e Mon Sep 17 00:00:00 2001 From: seonNoh Date: Thu, 24 Sep 2026 02:22:29 +0900 Subject: [PATCH] fix(harness): JSON-safe SLO state, unscored AI errors, mark recorded-only proposals executed --- harness/actions.go | 5 ++++- harness/alerts.go | 11 +++++++---- harness/judge.go | 42 +++++++++++++++++++++--------------------- harness/run.go | 23 +++++++++++++---------- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/harness/actions.go b/harness/actions.go index 734b4ef..0dc9738 100644 --- a/harness/actions.go +++ b/harness/actions.go @@ -14,7 +14,7 @@ import ( // human-only (layer 3) action that the harness records but does not perform. type CatalogEntry struct { ActionID string `json:"action_id"` - SLOImpact string `json:"slo_impact"` // small | large + SLOImpact string `json:"slo_impact"` // small | large Reversible bool `json:"reversible"` Layer string `json:"layer"` Exec bool `json:"harness_executes"` @@ -92,6 +92,9 @@ func (a *App) handleActionExecute(w http.ResponseWriter, r *http.Request) { if !ce.Exec { a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "recorded_only": true, "note": req.Note, "layer": ce.Layer}) mActs.WithLabelValues(req.ActionID, actor, "recorded").Inc() + if actor == "ai-approved" { + _ = a.store.MarkProposalExecuted(ctx, req.Token) + } writeJSON(w, 202, map[string]any{"action_id": req.ActionID, "recorded_only": true, "layer": ce.Layer, "note": "not executed by the harness (platform-internal or human-only); recorded in events"}) return } diff --git a/harness/alerts.go b/harness/alerts.go index 94f12fd..316f7f3 100644 --- a/harness/alerts.go +++ b/harness/alerts.go @@ -14,16 +14,19 @@ import ( ) // AlertRouter implements design 5.2 "알림 수신기": Grafana webhook -> t_alert, then -// signature present -> layer 1: the platform mechanism is already acting; the harness observes and, if the alert -// is still firing after escalate_after_s, hands it to layer 2 -// no signature -> layer 2: POST the alert to the AI VM /diagnose and record the diagnosis +// +// signature present -> layer 1: the platform mechanism is already acting; the harness observes and, if the alert +// is still firing after escalate_after_s, hands it to layer 2 +// no signature -> layer 2: POST the alert to the AI VM /diagnose and record the diagnosis type AlertRouter struct { app *App mu sync.Mutex timers map[string]*time.Timer // fingerprint -> escalation timer } -func NewAlertRouter(a *App) *AlertRouter { return &AlertRouter{app: a, timers: map[string]*time.Timer{}} } +func NewAlertRouter(a *App) *AlertRouter { + return &AlertRouter{app: a, timers: map[string]*time.Timer{}} +} // Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/) type grafanaWebhook struct { diff --git a/harness/judge.go b/harness/judge.go index 6d52e8e..bf8408f 100644 --- a/harness/judge.go +++ b/harness/judge.go @@ -46,17 +46,18 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) { } // SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m). +// Values are nil when the series is absent (JSON has no NaN). type SLOState struct { - At time.Time `json:"at"` - ProbeMin3m float64 `json:"probe_min_3m"` - P95 float64 `json:"p95_s"` - ErrorRatio float64 `json:"error_ratio"` - ProbeOK bool `json:"probe_ok"` - LatencyOK bool `json:"latency_ok"` - ErrorsOK bool `json:"errors_ok"` - Recovered bool `json:"recovered"` - MissingData []string `json:"missing_data,omitempty"` - QueryErrors []string `json:"query_errors,omitempty"` + At time.Time `json:"at"` + ProbeMin3m *float64 `json:"probe_min_3m"` + P95 *float64 `json:"p95_s"` + ErrorRatio *float64 `json:"error_ratio"` + ProbeOK bool `json:"probe_ok"` + LatencyOK bool `json:"latency_ok"` + ErrorsOK bool `json:"errors_ok"` + Recovered bool `json:"recovered"` + MissingData []string `json:"missing_data,omitempty"` + QueryErrors []string `json:"query_errors,omitempty"` } const ( @@ -67,25 +68,24 @@ const ( func (a *App) evaluateSLO(ctx context.Context) SLOState { st := SLOState{At: time.Now().UTC()} - get := func(name, expr string) (float64, bool) { + get := func(name, expr string) *float64 { v, err := a.mimirScalar(ctx, expr) if err != nil { st.QueryErrors = append(st.QueryErrors, name+": "+err.Error()) - return math.NaN(), false + return nil } if math.IsNaN(v) { st.MissingData = append(st.MissingData, name) - return v, false + return nil } - return v, true + return &v } - var ok bool - st.ProbeMin3m, ok = get("probe", qProbeMin3m) - st.ProbeOK = ok && st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery - st.P95, ok = get("p95", qP95) - st.LatencyOK = !ok || st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO - st.ErrorRatio, ok = get("errors", qErrRatio) - st.ErrorsOK = !ok || st.ErrorRatio < 0.01 + st.ProbeMin3m = get("probe", qProbeMin3m) + st.ProbeOK = st.ProbeMin3m != nil && *st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery + st.P95 = get("p95", qP95) + st.LatencyOK = st.P95 == nil || *st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO + st.ErrorRatio = get("errors", qErrRatio) + st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01 st.Recovered = st.ProbeOK && st.LatencyOK && st.ErrorsOK && len(st.QueryErrors) == 0 return st } diff --git a/harness/run.go b/harness/run.go index e563c61..184db99 100644 --- a/harness/run.go +++ b/harness/run.go @@ -32,16 +32,16 @@ type activeRun struct { } type RunRequest struct { - Campaign string `json:"campaign"` - ScenarioIDs []string `json:"scenario_ids"` - BaselineS *int `json:"baseline_s"` // default 300 (design 8) - CooldownS *int `json:"cooldown_s"` // default 300 - ObserveTimeoutS *int `json:"observe_timeout_s"` // default 1800 - MinObserveS *int `json:"min_observe_s"` // default 180: earliest t_recover (12 probe successes after inject) - DriftBefore *int `json:"drift_before"` - GitSHA string `json:"git_sha"` - Notes string `json:"notes"` - SkipRevert bool `json:"skip_revert"` + Campaign string `json:"campaign"` + ScenarioIDs []string `json:"scenario_ids"` + BaselineS *int `json:"baseline_s"` // default 300 (design 8) + CooldownS *int `json:"cooldown_s"` // default 300 + ObserveTimeoutS *int `json:"observe_timeout_s"` // default 1800 + MinObserveS *int `json:"min_observe_s"` // default 180: earliest t_recover (12 probe successes after inject) + DriftBefore *int `json:"drift_before"` + GitSHA string `json:"git_sha"` + Notes string `json:"notes"` + SkipRevert bool `json:"skip_revert"` } func NewRunManager(a *App) *RunManager { return &RunManager{app: a} } @@ -311,6 +311,9 @@ func (a *App) buildVerdict(ctx context.Context, runID string, sc *Scenario, tInj tAlert = &t } case "diagnosis": + if e, _ := p["error"].(string); e != "" { // AI VM unreachable / malformed: not a diagnosis, not a misjudge + continue + } if layer < 2 { layer = 2 }