fix(harness): JSON-safe SLO state, unscored AI errors, mark recorded-only proposals executed

This commit is contained in:
seon committed 2026-09-24 02:22:29 +09:00
1 parent 5daf13b669
commit de076a79e4
4 files changed
+24 -15

No files matched your search

+3
View File
@@ -92,6 +92,9 @@ func (a *App) handleActionExecute(w http.ResponseWriter, r *http.Request) {
if !ce.Exec { if !ce.Exec {
a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "recorded_only": true, "note": req.Note, "layer": ce.Layer}) a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "recorded_only": true, "note": req.Note, "layer": ce.Layer})
mActs.WithLabelValues(req.ActionID, actor, "recorded").Inc() mActs.WithLabelValues(req.ActionID, actor, "recorded").Inc()
if actor == "ai-approved" {
_ = a.store.MarkProposalExecuted(ctx, req.Token)
}
writeJSON(w, 202, map[string]any{"action_id": req.ActionID, "recorded_only": true, "layer": ce.Layer, "note": "not executed by the harness (platform-internal or human-only); recorded in events"}) writeJSON(w, 202, map[string]any{"action_id": req.ActionID, "recorded_only": true, "layer": ce.Layer, "note": "not executed by the harness (platform-internal or human-only); recorded in events"})
return return
} }
+4 -1
View File
@@ -14,6 +14,7 @@ import (
) )
// AlertRouter implements design 5.2 "알림 수신기": Grafana webhook -> t_alert, then // AlertRouter implements design 5.2 "알림 수신기": Grafana webhook -> t_alert, then
//
// signature present -> layer 1: the platform mechanism is already acting; the harness observes and, if the alert // signature present -> layer 1: the platform mechanism is already acting; the harness observes and, if the alert
// is still firing after escalate_after_s, hands it to layer 2 // is still firing after escalate_after_s, hands it to layer 2
// no signature -> layer 2: POST the alert to the AI VM /diagnose and record the diagnosis // no signature -> layer 2: POST the alert to the AI VM /diagnose and record the diagnosis
@@ -23,7 +24,9 @@ type AlertRouter struct {
timers map[string]*time.Timer // fingerprint -> escalation timer timers map[string]*time.Timer // fingerprint -> escalation timer
} }
func NewAlertRouter(a *App) *AlertRouter { return &AlertRouter{app: a, timers: map[string]*time.Timer{}} } func NewAlertRouter(a *App) *AlertRouter {
return &AlertRouter{app: a, timers: map[string]*time.Timer{}}
}
// Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/) // Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/)
type grafanaWebhook struct { type grafanaWebhook struct {
+14 -14
View File
@@ -46,11 +46,12 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
} }
// SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m). // SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
// Values are nil when the series is absent (JSON has no NaN).
type SLOState struct { type SLOState struct {
At time.Time `json:"at"` At time.Time `json:"at"`
ProbeMin3m float64 `json:"probe_min_3m"` ProbeMin3m *float64 `json:"probe_min_3m"`
P95 float64 `json:"p95_s"` P95 *float64 `json:"p95_s"`
ErrorRatio float64 `json:"error_ratio"` ErrorRatio *float64 `json:"error_ratio"`
ProbeOK bool `json:"probe_ok"` ProbeOK bool `json:"probe_ok"`
LatencyOK bool `json:"latency_ok"` LatencyOK bool `json:"latency_ok"`
ErrorsOK bool `json:"errors_ok"` ErrorsOK bool `json:"errors_ok"`
@@ -67,25 +68,24 @@ const (
func (a *App) evaluateSLO(ctx context.Context) SLOState { func (a *App) evaluateSLO(ctx context.Context) SLOState {
st := SLOState{At: time.Now().UTC()} st := SLOState{At: time.Now().UTC()}
get := func(name, expr string) (float64, bool) { get := func(name, expr string) *float64 {
v, err := a.mimirScalar(ctx, expr) v, err := a.mimirScalar(ctx, expr)
if err != nil { if err != nil {
st.QueryErrors = append(st.QueryErrors, name+": "+err.Error()) st.QueryErrors = append(st.QueryErrors, name+": "+err.Error())
return math.NaN(), false return nil
} }
if math.IsNaN(v) { if math.IsNaN(v) {
st.MissingData = append(st.MissingData, name) st.MissingData = append(st.MissingData, name)
return v, false return nil
} }
return v, true return &v
} }
var ok bool st.ProbeMin3m = get("probe", qProbeMin3m)
st.ProbeMin3m, ok = get("probe", qProbeMin3m) st.ProbeOK = st.ProbeMin3m != nil && *st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery
st.ProbeOK = ok && st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery st.P95 = get("p95", qP95)
st.P95, ok = get("p95", qP95) st.LatencyOK = st.P95 == nil || *st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO
st.LatencyOK = !ok || st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO st.ErrorRatio = get("errors", qErrRatio)
st.ErrorRatio, ok = get("errors", qErrRatio) st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01
st.ErrorsOK = !ok || st.ErrorRatio < 0.01
st.Recovered = st.ProbeOK && st.LatencyOK && st.ErrorsOK && len(st.QueryErrors) == 0 st.Recovered = st.ProbeOK && st.LatencyOK && st.ErrorsOK && len(st.QueryErrors) == 0
return st return st
} }
+3
View File
@@ -311,6 +311,9 @@ func (a *App) buildVerdict(ctx context.Context, runID string, sc *Scenario, tInj
tAlert = &t tAlert = &t
} }
case "diagnosis": case "diagnosis":
if e, _ := p["error"].(string); e != "" { // AI VM unreachable / malformed: not a diagnosis, not a misjudge
continue
}
if layer < 2 { if layer < 2 {
layer = 2 layer = 2
} }