diff --git a/harness/judge.go b/harness/judge.go index bf8408f..0aceb94 100644 --- a/harness/judge.go +++ b/harness/judge.go @@ -47,22 +47,27 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) { // SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m). // Values are nil when the series is absent (JSON has no NaN). +// 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the +// 5-minute p95 average diluted it to 0.375 s. The 1-minute p95 (p95_1m_s) must also be inside the SLO. type SLOState struct { - At time.Time `json:"at"` - ProbeMin3m *float64 `json:"probe_min_3m"` - P95 *float64 `json:"p95_s"` - ErrorRatio *float64 `json:"error_ratio"` - ProbeOK bool `json:"probe_ok"` - LatencyOK bool `json:"latency_ok"` - ErrorsOK bool `json:"errors_ok"` - Recovered bool `json:"recovered"` - MissingData []string `json:"missing_data,omitempty"` - QueryErrors []string `json:"query_errors,omitempty"` + At time.Time `json:"at"` + ProbeMin3m *float64 `json:"probe_min_3m"` + P95 *float64 `json:"p95_s"` + P95Now *float64 `json:"p95_1m_s"` + ErrorRatio *float64 `json:"error_ratio"` + ProbeOK bool `json:"probe_ok"` + LatencyOK bool `json:"latency_ok"` + LatencyNowOK bool `json:"latency_1m_ok"` + ErrorsOK bool `json:"errors_ok"` + Recovered bool `json:"recovered"` + MissingData []string `json:"missing_data,omitempty"` + QueryErrors []string `json:"query_errors,omitempty"` } const ( qProbeMin3m = `min_over_time(probe_success{job="blackbox",target="alb"}[3m])` qP95 = `histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))` + qP95Now = `histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m])))` qErrRatio = `sum(rate(demo_http_requests_total{route=~"orders|price",status=~"5.."}[5m])) / sum(rate(demo_http_requests_total{route=~"orders|price"}[5m]))` ) @@ -84,8 +89,10 @@ func (a *App) evaluateSLO(ctx context.Context) SLOState { st.ProbeOK = st.ProbeMin3m != nil && *st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery st.P95 = get("p95", qP95) st.LatencyOK = st.P95 == nil || *st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO + st.P95Now = get("p95_1m", qP95Now) + st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5 st.ErrorRatio = get("errors", qErrRatio) st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01 - st.Recovered = st.ProbeOK && st.LatencyOK && st.ErrorsOK && len(st.QueryErrors) == 0 + st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0 return st }