feat: add harness runner (scenario executor, alert receiver, action executor, judge, revert)

harness/judge.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// Mimir instant query (Prometheus HTTP API under <MIMIR_URL>/api/v1/query; single tenant, no X-Scope-OrgID).
+// Returns NaN when the result vector is empty.
+func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
+	u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr)
+	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
+	resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
+	if err != nil {
+		return math.NaN(), err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != 200 {
+		return math.NaN(), fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300))
+	}
+	var r struct {
+		Status string `json:"status"`
+		Data   struct {
+			ResultType string `json:"resultType"`
+			Result     []struct {
+				Value []any `json:"value"`
+			} `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(b, &r); err != nil {
+		return math.NaN(), err
+	}
+	if r.Status != "success" || len(r.Data.Result) == 0 || len(r.Data.Result[0].Value) < 2 {
+		return math.NaN(), nil
+	}
+	s, _ := r.Data.Result[0].Value[1].(string)
+	return strconv.ParseFloat(s, 64)
+}
+
+// SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
+type SLOState struct {
+	At           time.Time `json:"at"`
+	ProbeMin3m   float64   `json:"probe_min_3m"`
+	P95          float64   `json:"p95_s"`
+	ErrorRatio   float64   `json:"error_ratio"`
+	ProbeOK      bool      `json:"probe_ok"`
+	LatencyOK    bool      `json:"latency_ok"`
+	ErrorsOK     bool      `json:"errors_ok"`
+	Recovered    bool      `json:"recovered"`
+	MissingData  []string  `json:"missing_data,omitempty"`
+	QueryErrors  []string  `json:"query_errors,omitempty"`
+}
+
+const (
+	qProbeMin3m = `min_over_time(probe_success{job="blackbox",target="alb"}[3m])`
+	qP95        = `histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))`
+	qErrRatio   = `sum(rate(demo_http_requests_total{route=~"orders|price",status=~"5.."}[5m])) / sum(rate(demo_http_requests_total{route=~"orders|price"}[5m]))`
+)
+
+func (a *App) evaluateSLO(ctx context.Context) SLOState {
+	st := SLOState{At: time.Now().UTC()}
+	get := func(name, expr string) (float64, bool) {
+		v, err := a.mimirScalar(ctx, expr)
+		if err != nil {
+			st.QueryErrors = append(st.QueryErrors, name+": "+err.Error())
+			return math.NaN(), false
+		}
+		if math.IsNaN(v) {
+			st.MissingData = append(st.MissingData, name)
+			return v, false
+		}
+		return v, true
+	}
+	var ok bool
+	st.ProbeMin3m, ok = get("probe", qProbeMin3m)
+	st.ProbeOK = ok && st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery
+	st.P95, ok = get("p95", qP95)
+	st.LatencyOK = !ok || st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO
+	st.ErrorRatio, ok = get("errors", qErrRatio)
+	st.ErrorsOK = !ok || st.ErrorRatio < 0.01
+	st.Recovered = st.ProbeOK && st.LatencyOK && st.ErrorsOK && len(st.QueryErrors) == 0
+	return st
+}