seon
795be156d9
キャンペーン第 2 回の C-G1 で、300 ms 遅延が 3 分続いている最中に 5 分窓の p95 平均が 0.375 秒まで薄まり、復旧と判定された。SLO 状態に p95_1m_s / latency_1m_ok を追加し、 recovered の条件に 1 分窓の p95 も含める。
99 lines
3.6 KiB
Go
99 lines
3.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
// Mimir instant query (Prometheus HTTP API under <MIMIR_URL>/api/v1/query; single tenant, no X-Scope-OrgID).
|
|
// Returns NaN when the result vector is empty.
|
|
func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
|
|
u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr)
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
|
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
|
|
if err != nil {
|
|
return math.NaN(), err
|
|
}
|
|
defer resp.Body.Close()
|
|
b, _ := io.ReadAll(resp.Body)
|
|
if resp.StatusCode != 200 {
|
|
return math.NaN(), fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300))
|
|
}
|
|
var r struct {
|
|
Status string `json:"status"`
|
|
Data struct {
|
|
ResultType string `json:"resultType"`
|
|
Result []struct {
|
|
Value []any `json:"value"`
|
|
} `json:"result"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(b, &r); err != nil {
|
|
return math.NaN(), err
|
|
}
|
|
if r.Status != "success" || len(r.Data.Result) == 0 || len(r.Data.Result[0].Value) < 2 {
|
|
return math.NaN(), nil
|
|
}
|
|
s, _ := r.Data.Result[0].Value[1].(string)
|
|
return strconv.ParseFloat(s, 64)
|
|
}
|
|
|
|
// SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
|
|
// Values are nil when the series is absent (JSON has no NaN).
|
|
// 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the
|
|
// 5-minute p95 average diluted it to 0.375 s. The 1-minute p95 (p95_1m_s) must also be inside the SLO.
|
|
type SLOState struct {
|
|
At time.Time `json:"at"`
|
|
ProbeMin3m *float64 `json:"probe_min_3m"`
|
|
P95 *float64 `json:"p95_s"`
|
|
P95Now *float64 `json:"p95_1m_s"`
|
|
ErrorRatio *float64 `json:"error_ratio"`
|
|
ProbeOK bool `json:"probe_ok"`
|
|
LatencyOK bool `json:"latency_ok"`
|
|
LatencyNowOK bool `json:"latency_1m_ok"`
|
|
ErrorsOK bool `json:"errors_ok"`
|
|
Recovered bool `json:"recovered"`
|
|
MissingData []string `json:"missing_data,omitempty"`
|
|
QueryErrors []string `json:"query_errors,omitempty"`
|
|
}
|
|
|
|
const (
|
|
qProbeMin3m = `min_over_time(probe_success{job="blackbox",target="alb"}[3m])`
|
|
qP95 = `histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))`
|
|
qP95Now = `histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m])))`
|
|
qErrRatio = `sum(rate(demo_http_requests_total{route=~"orders|price",status=~"5.."}[5m])) / sum(rate(demo_http_requests_total{route=~"orders|price"}[5m]))`
|
|
)
|
|
|
|
func (a *App) evaluateSLO(ctx context.Context) SLOState {
|
|
st := SLOState{At: time.Now().UTC()}
|
|
get := func(name, expr string) *float64 {
|
|
v, err := a.mimirScalar(ctx, expr)
|
|
if err != nil {
|
|
st.QueryErrors = append(st.QueryErrors, name+": "+err.Error())
|
|
return nil
|
|
}
|
|
if math.IsNaN(v) {
|
|
st.MissingData = append(st.MissingData, name)
|
|
return nil
|
|
}
|
|
return &v
|
|
}
|
|
st.ProbeMin3m = get("probe", qProbeMin3m)
|
|
st.ProbeOK = st.ProbeMin3m != nil && *st.ProbeMin3m >= 1 // the probe series must exist: no data is not recovery
|
|
st.P95 = get("p95", qP95)
|
|
st.LatencyOK = st.P95 == nil || *st.P95 < 0.5 // no traffic in the window (no k6) counts as within SLO
|
|
st.P95Now = get("p95_1m", qP95Now)
|
|
st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5
|
|
st.ErrorRatio = get("errors", qErrRatio)
|
|
st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01
|
|
st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0
|
|
return st
|
|
}
|