feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution

キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状
(レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで
recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が
alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた
MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が
同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
seon committed 2026-10-08 19:20:35 +09:00
1 parent 13797e5741
commit cd8fbdddab
4 files changed
+109 -7

No files matched your search

+36 -1
View File
@@ -45,6 +45,35 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
return strconv.ParseFloat(s, 64)
}
// mimirTopInstance runs an instant query and returns the "instance" label of the first sample.
func (a *App) mimirTopInstance(ctx context.Context, expr string) (string, error) {
u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr)
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != 200 {
return "", fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300))
}
var r struct {
Data struct {
Result []struct {
Metric map[string]string `json:"metric"`
} `json:"result"`
} `json:"data"`
}
if err := json.Unmarshal(b, &r); err != nil {
return "", err
}
if len(r.Data.Result) == 0 || r.Data.Result[0].Metric["instance"] == "" {
return "", fmt.Errorf("no instance sample for %s", expr)
}
return r.Data.Result[0].Metric["instance"], nil
}
// SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
// Values are nil when the series is absent (JSON has no NaN).
// 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the
@@ -59,6 +88,8 @@ type SLOState struct {
LatencyOK bool `json:"latency_ok"`
LatencyNowOK bool `json:"latency_1m_ok"`
ErrorsOK bool `json:"errors_ok"`
AlertsFiring []string `json:"alerts_firing,omitempty"` // Grafana alerts received firing and not yet resolved (2026-10-08: grey-zone symptoms live outside the three SLO series)
AlertsOK bool `json:"alerts_ok"`
Recovered bool `json:"recovered"`
MissingData []string `json:"missing_data,omitempty"`
QueryErrors []string `json:"query_errors,omitempty"`
@@ -93,6 +124,10 @@ func (a *App) evaluateSLO(ctx context.Context) SLOState {
st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5
st.ErrorRatio = get("errors", qErrRatio)
st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01
st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0
if a.hooks != nil {
st.AlertsFiring = a.hooks.Firing()
}
st.AlertsOK = len(st.AlertsFiring) == 0
st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && st.AlertsOK && len(st.QueryErrors) == 0
return st
}