From cd8fbdddabd9369ec36ee0863a3c14e7af30b9e1 Mon Sep 17 00:00:00 2001 From: seonNoh Date: Thu, 8 Oct 2026 19:20:35 +0900 Subject: [PATCH] feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状 （レプリケーション一時停止、メモリ圧迫）でも、受信中のアラートが解決するまで recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた MIG インスタンス（Mimir topk）を指し、run+シナリオ単位でキャッシュして revert が 同じインスタンスに当たるようにする（ILB のハッシュで api の接続が 1 台に偏る）。 --- harness/alerts.go | 31 ++++++++++++++++++++++++++++++- harness/gcp.go | 36 ++++++++++++++++++++++++++++++++++++ harness/judge.go | 37 ++++++++++++++++++++++++++++++++++++- harness/main.go | 12 +++++++----- 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/harness/alerts.go b/harness/alerts.go index 316f7f3..7b38111 100644 --- a/harness/alerts.go +++ b/harness/alerts.go @@ -9,6 +9,7 @@ import ( "io" "log" "net/http" + "sort" "sync" "time" ) @@ -22,10 +23,37 @@ type AlertRouter struct { app *App mu sync.Mutex timers map[string]*time.Timer // fingerprint -> escalation timer + firing map[string]string // fingerprint -> alertname while Grafana reports it firing (judge: no recovery with an open alert) } func NewAlertRouter(a *App) *AlertRouter { - return &AlertRouter{app: a, timers: map[string]*time.Timer{}} + return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}} +} + +// Firing lists the alert names currently firing (deduplicated, sorted). +func (r *AlertRouter) Firing() []string { + r.mu.Lock() + defer r.mu.Unlock() + seen := map[string]bool{} + var out []string + for _, n := range r.firing { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + sort.Strings(out) + return out +} + +func (r *AlertRouter) setFiring(fp, name string, on bool) { + r.mu.Lock() + defer r.mu.Unlock() + if on { + r.firing[fp] = name + } else { + delete(r.firing, fp) + } } // Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/) @@ -80,6 +108,7 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) { "startsAt": al.StartsAt, "fingerprint": al.Fingerprint, "layer": al.Labels["layer"], "signature": sig} r.app.store.Event(bg, runID, sid, "alert", "grafana", payload) handled++ + r.setFiring(al.Fingerprint, name, al.Status == "firing") if al.Status != "firing" { r.cancelTimer(al.Fingerprint) continue diff --git a/harness/gcp.go b/harness/gcp.go index 2cd6af2..65d8223 100644 --- a/harness/gcp.go +++ b/harness/gcp.go @@ -60,6 +60,9 @@ func (a *App) gcpListInstances(ctx context.Context, prefix string) ([]string, er // resolveInstance turns "legacy:N" (N-th MIG instance by name) or a pet VM short name (db-primary -> selfheal-db-primary) into an instance. func (a *App) resolveInstance(ctx context.Context, ref string) (gceInstance, error) { + if ref == "legacy:busy" { + return a.resolveBusyLegacy(ctx) + } if strings.HasPrefix(ref, "legacy:") { n, _ := strconv.Atoi(strings.TrimPrefix(ref, "legacy:")) l, err := a.gcpInstances(ctx, "legacy-") @@ -170,3 +173,36 @@ func (a *App) gcpOp(ctx context.Context, op, instance, mig string, size int64) ( func (a *App) region(zone string) string { return zone[:strings.LastIndex(zone, "-")] } func (a *App) regionOfMIG() string { return env("GCP_REGION", "asia-northeast1") } + +// resolveBusyLegacy picks the MIG instance that served the most legacy-api requests over the last 2 minutes. The ILB +// hashes the api pods' few keep-alive connections onto one instance (campaign 2, 2026-10-08: legacy:0 served 0 rps while +// the delay was on it), so a fault must go where the traffic is. The choice is cached per run+scenario so the revert +// hits the same instance after the traffic has moved away from it. +func (a *App) resolveBusyLegacy(ctx context.Context) (gceInstance, error) { + key := "" + if cur := a.runs.current(); cur != nil { + key = cur.RunID + "/" + cur.ScenarioID + } + if key != "" { + if v, ok := a.busyLegacy.Load(key); ok { + return v.(gceInstance), nil + } + } + name, err := a.mimirTopInstance(ctx, `topk(1, sum by (instance) (rate(legacy_http_requests_total{route!="healthz"}[2m])))`) + if err != nil { + return gceInstance{}, fmt.Errorf("legacy:busy: %w", err) + } + l, err := a.gcpInstances(ctx, "legacy-") + if err != nil { + return gceInstance{}, err + } + for _, i := range l { + if i.Name == name { + if key != "" { + a.busyLegacy.Store(key, i) + } + return i, nil + } + } + return gceInstance{}, fmt.Errorf("legacy:busy: instance %q from Mimir not in the MIG (%d instances)", name, len(l)) +} diff --git a/harness/judge.go b/harness/judge.go index 0aceb94..4dd3430 100644 --- a/harness/judge.go +++ b/harness/judge.go @@ -45,6 +45,35 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) { return strconv.ParseFloat(s, 64) } +// mimirTopInstance runs an instant query and returns the "instance" label of the first sample. +func (a *App) mimirTopInstance(ctx context.Context, expr string) (string, error) { + u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr) + req, _ := http.NewRequestWithContext(ctx, "GET", u, nil) + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + return "", fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300)) + } + var r struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(b, &r); err != nil { + return "", err + } + if len(r.Data.Result) == 0 || r.Data.Result[0].Metric["instance"] == "" { + return "", fmt.Errorf("no instance sample for %s", expr) + } + return r.Data.Result[0].Metric["instance"], nil +} + // SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m). // Values are nil when the series is absent (JSON has no NaN). // 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the @@ -59,6 +88,8 @@ type SLOState struct { LatencyOK bool `json:"latency_ok"` LatencyNowOK bool `json:"latency_1m_ok"` ErrorsOK bool `json:"errors_ok"` + AlertsFiring []string `json:"alerts_firing,omitempty"` // Grafana alerts received firing and not yet resolved (2026-10-08: grey-zone symptoms live outside the three SLO series) + AlertsOK bool `json:"alerts_ok"` Recovered bool `json:"recovered"` MissingData []string `json:"missing_data,omitempty"` QueryErrors []string `json:"query_errors,omitempty"` @@ -93,6 +124,10 @@ func (a *App) evaluateSLO(ctx context.Context) SLOState { st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5 st.ErrorRatio = get("errors", qErrRatio) st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01 - st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0 + if a.hooks != nil { + st.AlertsFiring = a.hooks.Firing() + } + st.AlertsOK = len(st.AlertsFiring) == 0 + st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && st.AlertsOK && len(st.QueryErrors) == 0 return st } diff --git a/harness/main.go b/harness/main.go index 74d4f76..918223f 100644 --- a/harness/main.go +++ b/harness/main.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "strings" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -74,11 +75,12 @@ var ( ) type App struct { - cfg Config - store *Store - scen map[string]*Scenario - runs *RunManager - hooks *AlertRouter + cfg Config + store *Store + scen map[string]*Scenario + runs *RunManager + hooks *AlertRouter + busyLegacy sync.Map // run/scenario -> gceInstance chosen for legacy:busy (gcp.go) } func main() {