feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution

キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状
(レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで
recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が
alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた
MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が
同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
seon committed 2026-10-08 19:20:35 +09:00
1 parent 13797e5741
commit cd8fbdddab
4 files changed
+109 -7

No files matched your search

+36
View File
@@ -60,6 +60,9 @@ func (a *App) gcpListInstances(ctx context.Context, prefix string) ([]string, er
// resolveInstance turns "legacy:N" (N-th MIG instance by name) or a pet VM short name (db-primary -> selfheal-db-primary) into an instance.
func (a *App) resolveInstance(ctx context.Context, ref string) (gceInstance, error) {
if ref == "legacy:busy" {
return a.resolveBusyLegacy(ctx)
}
if strings.HasPrefix(ref, "legacy:") {
n, _ := strconv.Atoi(strings.TrimPrefix(ref, "legacy:"))
l, err := a.gcpInstances(ctx, "legacy-")
@@ -170,3 +173,36 @@ func (a *App) gcpOp(ctx context.Context, op, instance, mig string, size int64) (
func (a *App) region(zone string) string { return zone[:strings.LastIndex(zone, "-")] }
func (a *App) regionOfMIG() string { return env("GCP_REGION", "asia-northeast1") }
// resolveBusyLegacy picks the MIG instance that served the most legacy-api requests over the last 2 minutes. The ILB
// hashes the api pods' few keep-alive connections onto one instance (campaign 2, 2026-10-08: legacy:0 served 0 rps while
// the delay was on it), so a fault must go where the traffic is. The choice is cached per run+scenario so the revert
// hits the same instance after the traffic has moved away from it.
func (a *App) resolveBusyLegacy(ctx context.Context) (gceInstance, error) {
key := ""
if cur := a.runs.current(); cur != nil {
key = cur.RunID + "/" + cur.ScenarioID
}
if key != "" {
if v, ok := a.busyLegacy.Load(key); ok {
return v.(gceInstance), nil
}
}
name, err := a.mimirTopInstance(ctx, `topk(1, sum by (instance) (rate(legacy_http_requests_total{route!="healthz"}[2m])))`)
if err != nil {
return gceInstance{}, fmt.Errorf("legacy:busy: %w", err)
}
l, err := a.gcpInstances(ctx, "legacy-")
if err != nil {
return gceInstance{}, err
}
for _, i := range l {
if i.Name == name {
if key != "" {
a.busyLegacy.Store(key, i)
}
return i, nil
}
}
return gceInstance{}, fmt.Errorf("legacy:busy: instance %q from Mimir not in the MIG (%d instances)", name, len(l))
}