feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution
キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状 (レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が 同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
1 parent
13797e5741
commit
cd8fbdddab
4 files changed
+109
-7
No files matched your search
@@ -60,6 +60,9 @@ func (a *App) gcpListInstances(ctx context.Context, prefix string) ([]string, er
|
||||
|
||||
// resolveInstance turns "legacy:N" (N-th MIG instance by name) or a pet VM short name (db-primary -> selfheal-db-primary) into an instance.
|
||||
func (a *App) resolveInstance(ctx context.Context, ref string) (gceInstance, error) {
|
||||
if ref == "legacy:busy" {
|
||||
return a.resolveBusyLegacy(ctx)
|
||||
}
|
||||
if strings.HasPrefix(ref, "legacy:") {
|
||||
n, _ := strconv.Atoi(strings.TrimPrefix(ref, "legacy:"))
|
||||
l, err := a.gcpInstances(ctx, "legacy-")
|
||||
@@ -170,3 +173,36 @@ func (a *App) gcpOp(ctx context.Context, op, instance, mig string, size int64) (
|
||||
|
||||
func (a *App) region(zone string) string { return zone[:strings.LastIndex(zone, "-")] }
|
||||
func (a *App) regionOfMIG() string { return env("GCP_REGION", "asia-northeast1") }
|
||||
|
||||
// resolveBusyLegacy picks the MIG instance that served the most legacy-api requests over the last 2 minutes. The ILB
|
||||
// hashes the api pods' few keep-alive connections onto one instance (campaign 2, 2026-10-08: legacy:0 served 0 rps while
|
||||
// the delay was on it), so a fault must go where the traffic is. The choice is cached per run+scenario so the revert
|
||||
// hits the same instance after the traffic has moved away from it.
|
||||
func (a *App) resolveBusyLegacy(ctx context.Context) (gceInstance, error) {
|
||||
key := ""
|
||||
if cur := a.runs.current(); cur != nil {
|
||||
key = cur.RunID + "/" + cur.ScenarioID
|
||||
}
|
||||
if key != "" {
|
||||
if v, ok := a.busyLegacy.Load(key); ok {
|
||||
return v.(gceInstance), nil
|
||||
}
|
||||
}
|
||||
name, err := a.mimirTopInstance(ctx, `topk(1, sum by (instance) (rate(legacy_http_requests_total{route!="healthz"}[2m])))`)
|
||||
if err != nil {
|
||||
return gceInstance{}, fmt.Errorf("legacy:busy: %w", err)
|
||||
}
|
||||
l, err := a.gcpInstances(ctx, "legacy-")
|
||||
if err != nil {
|
||||
return gceInstance{}, err
|
||||
}
|
||||
for _, i := range l {
|
||||
if i.Name == name {
|
||||
if key != "" {
|
||||
a.busyLegacy.Store(key, i)
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
return gceInstance{}, fmt.Errorf("legacy:busy: instance %q from Mimir not in the MIG (%d instances)", name, len(l))
|
||||
}
|
||||
Reference in new issue
Block a user