feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution

キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状
（レプリケーション一時停止、メモリ圧迫）でも、受信中のアラートが解決するまで
recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が
alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた
MIG インスタンス（Mimir topk）を指し、run+シナリオ単位でキャッシュして revert が
同じインスタンスに当たるようにする（ILB のハッシュで api の接続が 1 台に偏る）。

harness/alerts.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"sort"
 	"sync"
 	"time"
 )
@@ -22,10 +23,37 @@ type AlertRouter struct {
 	app    *App
 	mu     sync.Mutex
 	timers map[string]*time.Timer // fingerprint -> escalation timer
+	firing map[string]string      // fingerprint -> alertname while Grafana reports it firing (judge: no recovery with an open alert)
 }
 
 func NewAlertRouter(a *App) *AlertRouter {
-	return &AlertRouter{app: a, timers: map[string]*time.Timer{}}
+	return &AlertRouter{app: a, timers: map[string]*time.Timer{}, firing: map[string]string{}}
+}
+
+// Firing lists the alert names currently firing (deduplicated, sorted).
+func (r *AlertRouter) Firing() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	seen := map[string]bool{}
+	var out []string
+	for _, n := range r.firing {
+		if !seen[n] {
+			seen[n] = true
+			out = append(out, n)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (r *AlertRouter) setFiring(fp, name string, on bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if on {
+		r.firing[fp] = name
+	} else {
+		delete(r.firing, fp)
+	}
 }
 
 // Grafana webhook payload (https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier/)
@@ -80,6 +108,7 @@ func (r *AlertRouter) handleWebhook(w http.ResponseWriter, req *http.Request) {
 			"startsAt": al.StartsAt, "fingerprint": al.Fingerprint, "layer": al.Labels["layer"], "signature": sig}
 		r.app.store.Event(bg, runID, sid, "alert", "grafana", payload)
 		handled++
+		r.setFiring(al.Fingerprint, name, al.Status == "firing")
 		if al.Status != "firing" {
 			r.cancelTimer(al.Fingerprint)
 			continue

harness/gcp.go

@@ -60,6 +60,9 @@ func (a *App) gcpListInstances(ctx context.Context, prefix string) ([]string, er
 
 // resolveInstance turns "legacy:N" (N-th MIG instance by name) or a pet VM short name (db-primary -> selfheal-db-primary) into an instance.
 func (a *App) resolveInstance(ctx context.Context, ref string) (gceInstance, error) {
+	if ref == "legacy:busy" {
+		return a.resolveBusyLegacy(ctx)
+	}
 	if strings.HasPrefix(ref, "legacy:") {
 		n, _ := strconv.Atoi(strings.TrimPrefix(ref, "legacy:"))
 		l, err := a.gcpInstances(ctx, "legacy-")
@@ -170,3 +173,36 @@ func (a *App) gcpOp(ctx context.Context, op, instance, mig string, size int64) (
 
 func (a *App) region(zone string) string { return zone[:strings.LastIndex(zone, "-")] }
 func (a *App) regionOfMIG() string       { return env("GCP_REGION", "asia-northeast1") }
+
+// resolveBusyLegacy picks the MIG instance that served the most legacy-api requests over the last 2 minutes. The ILB
+// hashes the api pods' few keep-alive connections onto one instance (campaign 2, 2026-10-08: legacy:0 served 0 rps while
+// the delay was on it), so a fault must go where the traffic is. The choice is cached per run+scenario so the revert
+// hits the same instance after the traffic has moved away from it.
+func (a *App) resolveBusyLegacy(ctx context.Context) (gceInstance, error) {
+	key := ""
+	if cur := a.runs.current(); cur != nil {
+		key = cur.RunID + "/" + cur.ScenarioID
+	}
+	if key != "" {
+		if v, ok := a.busyLegacy.Load(key); ok {
+			return v.(gceInstance), nil
+		}
+	}
+	name, err := a.mimirTopInstance(ctx, `topk(1, sum by (instance) (rate(legacy_http_requests_total{route!="healthz"}[2m])))`)
+	if err != nil {
+		return gceInstance{}, fmt.Errorf("legacy:busy: %w", err)
+	}
+	l, err := a.gcpInstances(ctx, "legacy-")
+	if err != nil {
+		return gceInstance{}, err
+	}
+	for _, i := range l {
+		if i.Name == name {
+			if key != "" {
+				a.busyLegacy.Store(key, i)
+			}
+			return i, nil
+		}
+	}
+	return gceInstance{}, fmt.Errorf("legacy:busy: instance %q from Mimir not in the MIG (%d instances)", name, len(l))
+}

harness/judge.go

@@ -45,6 +45,35 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
 	return strconv.ParseFloat(s, 64)
 }
 
+// mimirTopInstance runs an instant query and returns the "instance" label of the first sample.
+func (a *App) mimirTopInstance(ctx context.Context, expr string) (string, error) {
+	u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr)
+	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
+	resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300))
+	}
+	var r struct {
+		Data struct {
+			Result []struct {
+				Metric map[string]string `json:"metric"`
+			} `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(b, &r); err != nil {
+		return "", err
+	}
+	if len(r.Data.Result) == 0 || r.Data.Result[0].Metric["instance"] == "" {
+		return "", fmt.Errorf("no instance sample for %s", expr)
+	}
+	return r.Data.Result[0].Metric["instance"], nil
+}
+
 // SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
 // Values are nil when the series is absent (JSON has no NaN).
 // 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the
@@ -59,6 +88,8 @@ type SLOState struct {
 	LatencyOK    bool      `json:"latency_ok"`
 	LatencyNowOK bool      `json:"latency_1m_ok"`
 	ErrorsOK     bool      `json:"errors_ok"`
+	AlertsFiring []string  `json:"alerts_firing,omitempty"` // Grafana alerts received firing and not yet resolved (2026-10-08: grey-zone symptoms live outside the three SLO series)
+	AlertsOK     bool      `json:"alerts_ok"`
 	Recovered    bool      `json:"recovered"`
 	MissingData  []string  `json:"missing_data,omitempty"`
 	QueryErrors  []string  `json:"query_errors,omitempty"`
@@ -93,6 +124,10 @@ func (a *App) evaluateSLO(ctx context.Context) SLOState {
 	st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5
 	st.ErrorRatio = get("errors", qErrRatio)
 	st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01
-	st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0
+	if a.hooks != nil {
+		st.AlertsFiring = a.hooks.Firing()
+	}
+	st.AlertsOK = len(st.AlertsFiring) == 0
+	st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && st.AlertsOK && len(st.QueryErrors) == 0
 	return st
 }

harness/main.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -74,11 +75,12 @@ var (
 )
 
 type App struct {
-	cfg   Config
+	cfg        Config
-	store *Store
+	store      *Store
-	scen  map[string]*Scenario
+	scen       map[string]*Scenario
-	runs  *RunManager
+	runs       *RunManager
-	hooks *AlertRouter
+	hooks      *AlertRouter
+	busyLegacy sync.Map // run/scenario -> gceInstance chosen for legacy:busy (gcp.go)
 }
 
 func main() {