feat(harness): hold recovery while a Grafana alert is firing, add legacy:busy host resolution
キャンペーン第 2 回のグレーゾーン 3 件を受けた変更。SLO 3 系列に現れない症状 (レプリケーション一時停止、メモリ圧迫)でも、受信中のアラートが解決するまで recovered を出さないよう、AlertRouter が firing 中のアラートを保持し judge が alerts_ok を条件に加える。legacy:busy は直近 2 分で最も多くリクエストを受けた MIG インスタンス(Mimir topk)を指し、run+シナリオ単位でキャッシュして revert が 同じインスタンスに当たるようにする(ILB のハッシュで api の接続が 1 台に偏る)。
This commit is contained in:
1 parent
13797e5741
commit
cd8fbdddab
4 files changed
+109
-7
No files matched your search
+36
-1
@@ -45,6 +45,35 @@ func (a *App) mimirScalar(ctx context.Context, expr string) (float64, error) {
|
||||
return strconv.ParseFloat(s, 64)
|
||||
}
|
||||
|
||||
// mimirTopInstance runs an instant query and returns the "instance" label of the first sample.
|
||||
func (a *App) mimirTopInstance(ctx context.Context, expr string) (string, error) {
|
||||
u := a.cfg.MimirURL + "/api/v1/query?query=" + url.QueryEscape(expr)
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
||||
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("mimir %s: %s", resp.Status, truncate(string(b), 300))
|
||||
}
|
||||
var r struct {
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Metric map[string]string `json:"metric"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(r.Data.Result) == 0 || r.Data.Result[0].Metric["instance"] == "" {
|
||||
return "", fmt.Errorf("no instance sample for %s", expr)
|
||||
}
|
||||
return r.Data.Result[0].Metric["instance"], nil
|
||||
}
|
||||
|
||||
// SLOState is one evaluation of experiment design 4.2 (probe 12 consecutive successes = min over 3m; p95 and 5xx ratio over 5m).
|
||||
// Values are nil when the series is absent (JSON has no NaN).
|
||||
// 2026-10-08 (campaign 2, C-G1): a 300 ms delay that lasted 3 minutes was declared recovered while still active because the
|
||||
@@ -59,6 +88,8 @@ type SLOState struct {
|
||||
LatencyOK bool `json:"latency_ok"`
|
||||
LatencyNowOK bool `json:"latency_1m_ok"`
|
||||
ErrorsOK bool `json:"errors_ok"`
|
||||
AlertsFiring []string `json:"alerts_firing,omitempty"` // Grafana alerts received firing and not yet resolved (2026-10-08: grey-zone symptoms live outside the three SLO series)
|
||||
AlertsOK bool `json:"alerts_ok"`
|
||||
Recovered bool `json:"recovered"`
|
||||
MissingData []string `json:"missing_data,omitempty"`
|
||||
QueryErrors []string `json:"query_errors,omitempty"`
|
||||
@@ -93,6 +124,10 @@ func (a *App) evaluateSLO(ctx context.Context) SLOState {
|
||||
st.LatencyNowOK = st.P95Now == nil || *st.P95Now < 0.5
|
||||
st.ErrorRatio = get("errors", qErrRatio)
|
||||
st.ErrorsOK = st.ErrorRatio == nil || *st.ErrorRatio < 0.01
|
||||
st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && len(st.QueryErrors) == 0
|
||||
if a.hooks != nil {
|
||||
st.AlertsFiring = a.hooks.Firing()
|
||||
}
|
||||
st.AlertsOK = len(st.AlertsFiring) == 0
|
||||
st.Recovered = st.ProbeOK && st.LatencyOK && st.LatencyNowOK && st.ErrorsOK && st.AlertsOK && len(st.QueryErrors) == 0
|
||||
return st
|
||||
}
|
||||
Reference in new issue
Block a user