seon
1e0e17b33c
DbReplicationLag の式は PromQL の or が左辺のサンプル（常に 0）を残すため pg_is_wal_replay_paused の分岐が評価されず、第 1・2 回とも発火しなかった。2 つの bool を加算して clamp する式に直す。api コンテナのメモリが limit の 80% を超えて 2 分続いたときの ApiMemoryPressure（第 2 層）を追加する。C-G2 の StressChaos は size "80%"（ノード基準）で即 OOM になっていたため 200MB に、V-G1 の対象は トラフィックを受けているインスタンス legacy:busy に変更する。slo.md に新ルールを追記。
293 lines
7.0 KiB
YAML
293 lines
7.0 KiB
YAML
# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
|
|
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: selfheal
|
|
folder: selfheal
|
|
interval: 30s
|
|
rules:
|
|
- uid: probe-down
|
|
title: ProbeDown
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: ALB synthetic probe failing (symptom alert, cause unknown)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: api-latency-high
|
|
title: ApiLatencyHigh
|
|
condition: C
|
|
for: 3m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: demo api p95 latency above 500ms (grey zone)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m]))) > bool 0.5
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: api-error-rate-high
|
|
title: ApiErrorRateHigh
|
|
condition: C
|
|
for: 3m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: demo api 5xx ratio above 1% (grey zone)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m]))) > bool 0.01
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: pod-crashlooping
|
|
title: PodCrashLooping
|
|
condition: C
|
|
for: 0s
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '1'
|
|
signature: pod_crashloop
|
|
annotations:
|
|
summary: container restarted 3+ times in 5m (kubelet is handling; escalate after 5m)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m])) >= bool 3
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: legacy-unhealthy
|
|
title: LegacyUnhealthy
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '1'
|
|
signature: legacy_unhealthy
|
|
annotations:
|
|
summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling; escalate after 10m)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: count(up{job="legacy-api"} == 1) < bool 2
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: db-replication-lag
|
|
title: DbReplicationLag
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: clamp_max((max(pg_replication_lag_seconds) > bool 60) + (max(pg_is_wal_replay_paused) == bool 1), 1)
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: db-primary-down
|
|
title: DbPrimaryDown
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: gitops-drift
|
|
title: GitOpsDrift
|
|
condition: C
|
|
for: 5m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '1'
|
|
signature: gitops_drift
|
|
annotations:
|
|
summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|
|
- uid: api-memory-pressure
|
|
title: ApiMemoryPressure
|
|
condition: C
|
|
for: 2m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
labels:
|
|
layer: '2'
|
|
annotations:
|
|
summary: an api container holds more than 80% of its memory limit for 2 minutes (grey zone; rollout restart after approval)
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: mimir
|
|
model:
|
|
refId: A
|
|
instant: true
|
|
expr: max by (pod) (container_memory_working_set_bytes{namespace="demo",container="api"}) / on (pod) max by (pod) (kube_pod_container_resource_limits{namespace="demo",container="api",resource="memory"})
|
|
> bool 0.8
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
refId: C
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator:
|
|
type: gt
|
|
params:
|
|
- 0
|