diff --git a/vm/ai-env/prompts/slo.md b/vm/ai-env/prompts/slo.md index 87700b4..344b3d7 100644 --- a/vm/ai-env/prompts/slo.md +++ b/vm/ai-env/prompts/slo.md @@ -1,4 +1,4 @@ 可用性: キャンペーン期間を通じて avg_over_time(probe_success[5m]) >= 99.5%。レイテンシ: api の p95(histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[5m]))))が 0.5 秒未満。エラー率: 5 分間の 5xx / 全リクエストが 1% 未満。 復旧(t_recover)は、プローブが 12 回連続(3 分)で成功し、かつ 5 分窓のレイテンシとエラー率が SLO の内側に戻ったときにハーネスが宣言する。 層: 1 = プラットフォーム内部の自動復旧(kubelet の再起動、ReplicaSet、Argo CD selfHeal、MIG autohealing、compose の restart)。2 = あなたの提案 + Discord での人の承認、実行はハーネス。3 = 人だけが行う(不可逆な操作、15 分以内に承認がない場合、確信度 0.5 未満)。 -アラートルールと意味: ProbeDown(probe_success == 0 が 1 分、症状のみ)、ApiLatencyHigh(p95 > 500 ms が 3 分)、ApiErrorRateHigh(5xx > 1% が 3 分)、PodCrashLooping(5 分で 3 回以上の再起動。まず層 1、5 分後に層 2)、LegacyUnhealthy(up な legacy-api インスタンスが 2 未満。まず autohealing、10 分後に層 2)、DbReplicationLag(遅延 > 60 秒、または replay が一時停止)、DbPrimaryDown(TCP 5432 の失敗が 1 分)、GitOpsDrift(OutOfSync が 5 分。selfHeal が処理する)。 +アラートルールと意味: ProbeDown(probe_success == 0 が 1 分、症状のみ)、ApiLatencyHigh(p95 > 500 ms が 3 分)、ApiErrorRateHigh(5xx > 1% が 3 分)、PodCrashLooping(5 分で 3 回以上の再起動。まず層 1、5 分後に層 2)、LegacyUnhealthy(up な legacy-api インスタンスが 2 未満。まず autohealing、10 分後に層 2)、DbReplicationLag(遅延 > 60 秒、または replay が一時停止)、ApiMemoryPressure(api コンテナのメモリが limit の 80% 超を 2 分継続。グレーゾーン、承認後に rollout restart)、DbPrimaryDown(TCP 5432 の失敗が 1 分)、GitOpsDrift(OutOfSync が 5 分。selfHeal が処理する)。 diff --git a/vm/harness/scenarios/C-G2.yaml b/vm/harness/scenarios/C-G2.yaml index 5d8ebab..2ded628 100644 --- a/vm/harness/scenarios/C-G2.yaml +++ b/vm/harness/scenarios/C-G2.yaml @@ -5,7 +5,7 @@ asset: container expected_layer: 2 ground_truth_label: memory_pressure_api ground_truth_action: k8s.rollout_restart -description: StressChaos memory workers 2 size 80%, app=api, duration 10m +description: StressChaos memory workers 2 size 200MB (api limit 256Mi), app=api, duration 10m notes: open_item: "design 11.3: whether the stress re-attaches to the restarted pods for the remaining duration" inject: @@ -24,7 +24,10 @@ inject: stressors: memory: workers: 2 - size: "80%" + # 2026-10-08: "80%" is a share of the node memory (memStress reached 4.4 GB), so the cgroup OOM killer took memStress + # and the api process within a second: one restart per pod, no sustained pressure, no alert. 200MB stays under the + # 256Mi limit and keeps the working set near 90% for the whole duration (ApiMemoryPressure fires after 2m). + size: "200MB" duration: "10m" revert: method: chaos_delete diff --git a/vm/harness/scenarios/V-G1.yaml b/vm/harness/scenarios/V-G1.yaml index 9a14a83..81fcfa0 100644 --- a/vm/harness/scenarios/V-G1.yaml +++ b/vm/harness/scenarios/V-G1.yaml @@ -5,12 +5,15 @@ asset: vm expected_layer: 2 ground_truth_label: network_latency_legacy ground_truth_action: mig.recreate_instance -description: tc qdisc add dev ens4 root netem delay 400ms on legacy instance 0 +description: tc qdisc add dev ens4 root netem delay 400ms on the legacy instance that currently serves the api traffic (legacy:busy) +# 2026-10-08: the ILB hashes the api pods' few keep-alive connections onto one instance; in campaign 2 legacy:0 served 0 rps, +# so the delay was never on the request path. legacy:busy = the instance with the most legacy-api requests over the last +# 2 minutes (Mimir); the harness caches the choice per scenario so the revert hits the same instance. inject: method: ssh - host: legacy:0 + host: legacy:busy command: sudo tc qdisc add dev ens4 root netem delay 400ms revert: method: ssh - host: legacy:0 + host: legacy:busy command: sudo tc qdisc del dev ens4 root || true diff --git a/vm/observability/grafana/provisioning/alerting/rules.yaml b/vm/observability/grafana/provisioning/alerting/rules.yaml index 865e0ac..f74e0e6 100644 --- a/vm/observability/grafana/provisioning/alerting/rules.yaml +++ b/vm/observability/grafana/provisioning/alerting/rules.yaml @@ -57,8 +57,7 @@ groups: model: refId: A instant: true - expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m]))) - > bool 0.5 + expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m]))) > bool 0.5 - refId: C datasourceUid: __expr__ model: @@ -89,8 +88,7 @@ groups: model: refId: A instant: true - expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m]))) - > bool 0.01 + expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m]))) > bool 0.01 - refId: C datasourceUid: __expr__ model: @@ -112,8 +110,7 @@ groups: layer: '1' signature: pod_crashloop annotations: - summary: container restarted 3+ times in 5m (kubelet is handling; escalate after - 5m) + summary: container restarted 3+ times in 5m (kubelet is handling; escalate after 5m) data: - refId: A relativeTimeRange: @@ -123,8 +120,7 @@ groups: model: refId: A instant: true - expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m])) - >= bool 3 + expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m])) >= bool 3 - refId: C datasourceUid: __expr__ model: @@ -146,8 +142,7 @@ groups: layer: '1' signature: legacy_unhealthy annotations: - summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling; - escalate after 10m) + summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling; escalate after 10m) data: - refId: A relativeTimeRange: @@ -158,7 +153,6 @@ groups: refId: A instant: true expr: count(up{job="legacy-api"} == 1) < bool 2 - # no series until WP-15 ships Alloy on the legacy VMs: absent -> NoData -> OK (the `or vector(0)` form fired continuously) - refId: C datasourceUid: __expr__ model: @@ -189,8 +183,7 @@ groups: model: refId: A instant: true - expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused) - == bool 1) + expr: clamp_max((max(pg_replication_lag_seconds) > bool 60) + (max(pg_is_wal_replay_paused) == bool 1), 1) - refId: C datasourceUid: __expr__ model: @@ -265,3 +258,35 @@ groups: type: gt params: - 0 + - uid: api-memory-pressure + title: ApiMemoryPressure + condition: C + for: 2m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: an api container holds more than 80% of its memory limit for 2 minutes (grey zone; rollout restart after approval) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: max by (pod) (container_memory_working_set_bytes{namespace="demo",container="api"}) / on (pod) max by (pod) (kube_pod_container_resource_limits{namespace="demo",container="api",resource="memory"}) + > bool 0.8 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0