diff --git a/vm/ai-env/prompts/slo.md b/vm/ai-env/prompts/slo.md index 344b3d7..e8c1d56 100644 --- a/vm/ai-env/prompts/slo.md +++ b/vm/ai-env/prompts/slo.md @@ -1,4 +1,4 @@ 可用性: キャンペーン期間を通じて avg_over_time(probe_success[5m]) >= 99.5%。レイテンシ: api の p95(histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[5m]))))が 0.5 秒未満。エラー率: 5 分間の 5xx / 全リクエストが 1% 未満。 復旧(t_recover)は、プローブが 12 回連続(3 分)で成功し、かつ 5 分窓のレイテンシとエラー率が SLO の内側に戻ったときにハーネスが宣言する。 層: 1 = プラットフォーム内部の自動復旧(kubelet の再起動、ReplicaSet、Argo CD selfHeal、MIG autohealing、compose の restart)。2 = あなたの提案 + Discord での人の承認、実行はハーネス。3 = 人だけが行う(不可逆な操作、15 分以内に承認がない場合、確信度 0.5 未満)。 -アラートルールと意味: ProbeDown(probe_success == 0 が 1 分、症状のみ)、ApiLatencyHigh(p95 > 500 ms が 3 分)、ApiErrorRateHigh(5xx > 1% が 3 分)、PodCrashLooping(5 分で 3 回以上の再起動。まず層 1、5 分後に層 2)、LegacyUnhealthy(up な legacy-api インスタンスが 2 未満。まず autohealing、10 分後に層 2)、DbReplicationLag(遅延 > 60 秒、または replay が一時停止)、ApiMemoryPressure(api コンテナのメモリが limit の 80% 超を 2 分継続。グレーゾーン、承認後に rollout restart)、DbPrimaryDown(TCP 5432 の失敗が 1 分)、GitOpsDrift(OutOfSync が 5 分。selfHeal が処理する)。 +アラートルールと意味: ProbeDown(probe_success == 0 が 1 分、症状のみ)、ApiLatencyHigh(直近 1 分の p95 > 500 ms が 2 分)、ApiErrorRateHigh(5xx > 1% が 3 分)、PodCrashLooping(5 分で 3 回以上の再起動。まず層 1、5 分後に層 2)、LegacyUnhealthy(up な legacy-api インスタンスが 2 未満。まず autohealing、10 分後に層 2)、DbReplicationLag(遅延 > 60 秒、または replay が一時停止)、ApiMemoryPressure(api コンテナのメモリが limit の 80% 超を 1 分継続。グレーゾーン、承認後に rollout restart)、DbPrimaryDown(TCP 5432 の失敗が 1 分)、GitOpsDrift(OutOfSync が 5 分。selfHeal が処理する)。 diff --git a/vm/harness/scenarios/C-G2.yaml b/vm/harness/scenarios/C-G2.yaml index 2ded628..1bcd0e0 100644 --- a/vm/harness/scenarios/C-G2.yaml +++ b/vm/harness/scenarios/C-G2.yaml @@ -5,7 +5,7 @@ asset: container expected_layer: 2 ground_truth_label: memory_pressure_api ground_truth_action: k8s.rollout_restart -description: StressChaos memory workers 2 size 200MB (api limit 256Mi), app=api, duration 10m +description: StressChaos memory workers 2 size 225MB (api limit 256Mi, working set about 0.9), app=api, duration 10m notes: open_item: "design 11.3: whether the stress re-attaches to the restarted pods for the remaining duration" inject: @@ -26,8 +26,9 @@ inject: workers: 2 # 2026-10-08: "80%" is a share of the node memory (memStress reached 4.4 GB), so the cgroup OOM killer took memStress # and the api process within a second: one restart per pod, no sustained pressure, no alert. 200MB stays under the - # 256Mi limit and keeps the working set near 90% for the whole duration (ApiMemoryPressure fires after 2m). - size: "200MB" + # 256Mi limit. 200MB left the working set at 0.797-0.803 of the limit (campaign 3), flapping around the 0.8 + # threshold; 225MB keeps it near 0.9 so ApiMemoryPressure holds. + size: "225MB" duration: "10m" revert: method: chaos_delete diff --git a/vm/observability/grafana/provisioning/alerting/rules.yaml b/vm/observability/grafana/provisioning/alerting/rules.yaml index f74e0e6..99c06fb 100644 --- a/vm/observability/grafana/provisioning/alerting/rules.yaml +++ b/vm/observability/grafana/provisioning/alerting/rules.yaml @@ -41,13 +41,13 @@ groups: - uid: api-latency-high title: ApiLatencyHigh condition: C - for: 3m + for: 2m noDataState: OK execErrState: Error labels: layer: '2' annotations: - summary: demo api p95 latency above 500ms (grey zone) + summary: api p95 over the last minute above 500 ms for 2 minutes data: - refId: A relativeTimeRange: @@ -57,7 +57,7 @@ groups: model: refId: A instant: true - expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m]))) > bool 0.5 + expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m]))) > bool 0.5 - refId: C datasourceUid: __expr__ model: @@ -261,13 +261,13 @@ groups: - uid: api-memory-pressure title: ApiMemoryPressure condition: C - for: 2m + for: 1m noDataState: OK execErrState: Error labels: layer: '2' annotations: - summary: an api container holds more than 80% of its memory limit for 2 minutes (grey zone; rollout restart after approval) + summary: an api container holds more than 80% of its memory limit for 1 minute (grey zone; rollout restart after approval) data: - refId: A relativeTimeRange: