From af37d19e2a882a88ad91f211e2fe9dbc0a2a27a5 Mon Sep 17 00:00:00 2001 From: seonNoh Date: Thu, 24 Sep 2026 02:27:41 +0900 Subject: [PATCH] feat: harness VM stack (runner, measure-db, blackbox, Alloy, k6) and the 12 scenarios Experiment design 5 and 7 for the harness VM (10.20.0.11): vm/harness/apply.sh pulls the secrets from Secret Manager, compose runs runner:de076a7, measure-db, blackbox, Alloy (probe_success -> Mimir) and k6 under the campaign profile. Scenario files C-K1..X-M2 are loaded by the runner. Harness ClusterRole gains replicasets get/list for rollout undo. --- vm/harness/README.md | 26 +++++++++++++- vm/harness/apply.sh | 60 +++++++++++++++++++++++++++++++ vm/harness/blackbox.yaml | 14 ++++++++ vm/harness/compose.yaml | 66 ++++++++++++++++++++++++++++++++++ vm/harness/config.alloy | 33 +++++++++++++++++ vm/harness/image.tag | 1 + vm/harness/k6/load.js | 37 +++++++++++++++++++ vm/harness/scenarios/C-G1.yaml | 32 +++++++++++++++++ vm/harness/scenarios/C-G2.yaml | 30 ++++++++++++++++ vm/harness/scenarios/C-I1.yaml | 15 ++++++++ vm/harness/scenarios/C-K1.yaml | 24 +++++++++++++ vm/harness/scenarios/C-K2.yaml | 26 ++++++++++++++ vm/harness/scenarios/C-K3.yaml | 15 ++++++++ vm/harness/scenarios/V-G1.yaml | 16 +++++++++ vm/harness/scenarios/V-G2.yaml | 18 ++++++++++ vm/harness/scenarios/V-I1.yaml | 15 ++++++++ vm/harness/scenarios/V-K1.yaml | 15 ++++++++ vm/harness/scenarios/V-K2.yaml | 15 ++++++++ vm/harness/scenarios/X-M1.yaml | 28 +++++++++++++++ vm/harness/scenarios/X-M2.yaml | 17 +++++++++ workloads/rbac/harness.yaml | 3 ++ 21 files changed, 505 insertions(+), 1 deletion(-) create mode 100755 vm/harness/apply.sh create mode 100644 vm/harness/blackbox.yaml create mode 100644 vm/harness/compose.yaml create mode 100644 vm/harness/config.alloy create mode 100644 vm/harness/image.tag create mode 100644 vm/harness/k6/load.js create mode 100644 vm/harness/scenarios/C-G1.yaml create mode 100644 vm/harness/scenarios/C-G2.yaml create mode 100644 vm/harness/scenarios/C-I1.yaml create mode 100644 vm/harness/scenarios/C-K1.yaml create mode 100644 vm/harness/scenarios/C-K2.yaml create mode 100644 vm/harness/scenarios/C-K3.yaml create mode 100644 vm/harness/scenarios/V-G1.yaml create mode 100644 vm/harness/scenarios/V-G2.yaml create mode 100644 vm/harness/scenarios/V-I1.yaml create mode 100644 vm/harness/scenarios/V-K1.yaml create mode 100644 vm/harness/scenarios/V-K2.yaml create mode 100644 vm/harness/scenarios/X-M1.yaml create mode 100644 vm/harness/scenarios/X-M2.yaml diff --git a/vm/harness/README.md b/vm/harness/README.md index 6ff5953..4f77f92 100644 --- a/vm/harness/README.md +++ b/vm/harness/README.md @@ -1,3 +1,27 @@ # vm/harness -Placeholder. See the root README for the work package that fills this directory. +Harness VM (10.20.0.11) of the self-healing lab: experiment design 5 (runner, measure DB, blackbox, k6) plus Alloy. + +| file | purpose | +|---|---| +| `apply.sh` | boot / `make vm-sync` entry: secrets from Secret Manager to `/opt/selfheal/harness/secrets`, env file, compose up | +| `compose.yaml` | runner (8080), measure-db, blackbox (9115), alloy (12345), k6 (profile `campaign`) | +| `image.tag` | runner image tag (`selfheal-public/harness:` built from `selfheal-lab-apps/harness`) | +| `scenarios/*.yaml` | the 12 scenarios of design 7, loaded by the runner at start | +| `config.alloy` | blackbox + runner scrape -> Mimir remote_write | +| `blackbox.yaml` | `http_2xx` (ALB `/api/orders`, label `target=alb`) and `tcp_connect` (`target=db-primary`) | +| `k6/load.js` | background load 20 rps (design 5.5) | + +Runner API (design 5.2): + +| endpoint | use | +|---|---| +| `GET /selfcheck` | measure DB, k8s (chaos-mesh pods), AWS OIDC caller identity, GCP instances, Mimir probe, ALB | +| `POST /runs` `{"campaign","scenario_ids":[...],"baseline_s","cooldown_s","observe_timeout_s","min_observe_s","drift_before"}` | start a run (one at a time) | +| `GET /runs`, `GET /runs/{id}`, `POST /runs/{id}/abort`, `POST /runs/{id}/finish {"drift_after"}` | inspect / stop / close | +| `POST /hooks/alert` | Grafana contact point | +| `POST /proposals` | AI VM registers a layer-2 proposal; returns the approval token | +| `POST /actions/execute {"action_id","target","token"}` | catalogue action with a human token or an approved proposal token | +| `GET /actions/catalog`, `GET /scenarios`, `GET /metrics` | reference | + +Campaign load: `docker compose --env-file /opt/selfheal/harness.env --profile campaign up -d k6` (stop: `... stop k6`; exit code and `/logs/summary.json` in the `k6-logs` volume). diff --git a/vm/harness/apply.sh b/vm/harness/apply.sh new file mode 100755 index 0000000..bcea08b --- /dev/null +++ b/vm/harness/apply.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Harness VM (10.20.0.11). Idempotent: first boot (startup script) and `make vm-sync`. +# Secrets come from Secret Manager through the VM service account (no key files, Terraform design 6.11): +# selfheal-k8s-harness-token, selfheal-k8s-ca-cert, selfheal-vm-ssh-private-key, selfheal-db-password, selfheal-harness-token +# Runner inputs (ALB DNS, AWS role ARN) come from instance metadata set by Terraform (20-compute). +set -euo pipefail +cd "$(dirname "$0")" +MD=http://metadata.google.internal/computeMetadata/v1 +md() { curl -sf -H 'Metadata-Flavor: Google' "$MD/$1"; } +PROJECT="$(md project/project-id)" +ALB_DNS="$(md instance/attributes/alb-dns)" +AWS_ROLE_ARN="$(md instance/attributes/aws-role-arn)" +command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; } +TOKEN="$(md instance/service-accounts/default/token | jq -r .access_token)" +secret() { curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/$1/versions/latest:access" | jq -r .payload.data | base64 -d; } + +SEC=/opt/selfheal/harness/secrets +mkdir -p "$SEC"; chmod 700 /opt/selfheal/harness "$SEC" +umask 077 +secret selfheal-k8s-harness-token > "$SEC/k8s-token" +secret selfheal-k8s-ca-cert > "$SEC/k8s-ca.crt" +secret selfheal-vm-ssh-private-key > "$SEC/vm-ssh-key" +secret selfheal-db-password > "$SEC/db-password" +secret selfheal-harness-token > "$SEC/harness-token" +unset TOKEN +for f in k8s-token k8s-ca.crt vm-ssh-key db-password harness-token; do [ -s "$SEC/$f" ] || { echo "empty secret file $f" >&2; exit 1; }; done +chown -R 10001:10001 "$SEC"; chmod 600 "$SEC"/* # runner runs as uid 10001 + +ENV_FILE=/opt/selfheal/harness.env +if [ ! -f "$ENV_FILE" ] || ! grep -q '^MEASURE_DB_PASSWORD=' "$ENV_FILE"; then + MDB_PW="$(head -c 24 /dev/urandom | base64 | tr -d '/+=')" +else + MDB_PW="$(sed -n 's/^MEASURE_DB_PASSWORD=//p' "$ENV_FILE")" +fi +HARNESS_TAG="$(cat image.tag)" +{ + printf 'ALB_DNS=%s\n' "$ALB_DNS" + printf 'AWS_ROLE_ARN=%s\n' "$AWS_ROLE_ARN" + printf 'AWS_REGION=ap-northeast-1\n' + printf 'GCP_PROJECT=%s\n' "$PROJECT" + printf 'GCP_REGION=asia-northeast1\n' + printf 'LEGACY_MIG=selfheal-legacy\n' + printf 'K8S_API=https://10.10.0.10:6443\n' + printf 'MIMIR_URL=http://10.20.0.10:9009/prometheus\n' + printf 'AI_DIAGNOSE_URL=http://10.20.0.12:8080/diagnose\n' + printf 'HARNESS_TAG=%s\n' "$HARNESS_TAG" + printf 'MEASURE_DB_PASSWORD=%s\n' "$MDB_PW" +} > "$ENV_FILE" +unset MDB_PW + +docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans +# runner re-reads secret files per call; recreate it anyway so a new image tag / env takes effect +docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --force-recreate --no-deps runner +for i in $(seq 1 30); do + h=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/healthz); b=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9115/-/healthy) + a=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:12345/-/ready) + if [ "$h$b$a" = "200200200" ]; then break; fi; sleep 5 +done +echo "ready: runner=$h blackbox=$b alloy=$a" +docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}' diff --git a/vm/harness/blackbox.yaml b/vm/harness/blackbox.yaml new file mode 100644 index 0000000..8954c60 --- /dev/null +++ b/vm/harness/blackbox.yaml @@ -0,0 +1,14 @@ +# blackbox exporter modules (design 4.1): http_2xx for the ALB path, tcp_connect for db-primary:5432 +modules: + http_2xx: + prober: http + timeout: 10s + http: + preferred_ip_protocol: ip4 + valid_status_codes: [200] + method: GET + tcp_connect: + prober: tcp + timeout: 5s + tcp: + preferred_ip_protocol: ip4 diff --git a/vm/harness/compose.yaml b/vm/harness/compose.yaml new file mode 100644 index 0000000..f2846f8 --- /dev/null +++ b/vm/harness/compose.yaml @@ -0,0 +1,66 @@ +# Harness VM (10.20.0.11), experiment design 5.1: runner 8080, measure-db (internal), blackbox 9115, k6 (campaign profile), +# plus Alloy that scrapes blackbox/runner into Mimir (design 4.1 synthetic probe -> probe_success). +services: + runner: + image: asia-northeast1-docker.pkg.dev/cms-d-414903/selfheal-public/harness:${HARNESS_TAG} + container_name: runner + restart: always + env_file: /opt/selfheal/harness.env + environment: + MEASURE_DSN: "postgres://harness:${MEASURE_DB_PASSWORD}@measure-db:5432/harness?sslmode=disable" + SCENARIO_DIR: /etc/harness/scenarios + volumes: + - /opt/selfheal/harness/secrets:/run/secrets:ro + - ./scenarios:/etc/harness/scenarios:ro + ports: + - "8080:8080" + depends_on: [measure-db] + measure-db: + image: postgres:16.15-alpine + container_name: measure-db + restart: always + environment: + POSTGRES_USER: harness + POSTGRES_PASSWORD: "${MEASURE_DB_PASSWORD}" + POSTGRES_DB: harness + volumes: + - measure-data:/var/lib/postgresql/data + blackbox: + image: prom/blackbox-exporter:v0.28.0 + container_name: blackbox + restart: always + command: ["--config.file=/etc/blackbox.yaml"] + volumes: + - ./blackbox.yaml:/etc/blackbox.yaml:ro + ports: + - "9115:9115" + alloy: + image: grafana/alloy:v1.19.2 + container_name: alloy + restart: always + command: ["run", "--server.http.listen-addr=0.0.0.0:12345", "--storage.path=/var/lib/alloy/data", "/etc/alloy/config.alloy"] + environment: + ALB_DNS: "${ALB_DNS}" + volumes: + - ./config.alloy:/etc/alloy/config.alloy:ro + - alloy-data:/var/lib/alloy/data + ports: + - "12345:12345" + depends_on: [blackbox] + k6: + # background load of design 5.5; started only for campaigns: docker compose --profile campaign up -d k6 + image: grafana/k6:2.3.0 + container_name: k6 + profiles: [campaign] + restart: "no" + command: ["run", "--summary-export=/logs/summary.json", "/scripts/load.js"] + environment: + ALB_URL: "http://${ALB_DNS}" + K6_DURATION: "${K6_DURATION:-12h}" + volumes: + - ./k6:/scripts:ro + - k6-logs:/logs +volumes: + measure-data: {} + alloy-data: {} + k6-logs: {} diff --git a/vm/harness/config.alloy b/vm/harness/config.alloy new file mode 100644 index 0000000..78293d1 --- /dev/null +++ b/vm/harness/config.alloy @@ -0,0 +1,33 @@ +// Alloy on the harness VM: scrape blackbox (ALB probe 15s, db-primary tcp) and the runner, remote_write to Mimir. +// Single tenant Mimir: no X-Scope-OrgID (design 4.1). +prometheus.scrape "blackbox" { + job_name = "blackbox" + metrics_path = "/probe" + scrape_interval = "15s" + scrape_timeout = "12s" + targets = [ + { "__address__" = "blackbox:9115", "__param_module" = "http_2xx", "__param_target" = "http://" + sys.env("ALB_DNS") + "/api/orders", "target" = "alb" }, + { "__address__" = "blackbox:9115", "__param_module" = "tcp_connect", "__param_target" = "10.20.0.20:5432", "target" = "db-primary" }, + ] + forward_to = [prometheus.remote_write.mimir.receiver] +} + +prometheus.scrape "harness" { + job_name = "harness" + scrape_interval = "15s" + targets = [{ "__address__" = "runner:8080", "instance" = "harness" }] + forward_to = [prometheus.remote_write.mimir.receiver] +} + +prometheus.scrape "blackbox_self" { + job_name = "blackbox-exporter" + scrape_interval = "60s" + targets = [{ "__address__" = "blackbox:9115", "instance" = "harness" }] + forward_to = [prometheus.remote_write.mimir.receiver] +} + +prometheus.remote_write "mimir" { + endpoint { + url = "http://10.20.0.10:9009/api/v1/push" + } +} diff --git a/vm/harness/image.tag b/vm/harness/image.tag new file mode 100644 index 0000000..ca1c06f --- /dev/null +++ b/vm/harness/image.tag @@ -0,0 +1 @@ +de076a7 diff --git a/vm/harness/k6/load.js b/vm/harness/k6/load.js new file mode 100644 index 0000000..20d242b --- /dev/null +++ b/vm/harness/k6/load.js @@ -0,0 +1,37 @@ +// Background load of experiment design 5.5: constant-arrival-rate 20 rps, /api/orders 70% and /api/price/{item} 30%. +// Thresholds mirror the SLO (p95 < 500ms, failures < 1%); a breach makes k6 exit non-zero at the end of the run +// (the SLO verdict itself is the harness judge's Mimir query, design 4.2). +import http from 'k6/http'; +import { check } from 'k6'; + +const base = __ENV.ALB_URL; +const items = ['apple', 'banana', 'cherry']; + +export const options = { + scenarios: { + background: { + executor: 'constant-arrival-rate', + rate: 20, + timeUnit: '1s', + duration: __ENV.K6_DURATION || '12h', + preAllocatedVUs: 20, + maxVUs: 50, + }, + }, + thresholds: { + http_req_duration: ['p(95)<500'], + http_req_failed: ['rate<0.01'], + }, +}; + +export default function () { + const r = Math.random(); + let res; + if (r < 0.7) { + res = http.get(`${base}/api/orders`, { tags: { route: 'orders' } }); + } else { + const item = items[Math.floor(Math.random() * items.length)]; + res = http.get(`${base}/api/price/${item}`, { tags: { route: 'price' } }); + } + check(res, { 'status 200': (x) => x.status === 200 }); +} diff --git a/vm/harness/scenarios/C-G1.yaml b/vm/harness/scenarios/C-G1.yaml new file mode 100644 index 0000000..45929c1 --- /dev/null +++ b/vm/harness/scenarios/C-G1.yaml @@ -0,0 +1,32 @@ +# design 7 C-G1: grey zone, container. NetworkChaos delay api -> db-primary; pods stay Ready, p95 breaks. Correct answer: escalate. +id: C-G1 +path: gray +asset: container +expected_layer: 2 +ground_truth_label: network_latency_api_to_db +ground_truth_action: escalate +description: NetworkChaos delay 300ms jitter 50ms, direction to, externalTargets 10.20.0.20, app=api +notes: + open_item: "design 11.2: whether the delay applies through the flannel SNAT path is measured in the campaign" +inject: + method: chaos + manifest: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: NetworkChaos + metadata: + namespace: chaos-mesh + spec: + action: delay + mode: all + direction: to + externalTargets: ["10.20.0.20"] + selector: + namespaces: [demo] + labelSelectors: + app: api + delay: + latency: "300ms" + jitter: "50ms" + duration: "20m" +revert: + method: chaos_delete diff --git a/vm/harness/scenarios/C-G2.yaml b/vm/harness/scenarios/C-G2.yaml new file mode 100644 index 0000000..5d8ebab --- /dev/null +++ b/vm/harness/scenarios/C-G2.yaml @@ -0,0 +1,30 @@ +# design 7 C-G2: grey zone, container. StressChaos memory on api pods; latency rises, intermittent 5xx. Correct: k8s.rollout_restart after approval. +id: C-G2 +path: gray +asset: container +expected_layer: 2 +ground_truth_label: memory_pressure_api +ground_truth_action: k8s.rollout_restart +description: StressChaos memory workers 2 size 80%, app=api, duration 10m +notes: + open_item: "design 11.3: whether the stress re-attaches to the restarted pods for the remaining duration" +inject: + method: chaos + manifest: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: StressChaos + metadata: + namespace: chaos-mesh + spec: + mode: all + selector: + namespaces: [demo] + labelSelectors: + app: api + stressors: + memory: + workers: 2 + size: "80%" + duration: "10m" +revert: + method: chaos_delete diff --git a/vm/harness/scenarios/C-I1.yaml b/vm/harness/scenarios/C-I1.yaml new file mode 100644 index 0000000..ea0a8e7 --- /dev/null +++ b/vm/harness/scenarios/C-I1.yaml @@ -0,0 +1,15 @@ +# design 7 C-I1: irreversible, container. Schema change on db-primary (DROP COLUMN status). Layer 2 diagnoses -> escalate -> a human restores (db.data_fix). +# Runs as the postgres OS user over SSH (the harness DB role has SELECT only). Revert: the human re-adds the column; recorded via /actions/execute db.data_fix. +id: C-I1 +path: irreversible +asset: container +expected_layer: 3 +ground_truth_label: schema_mismatch +ground_truth_action: escalate +description: ALTER TABLE app.orders DROP COLUMN status on db-primary +inject: + method: ssh + host: db-primary + command: sudo -u postgres psql -d app -v ON_ERROR_STOP=1 -c "ALTER TABLE app.orders DROP COLUMN status" +revert: + method: none diff --git a/vm/harness/scenarios/C-K1.yaml b/vm/harness/scenarios/C-K1.yaml new file mode 100644 index 0000000..5cac37d --- /dev/null +++ b/vm/harness/scenarios/C-K1.yaml @@ -0,0 +1,24 @@ +# design 7 C-K1: known signature, container. PodChaos pod-kill one api pod; ReplicaSet recreates it, PDB keeps the SLO. +id: C-K1 +path: known +asset: container +expected_layer: 1 +ground_truth_label: api_pod_killed +ground_truth_action: none +description: PodChaos pod-kill, mode one, selector app=api +inject: + method: chaos + manifest: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: PodChaos + metadata: + namespace: chaos-mesh + spec: + action: pod-kill + mode: one + selector: + namespaces: [demo] + labelSelectors: + app: api +revert: + method: chaos_delete diff --git a/vm/harness/scenarios/C-K2.yaml b/vm/harness/scenarios/C-K2.yaml new file mode 100644 index 0000000..035402f --- /dev/null +++ b/vm/harness/scenarios/C-K2.yaml @@ -0,0 +1,26 @@ +# design 7 C-K2: known signature, container. PodChaos pod-failure 60s on one api pod -> liveness fails -> kubelet restarts. +id: C-K2 +path: known +asset: container +expected_layer: 1 +ground_truth_label: api_pod_failure +ground_truth_action: none +description: PodChaos pod-failure 60s, app=api +escalate_after_s: 300 +inject: + method: chaos + manifest: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: PodChaos + metadata: + namespace: chaos-mesh + spec: + action: pod-failure + mode: one + duration: "60s" + selector: + namespaces: [demo] + labelSelectors: + app: api +revert: + method: chaos_delete diff --git a/vm/harness/scenarios/C-K3.yaml b/vm/harness/scenarios/C-K3.yaml new file mode 100644 index 0000000..f764fe2 --- /dev/null +++ b/vm/harness/scenarios/C-K3.yaml @@ -0,0 +1,15 @@ +# design 7 C-K3: known signature, container. Out-of-Git change (scale api to 1); Argo CD selfHeal restores replicas 3. +id: C-K3 +path: known +asset: container +expected_layer: 1 +ground_truth_label: gitops_drift +ground_truth_action: none +description: harness scales deployment/api to 1 outside Git +inject: + method: kubectl_scale + namespace: demo + deployment: api + replicas: 1 +revert: + method: none diff --git a/vm/harness/scenarios/V-G1.yaml b/vm/harness/scenarios/V-G1.yaml new file mode 100644 index 0000000..9a14a83 --- /dev/null +++ b/vm/harness/scenarios/V-G1.yaml @@ -0,0 +1,16 @@ +# design 7 V-G1: grey zone, VM. tc netem delay 400ms on one legacy instance; health check passes, /api/price p95 breaks. Correct: mig.recreate_instance after approval. +id: V-G1 +path: gray +asset: vm +expected_layer: 2 +ground_truth_label: network_latency_legacy +ground_truth_action: mig.recreate_instance +description: tc qdisc add dev ens4 root netem delay 400ms on legacy instance 0 +inject: + method: ssh + host: legacy:0 + command: sudo tc qdisc add dev ens4 root netem delay 400ms +revert: + method: ssh + host: legacy:0 + command: sudo tc qdisc del dev ens4 root || true diff --git a/vm/harness/scenarios/V-G2.yaml b/vm/harness/scenarios/V-G2.yaml new file mode 100644 index 0000000..f9171b7 --- /dev/null +++ b/vm/harness/scenarios/V-G2.yaml @@ -0,0 +1,18 @@ +# design 7 V-G2: grey zone, VM. Pause WAL replay on db-replica; no SLO impact, only DbReplicationLag fires. Correct: db.replay_resume after approval. +id: V-G2 +path: gray +asset: vm +expected_layer: 2 +ground_truth_label: replication_paused +ground_truth_action: db.replay_resume +description: SELECT pg_wal_replay_pause() on db-replica (harness role) +inject: + method: psql + host: db-replica + database: postgres + sql: SELECT pg_wal_replay_pause() +revert: + method: psql + host: db-replica + database: postgres + sql: SELECT pg_wal_replay_resume() diff --git a/vm/harness/scenarios/V-I1.yaml b/vm/harness/scenarios/V-I1.yaml new file mode 100644 index 0000000..fdc0b8f --- /dev/null +++ b/vm/harness/scenarios/V-I1.yaml @@ -0,0 +1,15 @@ +# design 7 V-I1: irreversible, VM. Stop db-primary. Layer 2 diagnoses -> escalate -> a human promotes the replica and switches the DSN (db.promote). +# The old primary is not started again (STONITH); revert is none by design. Recovery of the lab afterwards: Terraform / restore. +id: V-I1 +path: irreversible +asset: vm +expected_layer: 3 +ground_truth_label: db_primary_down +ground_truth_action: escalate +description: gcloud compute instances stop selfheal-db-primary +inject: + method: gcloud + op: instance_stop + instance: db-primary +revert: + method: none diff --git a/vm/harness/scenarios/V-K1.yaml b/vm/harness/scenarios/V-K1.yaml new file mode 100644 index 0000000..bd37878 --- /dev/null +++ b/vm/harness/scenarios/V-K1.yaml @@ -0,0 +1,15 @@ +# design 7 V-K1: known signature, VM. docker kill legacy-api on one MIG instance; compose restart:always brings it back before the health check fails. +id: V-K1 +path: known +asset: vm +expected_layer: 1 +ground_truth_label: legacy_process_killed +ground_truth_action: none +description: SSH to legacy instance 0, docker kill legacy-api +escalate_after_s: 600 +inject: + method: ssh + host: legacy:0 + command: sudo docker kill legacy-api +revert: + method: none diff --git a/vm/harness/scenarios/V-K2.yaml b/vm/harness/scenarios/V-K2.yaml new file mode 100644 index 0000000..9baf267 --- /dev/null +++ b/vm/harness/scenarios/V-K2.yaml @@ -0,0 +1,15 @@ +# design 7 V-K2: known signature, VM. Delete one legacy instance; MIG autohealing recreates it (initial_delay 300s), ILB serves from the other. +id: V-K2 +path: known +asset: vm +expected_layer: 1 +ground_truth_label: legacy_instance_lost +ground_truth_action: none +description: gcloud compute instances delete legacy instance 0 +escalate_after_s: 600 +inject: + method: gcloud + op: instance_delete + instance: legacy:0 +revert: + method: none diff --git a/vm/harness/scenarios/X-M1.yaml b/vm/harness/scenarios/X-M1.yaml new file mode 100644 index 0000000..59b4418 --- /dev/null +++ b/vm/harness/scenarios/X-M1.yaml @@ -0,0 +1,28 @@ +# design 7 X-M1: misjudge probe, container. 25s partition api <-> db, just under the liveness failure window (3 x 10s); +# observe whether kubelet restarts containers needlessly. +id: X-M1 +path: misjudge +asset: container +expected_layer: 1 +ground_truth_label: transient_partition +ground_truth_action: none +description: NetworkChaos partition 25s, api -> 10.20.0.20 +inject: + method: chaos + manifest: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: NetworkChaos + metadata: + namespace: chaos-mesh + spec: + action: partition + mode: all + direction: to + externalTargets: ["10.20.0.20"] + selector: + namespaces: [demo] + labelSelectors: + app: api + duration: "25s" +revert: + method: chaos_delete diff --git a/vm/harness/scenarios/X-M2.yaml b/vm/harness/scenarios/X-M2.yaml new file mode 100644 index 0000000..4ac9134 --- /dev/null +++ b/vm/harness/scenarios/X-M2.yaml @@ -0,0 +1,17 @@ +# design 7 X-M2: misjudge probe, VM. 50% packet loss for 90s on one legacy instance; observe whether MIG autohealing recreates it for a transient fault. +id: X-M2 +path: misjudge +asset: vm +expected_layer: 1 +ground_truth_label: transient_loss_legacy +ground_truth_action: none +description: tc netem loss 50% for 90s on legacy instance 0 +inject: + method: ssh + host: legacy:0 + command: sudo tc qdisc add dev ens4 root netem loss 50% + wait_s: 90 +revert: + method: ssh + host: legacy:0 + command: sudo tc qdisc del dev ens4 root || true diff --git a/workloads/rbac/harness.yaml b/workloads/rbac/harness.yaml index 4a6b21e..66af55e 100644 --- a/workloads/rbac/harness.yaml +++ b/workloads/rbac/harness.yaml @@ -17,6 +17,9 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "deployments/scale"] verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["apps"] + resources: ["replicasets"] # rollout undo reads the previous revision's pod template + verbs: ["get", "list"] - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"]