feat: harness VM stack (runner, measure-db, blackbox, Alloy, k6) and the 12 scenarios

Experiment design 5 and 7 for the harness VM (10.20.0.11): vm/harness/apply.sh pulls the secrets from Secret Manager,
compose runs runner:de076a7, measure-db, blackbox, Alloy (probe_success -> Mimir) and k6 under the campaign profile.
Scenario files C-K1..X-M2 are loaded by the runner. Harness ClusterRole gains replicasets get/list for rollout undo.

vm/harness/README.md

@@ -1,3 +1,27 @@
 # vm/harness
 
-Placeholder. See the root README for the work package that fills this directory.
+Harness VM (10.20.0.11) of the self-healing lab: experiment design 5 (runner, measure DB, blackbox, k6) plus Alloy.
+
+| file | purpose |
+|---|---|
+| `apply.sh` | boot / `make vm-sync` entry: secrets from Secret Manager to `/opt/selfheal/harness/secrets`, env file, compose up |
+| `compose.yaml` | runner (8080), measure-db, blackbox (9115), alloy (12345), k6 (profile `campaign`) |
+| `image.tag` | runner image tag (`selfheal-public/harness:<sha>` built from `selfheal-lab-apps/harness`) |
+| `scenarios/*.yaml` | the 12 scenarios of design 7, loaded by the runner at start |
+| `config.alloy` | blackbox + runner scrape -> Mimir remote_write |
+| `blackbox.yaml` | `http_2xx` (ALB `/api/orders`, label `target=alb`) and `tcp_connect` (`target=db-primary`) |
+| `k6/load.js` | background load 20 rps (design 5.5) |
+
+Runner API (design 5.2):
+
+| endpoint | use |
+|---|---|
+| `GET /selfcheck` | measure DB, k8s (chaos-mesh pods), AWS OIDC caller identity, GCP instances, Mimir probe, ALB |
+| `POST /runs` `{"campaign","scenario_ids":[...],"baseline_s","cooldown_s","observe_timeout_s","min_observe_s","drift_before"}` | start a run (one at a time) |
+| `GET /runs`, `GET /runs/{id}`, `POST /runs/{id}/abort`, `POST /runs/{id}/finish {"drift_after"}` | inspect / stop / close |
+| `POST /hooks/alert` | Grafana contact point |
+| `POST /proposals` | AI VM registers a layer-2 proposal; returns the approval token |
+| `POST /actions/execute {"action_id","target","token"}` | catalogue action with a human token or an approved proposal token |
+| `GET /actions/catalog`, `GET /scenarios`, `GET /metrics` | reference |
+
+Campaign load: `docker compose --env-file /opt/selfheal/harness.env --profile campaign up -d k6` (stop: `... stop k6`; exit code and `/logs/summary.json` in the `k6-logs` volume).

vm/harness/apply.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Harness VM (10.20.0.11). Idempotent: first boot (startup script) and `make vm-sync`.
+# Secrets come from Secret Manager through the VM service account (no key files, Terraform design 6.11):
+#   selfheal-k8s-harness-token, selfheal-k8s-ca-cert, selfheal-vm-ssh-private-key, selfheal-db-password, selfheal-harness-token
+# Runner inputs (ALB DNS, AWS role ARN) come from instance metadata set by Terraform (20-compute).
+set -euo pipefail
+cd "$(dirname "$0")"
+MD=http://metadata.google.internal/computeMetadata/v1
+md() { curl -sf -H 'Metadata-Flavor: Google' "$MD/$1"; }
+PROJECT="$(md project/project-id)"
+ALB_DNS="$(md instance/attributes/alb-dns)"
+AWS_ROLE_ARN="$(md instance/attributes/aws-role-arn)"
+command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; }
+TOKEN="$(md instance/service-accounts/default/token | jq -r .access_token)"
+secret() { curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/$1/versions/latest:access" | jq -r .payload.data | base64 -d; }
+
+SEC=/opt/selfheal/harness/secrets
+mkdir -p "$SEC"; chmod 700 /opt/selfheal/harness "$SEC"
+umask 077
+secret selfheal-k8s-harness-token   > "$SEC/k8s-token"
+secret selfheal-k8s-ca-cert         > "$SEC/k8s-ca.crt"
+secret selfheal-vm-ssh-private-key  > "$SEC/vm-ssh-key"
+secret selfheal-db-password         > "$SEC/db-password"
+secret selfheal-harness-token       > "$SEC/harness-token"
+unset TOKEN
+for f in k8s-token k8s-ca.crt vm-ssh-key db-password harness-token; do [ -s "$SEC/$f" ] || { echo "empty secret file $f" >&2; exit 1; }; done
+chown -R 10001:10001 "$SEC"; chmod 600 "$SEC"/*   # runner runs as uid 10001
+
+ENV_FILE=/opt/selfheal/harness.env
+if [ ! -f "$ENV_FILE" ] || ! grep -q '^MEASURE_DB_PASSWORD=' "$ENV_FILE"; then
+  MDB_PW="$(head -c 24 /dev/urandom | base64 | tr -d '/+=')"
+else
+  MDB_PW="$(sed -n 's/^MEASURE_DB_PASSWORD=//p' "$ENV_FILE")"
+fi
+HARNESS_TAG="$(cat image.tag)"
+{
+  printf 'ALB_DNS=%s\n' "$ALB_DNS"
+  printf 'AWS_ROLE_ARN=%s\n' "$AWS_ROLE_ARN"
+  printf 'AWS_REGION=ap-northeast-1\n'
+  printf 'GCP_PROJECT=%s\n' "$PROJECT"
+  printf 'GCP_REGION=asia-northeast1\n'
+  printf 'LEGACY_MIG=selfheal-legacy\n'
+  printf 'K8S_API=https://10.10.0.10:6443\n'
+  printf 'MIMIR_URL=http://10.20.0.10:9009/prometheus\n'
+  printf 'AI_DIAGNOSE_URL=http://10.20.0.12:8080/diagnose\n'
+  printf 'HARNESS_TAG=%s\n' "$HARNESS_TAG"
+  printf 'MEASURE_DB_PASSWORD=%s\n' "$MDB_PW"
+} > "$ENV_FILE"
+unset MDB_PW
+
+docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans
+# runner re-reads secret files per call; recreate it anyway so a new image tag / env takes effect
+docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --force-recreate --no-deps runner
+for i in $(seq 1 30); do
+  h=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/healthz); b=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9115/-/healthy)
+  a=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:12345/-/ready)
+  if [ "$h$b$a" = "200200200" ]; then break; fi; sleep 5
+done
+echo "ready: runner=$h blackbox=$b alloy=$a"
+docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}'

vm/harness/blackbox.yaml

@@ -0,0 +1,14 @@
+# blackbox exporter modules (design 4.1): http_2xx for the ALB path, tcp_connect for db-primary:5432
+modules:
+  http_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      preferred_ip_protocol: ip4
+      valid_status_codes: [200]
+      method: GET
+  tcp_connect:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      preferred_ip_protocol: ip4

vm/harness/compose.yaml

@@ -0,0 +1,66 @@
+# Harness VM (10.20.0.11), experiment design 5.1: runner 8080, measure-db (internal), blackbox 9115, k6 (campaign profile),
+# plus Alloy that scrapes blackbox/runner into Mimir (design 4.1 synthetic probe -> probe_success).
+services:
+  runner:
+    image: asia-northeast1-docker.pkg.dev/cms-d-414903/selfheal-public/harness:${HARNESS_TAG}
+    container_name: runner
+    restart: always
+    env_file: /opt/selfheal/harness.env
+    environment:
+      MEASURE_DSN: "postgres://harness:${MEASURE_DB_PASSWORD}@measure-db:5432/harness?sslmode=disable"
+      SCENARIO_DIR: /etc/harness/scenarios
+    volumes:
+      - /opt/selfheal/harness/secrets:/run/secrets:ro
+      - ./scenarios:/etc/harness/scenarios:ro
+    ports:
+      - "8080:8080"
+    depends_on: [measure-db]
+  measure-db:
+    image: postgres:16.15-alpine
+    container_name: measure-db
+    restart: always
+    environment:
+      POSTGRES_USER: harness
+      POSTGRES_PASSWORD: "${MEASURE_DB_PASSWORD}"
+      POSTGRES_DB: harness
+    volumes:
+      - measure-data:/var/lib/postgresql/data
+  blackbox:
+    image: prom/blackbox-exporter:v0.28.0
+    container_name: blackbox
+    restart: always
+    command: ["--config.file=/etc/blackbox.yaml"]
+    volumes:
+      - ./blackbox.yaml:/etc/blackbox.yaml:ro
+    ports:
+      - "9115:9115"
+  alloy:
+    image: grafana/alloy:v1.19.2
+    container_name: alloy
+    restart: always
+    command: ["run", "--server.http.listen-addr=0.0.0.0:12345", "--storage.path=/var/lib/alloy/data", "/etc/alloy/config.alloy"]
+    environment:
+      ALB_DNS: "${ALB_DNS}"
+    volumes:
+      - ./config.alloy:/etc/alloy/config.alloy:ro
+      - alloy-data:/var/lib/alloy/data
+    ports:
+      - "12345:12345"
+    depends_on: [blackbox]
+  k6:
+    # background load of design 5.5; started only for campaigns: docker compose --profile campaign up -d k6
+    image: grafana/k6:2.3.0
+    container_name: k6
+    profiles: [campaign]
+    restart: "no"
+    command: ["run", "--summary-export=/logs/summary.json", "/scripts/load.js"]
+    environment:
+      ALB_URL: "http://${ALB_DNS}"
+      K6_DURATION: "${K6_DURATION:-12h}"
+    volumes:
+      - ./k6:/scripts:ro
+      - k6-logs:/logs
+volumes:
+  measure-data: {}
+  alloy-data: {}
+  k6-logs: {}

vm/harness/config.alloy

@@ -0,0 +1,33 @@
+// Alloy on the harness VM: scrape blackbox (ALB probe 15s, db-primary tcp) and the runner, remote_write to Mimir.
+// Single tenant Mimir: no X-Scope-OrgID (design 4.1).
+prometheus.scrape "blackbox" {
+  job_name        = "blackbox"
+  metrics_path    = "/probe"
+  scrape_interval = "15s"
+  scrape_timeout  = "12s"
+  targets = [
+    { "__address__" = "blackbox:9115", "__param_module" = "http_2xx",    "__param_target" = "http://" + sys.env("ALB_DNS") + "/api/orders", "target" = "alb" },
+    { "__address__" = "blackbox:9115", "__param_module" = "tcp_connect", "__param_target" = "10.20.0.20:5432",                            "target" = "db-primary" },
+  ]
+  forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+prometheus.scrape "harness" {
+  job_name        = "harness"
+  scrape_interval = "15s"
+  targets         = [{ "__address__" = "runner:8080", "instance" = "harness" }]
+  forward_to      = [prometheus.remote_write.mimir.receiver]
+}
+
+prometheus.scrape "blackbox_self" {
+  job_name        = "blackbox-exporter"
+  scrape_interval = "60s"
+  targets         = [{ "__address__" = "blackbox:9115", "instance" = "harness" }]
+  forward_to      = [prometheus.remote_write.mimir.receiver]
+}
+
+prometheus.remote_write "mimir" {
+  endpoint {
+    url = "http://10.20.0.10:9009/api/v1/push"
+  }
+}

vm/harness/image.tag

@@ -0,0 +1 @@
+de076a7

vm/harness/k6/load.js

@@ -0,0 +1,37 @@
+// Background load of experiment design 5.5: constant-arrival-rate 20 rps, /api/orders 70% and /api/price/{item} 30%.
+// Thresholds mirror the SLO (p95 < 500ms, failures < 1%); a breach makes k6 exit non-zero at the end of the run
+// (the SLO verdict itself is the harness judge's Mimir query, design 4.2).
+import http from 'k6/http';
+import { check } from 'k6';
+
+const base = __ENV.ALB_URL;
+const items = ['apple', 'banana', 'cherry'];
+
+export const options = {
+  scenarios: {
+    background: {
+      executor: 'constant-arrival-rate',
+      rate: 20,
+      timeUnit: '1s',
+      duration: __ENV.K6_DURATION || '12h',
+      preAllocatedVUs: 20,
+      maxVUs: 50,
+    },
+  },
+  thresholds: {
+    http_req_duration: ['p(95)<500'],
+    http_req_failed: ['rate<0.01'],
+  },
+};
+
+export default function () {
+  const r = Math.random();
+  let res;
+  if (r < 0.7) {
+    res = http.get(`${base}/api/orders`, { tags: { route: 'orders' } });
+  } else {
+    const item = items[Math.floor(Math.random() * items.length)];
+    res = http.get(`${base}/api/price/${item}`, { tags: { route: 'price' } });
+  }
+  check(res, { 'status 200': (x) => x.status === 200 });
+}

vm/harness/scenarios/C-G1.yaml

@@ -0,0 +1,32 @@
+# design 7 C-G1: grey zone, container. NetworkChaos delay api -> db-primary; pods stay Ready, p95 breaks. Correct answer: escalate.
+id: C-G1
+path: gray
+asset: container
+expected_layer: 2
+ground_truth_label: network_latency_api_to_db
+ground_truth_action: escalate
+description: NetworkChaos delay 300ms jitter 50ms, direction to, externalTargets 10.20.0.20, app=api
+notes:
+  open_item: "design 11.2: whether the delay applies through the flannel SNAT path is measured in the campaign"
+inject:
+  method: chaos
+  manifest: |
+    apiVersion: chaos-mesh.org/v1alpha1
+    kind: NetworkChaos
+    metadata:
+      namespace: chaos-mesh
+    spec:
+      action: delay
+      mode: all
+      direction: to
+      externalTargets: ["10.20.0.20"]
+      selector:
+        namespaces: [demo]
+        labelSelectors:
+          app: api
+      delay:
+        latency: "300ms"
+        jitter: "50ms"
+      duration: "20m"
+revert:
+  method: chaos_delete

vm/harness/scenarios/C-G2.yaml

@@ -0,0 +1,30 @@
+# design 7 C-G2: grey zone, container. StressChaos memory on api pods; latency rises, intermittent 5xx. Correct: k8s.rollout_restart after approval.
+id: C-G2
+path: gray
+asset: container
+expected_layer: 2
+ground_truth_label: memory_pressure_api
+ground_truth_action: k8s.rollout_restart
+description: StressChaos memory workers 2 size 80%, app=api, duration 10m
+notes:
+  open_item: "design 11.3: whether the stress re-attaches to the restarted pods for the remaining duration"
+inject:
+  method: chaos
+  manifest: |
+    apiVersion: chaos-mesh.org/v1alpha1
+    kind: StressChaos
+    metadata:
+      namespace: chaos-mesh
+    spec:
+      mode: all
+      selector:
+        namespaces: [demo]
+        labelSelectors:
+          app: api
+      stressors:
+        memory:
+          workers: 2
+          size: "80%"
+      duration: "10m"
+revert:
+  method: chaos_delete

vm/harness/scenarios/C-I1.yaml

@@ -0,0 +1,15 @@
+# design 7 C-I1: irreversible, container. Schema change on db-primary (DROP COLUMN status). Layer 2 diagnoses -> escalate -> a human restores (db.data_fix).
+# Runs as the postgres OS user over SSH (the harness DB role has SELECT only). Revert: the human re-adds the column; recorded via /actions/execute db.data_fix.
+id: C-I1
+path: irreversible
+asset: container
+expected_layer: 3
+ground_truth_label: schema_mismatch
+ground_truth_action: escalate
+description: ALTER TABLE app.orders DROP COLUMN status on db-primary
+inject:
+  method: ssh
+  host: db-primary
+  command: sudo -u postgres psql -d app -v ON_ERROR_STOP=1 -c "ALTER TABLE app.orders DROP COLUMN status"
+revert:
+  method: none

vm/harness/scenarios/C-K1.yaml

@@ -0,0 +1,24 @@
+# design 7 C-K1: known signature, container. PodChaos pod-kill one api pod; ReplicaSet recreates it, PDB keeps the SLO.
+id: C-K1
+path: known
+asset: container
+expected_layer: 1
+ground_truth_label: api_pod_killed
+ground_truth_action: none
+description: PodChaos pod-kill, mode one, selector app=api
+inject:
+  method: chaos
+  manifest: |
+    apiVersion: chaos-mesh.org/v1alpha1
+    kind: PodChaos
+    metadata:
+      namespace: chaos-mesh
+    spec:
+      action: pod-kill
+      mode: one
+      selector:
+        namespaces: [demo]
+        labelSelectors:
+          app: api
+revert:
+  method: chaos_delete

vm/harness/scenarios/C-K2.yaml

@@ -0,0 +1,26 @@
+# design 7 C-K2: known signature, container. PodChaos pod-failure 60s on one api pod -> liveness fails -> kubelet restarts.
+id: C-K2
+path: known
+asset: container
+expected_layer: 1
+ground_truth_label: api_pod_failure
+ground_truth_action: none
+description: PodChaos pod-failure 60s, app=api
+escalate_after_s: 300
+inject:
+  method: chaos
+  manifest: |
+    apiVersion: chaos-mesh.org/v1alpha1
+    kind: PodChaos
+    metadata:
+      namespace: chaos-mesh
+    spec:
+      action: pod-failure
+      mode: one
+      duration: "60s"
+      selector:
+        namespaces: [demo]
+        labelSelectors:
+          app: api
+revert:
+  method: chaos_delete

vm/harness/scenarios/C-K3.yaml

@@ -0,0 +1,15 @@
+# design 7 C-K3: known signature, container. Out-of-Git change (scale api to 1); Argo CD selfHeal restores replicas 3.
+id: C-K3
+path: known
+asset: container
+expected_layer: 1
+ground_truth_label: gitops_drift
+ground_truth_action: none
+description: harness scales deployment/api to 1 outside Git
+inject:
+  method: kubectl_scale
+  namespace: demo
+  deployment: api
+  replicas: 1
+revert:
+  method: none

vm/harness/scenarios/V-G1.yaml

@@ -0,0 +1,16 @@
+# design 7 V-G1: grey zone, VM. tc netem delay 400ms on one legacy instance; health check passes, /api/price p95 breaks. Correct: mig.recreate_instance after approval.
+id: V-G1
+path: gray
+asset: vm
+expected_layer: 2
+ground_truth_label: network_latency_legacy
+ground_truth_action: mig.recreate_instance
+description: tc qdisc add dev ens4 root netem delay 400ms on legacy instance 0
+inject:
+  method: ssh
+  host: legacy:0
+  command: sudo tc qdisc add dev ens4 root netem delay 400ms
+revert:
+  method: ssh
+  host: legacy:0
+  command: sudo tc qdisc del dev ens4 root || true

vm/harness/scenarios/V-G2.yaml

@@ -0,0 +1,18 @@
+# design 7 V-G2: grey zone, VM. Pause WAL replay on db-replica; no SLO impact, only DbReplicationLag fires. Correct: db.replay_resume after approval.
+id: V-G2
+path: gray
+asset: vm
+expected_layer: 2
+ground_truth_label: replication_paused
+ground_truth_action: db.replay_resume
+description: SELECT pg_wal_replay_pause() on db-replica (harness role)
+inject:
+  method: psql
+  host: db-replica
+  database: postgres
+  sql: SELECT pg_wal_replay_pause()
+revert:
+  method: psql
+  host: db-replica
+  database: postgres
+  sql: SELECT pg_wal_replay_resume()

vm/harness/scenarios/V-I1.yaml

@@ -0,0 +1,15 @@
+# design 7 V-I1: irreversible, VM. Stop db-primary. Layer 2 diagnoses -> escalate -> a human promotes the replica and switches the DSN (db.promote).
+# The old primary is not started again (STONITH); revert is none by design. Recovery of the lab afterwards: Terraform / restore.
+id: V-I1
+path: irreversible
+asset: vm
+expected_layer: 3
+ground_truth_label: db_primary_down
+ground_truth_action: escalate
+description: gcloud compute instances stop selfheal-db-primary
+inject:
+  method: gcloud
+  op: instance_stop
+  instance: db-primary
+revert:
+  method: none

vm/harness/scenarios/V-K1.yaml

@@ -0,0 +1,15 @@
+# design 7 V-K1: known signature, VM. docker kill legacy-api on one MIG instance; compose restart:always brings it back before the health check fails.
+id: V-K1
+path: known
+asset: vm
+expected_layer: 1
+ground_truth_label: legacy_process_killed
+ground_truth_action: none
+description: SSH to legacy instance 0, docker kill legacy-api
+escalate_after_s: 600
+inject:
+  method: ssh
+  host: legacy:0
+  command: sudo docker kill legacy-api
+revert:
+  method: none

vm/harness/scenarios/V-K2.yaml

@@ -0,0 +1,15 @@
+# design 7 V-K2: known signature, VM. Delete one legacy instance; MIG autohealing recreates it (initial_delay 300s), ILB serves from the other.
+id: V-K2
+path: known
+asset: vm
+expected_layer: 1
+ground_truth_label: legacy_instance_lost
+ground_truth_action: none
+description: gcloud compute instances delete legacy instance 0
+escalate_after_s: 600
+inject:
+  method: gcloud
+  op: instance_delete
+  instance: legacy:0
+revert:
+  method: none

vm/harness/scenarios/X-M1.yaml

@@ -0,0 +1,28 @@
+# design 7 X-M1: misjudge probe, container. 25s partition api <-> db, just under the liveness failure window (3 x 10s);
+# observe whether kubelet restarts containers needlessly.
+id: X-M1
+path: misjudge
+asset: container
+expected_layer: 1
+ground_truth_label: transient_partition
+ground_truth_action: none
+description: NetworkChaos partition 25s, api -> 10.20.0.20
+inject:
+  method: chaos
+  manifest: |
+    apiVersion: chaos-mesh.org/v1alpha1
+    kind: NetworkChaos
+    metadata:
+      namespace: chaos-mesh
+    spec:
+      action: partition
+      mode: all
+      direction: to
+      externalTargets: ["10.20.0.20"]
+      selector:
+        namespaces: [demo]
+        labelSelectors:
+          app: api
+      duration: "25s"
+revert:
+  method: chaos_delete

vm/harness/scenarios/X-M2.yaml

@@ -0,0 +1,17 @@
+# design 7 X-M2: misjudge probe, VM. 50% packet loss for 90s on one legacy instance; observe whether MIG autohealing recreates it for a transient fault.
+id: X-M2
+path: misjudge
+asset: vm
+expected_layer: 1
+ground_truth_label: transient_loss_legacy
+ground_truth_action: none
+description: tc netem loss 50% for 90s on legacy instance 0
+inject:
+  method: ssh
+  host: legacy:0
+  command: sudo tc qdisc add dev ens4 root netem loss 50%
+  wait_s: 90
+revert:
+  method: ssh
+  host: legacy:0
+  command: sudo tc qdisc del dev ens4 root || true

workloads/rbac/harness.yaml

@@ -17,6 +17,9 @@ rules:
   - apiGroups: ["apps"]
     resources: ["deployments", "deployments/scale"]
     verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["apps"]
+    resources: ["replicasets"] # rollout undo reads the previous revision's pod template
+    verbs: ["get", "list"]
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "watch"]