feat: harness VM stack (runner, measure-db, blackbox, Alloy, k6) and the 12 scenarios

Experiment design 5 and 7 for the harness VM (10.20.0.11): vm/harness/apply.sh pulls the secrets from Secret Manager, compose runs runner:de076a7, measure-db, blackbox, Alloy (probe_success -> Mimir) and k6 under the campaign profile. Scenario files C-K1..X-M2 are loaded by the runner. Harness ClusterRole gains replicasets get/list for rollout undo.
2026-09-24 02:27:41 +09:00
1 parent de52d4a1ec
commit af37d19e2a
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Harness VM (10.20.0.11). Idempotent: first boot (startup script) and `make vm-sync`.
+# Secrets come from Secret Manager through the VM service account (no key files, Terraform design 6.11):
+#   selfheal-k8s-harness-token, selfheal-k8s-ca-cert, selfheal-vm-ssh-private-key, selfheal-db-password, selfheal-harness-token
+# Runner inputs (ALB DNS, AWS role ARN) come from instance metadata set by Terraform (20-compute).
+set -euo pipefail
+cd "$(dirname "$0")"
+MD=http://metadata.google.internal/computeMetadata/v1
+md() { curl -sf -H 'Metadata-Flavor: Google' "$MD/$1"; }
+PROJECT="$(md project/project-id)"
+ALB_DNS="$(md instance/attributes/alb-dns)"
+AWS_ROLE_ARN="$(md instance/attributes/aws-role-arn)"
+command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; }
+TOKEN="$(md instance/service-accounts/default/token | jq -r .access_token)"
+secret() { curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/$1/versions/latest:access" | jq -r .payload.data | base64 -d; }
+
+SEC=/opt/selfheal/harness/secrets
+mkdir -p "$SEC"; chmod 700 /opt/selfheal/harness "$SEC"
+umask 077
+secret selfheal-k8s-harness-token   > "$SEC/k8s-token"
+secret selfheal-k8s-ca-cert         > "$SEC/k8s-ca.crt"
+secret selfheal-vm-ssh-private-key  > "$SEC/vm-ssh-key"
+secret selfheal-db-password         > "$SEC/db-password"
+secret selfheal-harness-token       > "$SEC/harness-token"
+unset TOKEN
+for f in k8s-token k8s-ca.crt vm-ssh-key db-password harness-token; do [ -s "$SEC/$f" ] || { echo "empty secret file $f" >&2; exit 1; }; done
+chown -R 10001:10001 "$SEC"; chmod 600 "$SEC"/*   # runner runs as uid 10001
+
+ENV_FILE=/opt/selfheal/harness.env
+if [ ! -f "$ENV_FILE" ] || ! grep -q '^MEASURE_DB_PASSWORD=' "$ENV_FILE"; then
+  MDB_PW="$(head -c 24 /dev/urandom | base64 | tr -d '/+=')"
+else
+  MDB_PW="$(sed -n 's/^MEASURE_DB_PASSWORD=//p' "$ENV_FILE")"
+fi
+HARNESS_TAG="$(cat image.tag)"
+{
+  printf 'ALB_DNS=%s\n' "$ALB_DNS"
+  printf 'AWS_ROLE_ARN=%s\n' "$AWS_ROLE_ARN"
+  printf 'AWS_REGION=ap-northeast-1\n'
+  printf 'GCP_PROJECT=%s\n' "$PROJECT"
+  printf 'GCP_REGION=asia-northeast1\n'
+  printf 'LEGACY_MIG=selfheal-legacy\n'
+  printf 'K8S_API=https://10.10.0.10:6443\n'
+  printf 'MIMIR_URL=http://10.20.0.10:9009/prometheus\n'
+  printf 'AI_DIAGNOSE_URL=http://10.20.0.12:8080/diagnose\n'
+  printf 'HARNESS_TAG=%s\n' "$HARNESS_TAG"
+  printf 'MEASURE_DB_PASSWORD=%s\n' "$MDB_PW"
+} > "$ENV_FILE"
+unset MDB_PW
+
+docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans
+# runner re-reads secret files per call; recreate it anyway so a new image tag / env takes effect
+docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --force-recreate --no-deps runner
+for i in $(seq 1 30); do
+  h=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/healthz); b=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9115/-/healthy)
+  a=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:12345/-/ready)
+  if [ "$h$b$a" = "200200200" ]; then break; fi; sleep 5
+done
+echo "ready: runner=$h blackbox=$b alloy=$a"
+docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}'