#!/usr/bin/env bash
# Observability VM (LGTM). Idempotent: run at first boot by the startup script and again by `make vm-sync`.
# Grafana admin password comes from Secret Manager (selfheal-grafana-admin) via the VM service account.
set -euo pipefail
cd "$(dirname "$0")"
PROJECT="$(curl -sf -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/project/project-id)"
command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; }
TOKEN="$(curl -sf -H 'Metadata-Flavor: Google' 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' | jq -r .access_token)"
PW="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/selfheal-grafana-admin/versions/latest:access" | jq -r .payload.data | base64 -d)"
unset TOKEN
[ -n "$PW" ] || { echo "empty grafana admin password" >&2; exit 1; }
ENV_FILE=/opt/selfheal/observability.env
umask 077; printf 'GF_SECURITY_ADMIN_PASSWORD=%s\n' "$PW" > "$ENV_FILE"; unset PW
docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans
# Grafana reads provisioning files only at start-up; recreate it so datasource/alerting changes apply (state lives in the volume)
docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --force-recreate --no-deps grafana
for i in $(seq 1 30); do
  m=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9009/ready); l=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3100/ready)
  t=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3200/ready); g=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/api/health)
  if [ "$m$l$t$g" = "200200200200" ]; then break; fi; sleep 5
done
echo "ready: mimir=$m loki=$l tempo=$t grafana=$g"
docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}'