diff --git a/vm/observability/README.md b/vm/observability/README.md deleted file mode 100644 index 45be382..0000000 --- a/vm/observability/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vm/observability - -Placeholder. See the root README for the work package that fills this directory. diff --git a/vm/observability/apply.sh b/vm/observability/apply.sh new file mode 100755 index 0000000..a9a73c7 --- /dev/null +++ b/vm/observability/apply.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Observability VM (LGTM). Idempotent: run at first boot by the startup script and again by `make vm-sync`. +# Grafana admin password comes from Secret Manager (selfheal-grafana-admin) via the VM service account. +set -euo pipefail +cd "$(dirname "$0")" +PROJECT="$(curl -sf -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/project/project-id)" +command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; } +TOKEN="$(curl -sf -H 'Metadata-Flavor: Google' 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' | jq -r .access_token)" +PW="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/selfheal-grafana-admin/versions/latest:access" | jq -r .payload.data | base64 -d)" +unset TOKEN +[ -n "$PW" ] || { echo "empty grafana admin password" >&2; exit 1; } +ENV_FILE=/opt/selfheal/observability.env +umask 077; printf 'GF_SECURITY_ADMIN_PASSWORD=%s\n' "$PW" > "$ENV_FILE"; unset PW +docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans +for i in $(seq 1 30); do + m=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9009/ready); l=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3100/ready) + t=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3200/ready); g=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/api/health) + if [ "$m$l$t$g" = "200200200200" ]; then break; fi; sleep 5 +done +echo "ready: mimir=$m loki=$l tempo=$t grafana=$g" +docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}' diff --git a/vm/observability/compose.yaml b/vm/observability/compose.yaml new file mode 100644 index 0000000..dfb68f9 --- /dev/null +++ b/vm/observability/compose.yaml @@ -0,0 +1,55 @@ +# Observability VM (10.20.0.10): Mimir 9009, Loki 3100, Tempo 3200 (OTLP 4317/4318), Grafana 3000. +# Object storage: the three GCS buckets, authenticated through the VM service account (no key files). +services: + mimir: + image: grafana/mimir:3.2.1 + container_name: mimir + restart: always + command: ["-config.file=/etc/mimir.yaml"] + volumes: + - ./mimir.yaml:/etc/mimir.yaml:ro + - mimir-data:/data + ports: + - "9009:9009" + loki: + image: grafana/loki:3.7.8 + container_name: loki + restart: always + command: ["-config.file=/etc/loki.yaml"] + volumes: + - ./loki.yaml:/etc/loki.yaml:ro + - loki-data:/loki + ports: + - "3100:3100" + tempo: + image: grafana/tempo:3.0.3 + container_name: tempo + restart: always + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./tempo.yaml:/etc/tempo.yaml:ro + - tempo-data:/var/tempo + ports: + - "3200:3200" + - "4317:4317" + - "4318:4318" + grafana: + image: grafana/grafana:13.2.2 + container_name: grafana + restart: always + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: "${GF_SECURITY_ADMIN_PASSWORD}" + GF_SERVER_ROOT_URL: http://10.20.0.10:3000 + GF_ANALYTICS_REPORTING_ENABLED: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + depends_on: [mimir, loki, tempo] +volumes: + mimir-data: {} + loki-data: {} + tempo-data: {} + grafana-data: {} diff --git a/vm/observability/grafana/provisioning/datasources/datasources.yaml b/vm/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..0bb29f0 --- /dev/null +++ b/vm/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,20 @@ +apiVersion: 1 +datasources: + - name: Mimir + uid: mimir + type: prometheus + access: proxy + url: http://mimir:9009/prometheus + isDefault: true + jsonData: + timeInterval: 15s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/vm/observability/loki.yaml b/vm/observability/loki.yaml new file mode 100644 index 0000000..a6aaf3b --- /dev/null +++ b/vm/observability/loki.yaml @@ -0,0 +1,37 @@ +# Loki single binary, TSDB index, GCS object store via the VM service account (ADC). +auth_enabled: false +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + gcs: + bucket_name: selfheal-cms-d-414903-loki +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: gcs + schema: v13 + index: + prefix: index_ + period: 24h +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache +compactor: + working_directory: /loki/compactor +limits_config: + allow_structured_metadata: true +ruler: + storage: + type: local + local: + directory: /loki/rules diff --git a/vm/observability/mimir.yaml b/vm/observability/mimir.yaml new file mode 100644 index 0000000..50b22be --- /dev/null +++ b/vm/observability/mimir.yaml @@ -0,0 +1,47 @@ +# Mimir monolithic (target=all), single node, GCS backend via the VM service account (ADC; no key file). +# One bucket, separate prefixes: blocks must not share bucket+prefix with ruler/alertmanager stores. +target: all +multitenancy_enabled: false +server: + http_listen_port: 9009 + grpc_listen_port: 9095 + log_level: warn +common: + storage: + backend: gcs + gcs: + bucket_name: selfheal-cms-d-414903-mimir +blocks_storage: + storage_prefix: blocks + tsdb: + dir: /data/tsdb + bucket_store: + sync_dir: /data/tsdb-sync +ruler_storage: + storage_prefix: ruler +alertmanager_storage: + storage_prefix: alertmanager +ruler: + rule_path: /data/ruler +alertmanager: + data_dir: /data/alertmanager + external_url: http://10.20.0.10:9009/alertmanager +compactor: + data_dir: /data/compactor + sharding_ring: + kvstore: + store: memberlist +distributor: + ring: + kvstore: + store: memberlist +ingester: + ring: + kvstore: + store: memberlist + replication_factor: 1 +store_gateway: + sharding_ring: + replication_factor: 1 +memberlist: + join_members: [] diff --git a/vm/observability/tempo.yaml b/vm/observability/tempo.yaml new file mode 100644 index 0000000..ff9c55d --- /dev/null +++ b/vm/observability/tempo.yaml @@ -0,0 +1,24 @@ +# Tempo single binary, OTLP gRPC 4317 / HTTP 4318, GCS backend via the VM service account (ADC). +server: + http_listen_port: 3200 + log_level: warn +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +storage: + trace: + backend: gcs + gcs: + bucket_name: selfheal-cms-d-414903-tempo + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +compactor: + compaction: + block_retention: 168h