feat: add LGTM compose for the observability VM

WP-10(Terraform 설계서 6.10, 실험·앱 설계서 4.1). Mimir 3.2.1(monolithic, GCS
prefix blocks/ruler/alertmanager), Loki 3.7.8(tsdb v13, GCS), Tempo 3.0.3(OTLP
4317/4318, GCS), Grafana 13.2.2(데이터소스 Mimir·Loki·Tempo 프로비저닝). 오브젝트
스토리지는 VM SA의 ADC로 인증(키 파일 없음). Grafana 관리자 비밀번호는 Secret Manager
selfheal-grafana-admin에서 실행 시 취득. 알림 규칙 8건은 WP-14에서 추가.

vm/observability/README.md

@@ -1,3 +0,0 @@
-# vm/observability
-
-Placeholder. See the root README for the work package that fills this directory.

vm/observability/apply.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Observability VM (LGTM). Idempotent: run at first boot by the startup script and again by `make vm-sync`.
+# Grafana admin password comes from Secret Manager (selfheal-grafana-admin) via the VM service account.
+set -euo pipefail
+cd "$(dirname "$0")"
+PROJECT="$(curl -sf -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/project/project-id)"
+command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq jq >/dev/null; }
+TOKEN="$(curl -sf -H 'Metadata-Flavor: Google' 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' | jq -r .access_token)"
+PW="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://secretmanager.googleapis.com/v1/projects/${PROJECT}/secrets/selfheal-grafana-admin/versions/latest:access" | jq -r .payload.data | base64 -d)"
+unset TOKEN
+[ -n "$PW" ] || { echo "empty grafana admin password" >&2; exit 1; }
+ENV_FILE=/opt/selfheal/observability.env
+umask 077; printf 'GF_SECURITY_ADMIN_PASSWORD=%s\n' "$PW" > "$ENV_FILE"; unset PW
+docker compose --env-file "$ENV_FILE" -f compose.yaml up -d --pull always --remove-orphans
+for i in $(seq 1 30); do
+  m=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9009/ready); l=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3100/ready)
+  t=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3200/ready); g=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/api/health)
+  if [ "$m$l$t$g" = "200200200200" ]; then break; fi; sleep 5
+done
+echo "ready: mimir=$m loki=$l tempo=$t grafana=$g"
+docker compose --env-file "$ENV_FILE" -f compose.yaml ps --format 'table {{.Name}}\t{{.Status}}'

vm/observability/compose.yaml

@@ -0,0 +1,55 @@
+# Observability VM (10.20.0.10): Mimir 9009, Loki 3100, Tempo 3200 (OTLP 4317/4318), Grafana 3000.
+# Object storage: the three GCS buckets, authenticated through the VM service account (no key files).
+services:
+  mimir:
+    image: grafana/mimir:3.2.1
+    container_name: mimir
+    restart: always
+    command: ["-config.file=/etc/mimir.yaml"]
+    volumes:
+      - ./mimir.yaml:/etc/mimir.yaml:ro
+      - mimir-data:/data
+    ports:
+      - "9009:9009"
+  loki:
+    image: grafana/loki:3.7.8
+    container_name: loki
+    restart: always
+    command: ["-config.file=/etc/loki.yaml"]
+    volumes:
+      - ./loki.yaml:/etc/loki.yaml:ro
+      - loki-data:/loki
+    ports:
+      - "3100:3100"
+  tempo:
+    image: grafana/tempo:3.0.3
+    container_name: tempo
+    restart: always
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "3200:3200"
+      - "4317:4317"
+      - "4318:4318"
+  grafana:
+    image: grafana/grafana:13.2.2
+    container_name: grafana
+    restart: always
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: "${GF_SECURITY_ADMIN_PASSWORD}"
+      GF_SERVER_ROOT_URL: http://10.20.0.10:3000
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana-data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    depends_on: [mimir, loki, tempo]
+volumes:
+  mimir-data: {}
+  loki-data: {}
+  tempo-data: {}
+  grafana-data: {}

vm/observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,20 @@
+apiVersion: 1
+datasources:
+  - name: Mimir
+    uid: mimir
+    type: prometheus
+    access: proxy
+    url: http://mimir:9009/prometheus
+    isDefault: true
+    jsonData:
+      timeInterval: 15s
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+  - name: Tempo
+    uid: tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200

vm/observability/loki.yaml

@@ -0,0 +1,37 @@
+# Loki single binary, TSDB index, GCS object store via the VM service account (ADC).
+auth_enabled: false
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: warn
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  storage:
+    gcs:
+      bucket_name: selfheal-cms-d-414903-loki
+schema_config:
+  configs:
+    - from: "2024-01-01"
+      store: tsdb
+      object_store: gcs
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+compactor:
+  working_directory: /loki/compactor
+limits_config:
+  allow_structured_metadata: true
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules

vm/observability/mimir.yaml

@@ -0,0 +1,47 @@
+# Mimir monolithic (target=all), single node, GCS backend via the VM service account (ADC; no key file).
+# One bucket, separate prefixes: blocks must not share bucket+prefix with ruler/alertmanager stores.
+target: all
+multitenancy_enabled: false
+server:
+  http_listen_port: 9009
+  grpc_listen_port: 9095
+  log_level: warn
+common:
+  storage:
+    backend: gcs
+    gcs:
+      bucket_name: selfheal-cms-d-414903-mimir
+blocks_storage:
+  storage_prefix: blocks
+  tsdb:
+    dir: /data/tsdb
+  bucket_store:
+    sync_dir: /data/tsdb-sync
+ruler_storage:
+  storage_prefix: ruler
+alertmanager_storage:
+  storage_prefix: alertmanager
+ruler:
+  rule_path: /data/ruler
+alertmanager:
+  data_dir: /data/alertmanager
+  external_url: http://10.20.0.10:9009/alertmanager
+compactor:
+  data_dir: /data/compactor
+  sharding_ring:
+    kvstore:
+      store: memberlist
+distributor:
+  ring:
+    kvstore:
+      store: memberlist
+ingester:
+  ring:
+    kvstore:
+      store: memberlist
+    replication_factor: 1
+store_gateway:
+  sharding_ring:
+    replication_factor: 1
+memberlist:
+  join_members: []

vm/observability/tempo.yaml

@@ -0,0 +1,24 @@
+# Tempo single binary, OTLP gRPC 4317 / HTTP 4318, GCS backend via the VM service account (ADC).
+server:
+  http_listen_port: 3200
+  log_level: warn
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+storage:
+  trace:
+    backend: gcs
+    gcs:
+      bucket_name: selfheal-cms-d-414903-tempo
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/blocks
+compactor:
+  compaction:
+    block_retention: 168h