feat: AI VM stack (WP-12) and Alloy telemetry on k8s, legacy and DB VMs (WP-15)
WP-12(실험 설계서 6장) vm/ai-env: apply.sh(시크릿 4종을 Secret Manager에서 파일로, 미등록이면
성능 저하 모드로 기동), compose(agent 8080·hitl 8081·rag-db pgvector 0.8.7-pg16), ai.env(Discord
서버 Selfheal Lab 1557424985793101824, approvals 1557425140302872698, escalation 1557425282363691048,
OPENAI_MODEL gpt-6.1-sol 후보), 프롬프트 3종, 러너북 11종(조치별·토폴로지·3층 절차).
WP-15(실험 설계서 4.1·4.3): apps/alloy.yaml(Alloy Helm 1.13.0 DaemonSet, 클러스터링으로 KSM·Argo CD·
cadvisor 스크랩 분산, demo 등 파드 로그 → Loki), apps/kube-state-metrics.yaml(Helm 8.6.0).
legacy-api compose에 Alloy 사이드카(up{job="legacy-api"}, 컨테이너 로그). DB VM은
vm/postgres-common/alloy.sh로 apt 설치 + prometheus.exporter.postgres(pg_replication_lag_seconds)
+ custom query pg_is_wal_replay_paused + PostgreSQL 로그.
vm/harness/apply.sh: 준비 대기 루프의 curl에 || true(러너 재기동 순간 curl 56으로 중단되던 문제,
2026-10-08 harness-sync 실측).
This commit is contained in:
seon committed
1 parent
59995f3601
commit
7ce2dc68fe
31 files changed
+528
-4
No files matched your search
+121
@@ -0,0 +1,121 @@
|
||||
# Grafana Alloy DaemonSet (experiment design 4.1, WP-15): pod metrics (prometheus.io/scrape annotations), cadvisor,
|
||||
# kube-state-metrics, Argo CD metrics -> Mimir; pod logs of demo/argocd/ingress-nginx/chaos-mesh -> Loki.
|
||||
# Clustering spreads cluster-wide scrape targets and log tailing over the DaemonSet members (one scraper per target).
|
||||
# Official chart: https://github.com/grafana/alloy/tree/main/operations/helm/charts/alloy (1.13.0 = app v1.20.0, index checked 2026-10-08)
|
||||
# Components: https://grafana.com/docs/alloy/latest/reference/components/
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://grafana.github.io/helm-charts
|
||||
chart: alloy
|
||||
targetRevision: 1.13.0
|
||||
helm:
|
||||
releaseName: alloy
|
||||
valuesObject:
|
||||
fullnameOverride: alloy
|
||||
controller:
|
||||
type: daemonset
|
||||
alloy:
|
||||
clustering:
|
||||
enabled: true
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
configMap:
|
||||
content: |
|
||||
// ---- outputs (single-tenant Mimir / Loki on the observability VM, design 4.1) ----
|
||||
prometheus.remote_write "mimir" {
|
||||
endpoint { url = "http://10.20.0.10:9009/api/v1/push" }
|
||||
}
|
||||
loki.write "loki" {
|
||||
endpoint { url = "http://10.20.0.10:3100/loki/api/v1/push" }
|
||||
}
|
||||
|
||||
// ---- pods with prometheus.io/scrape=true (demo api: demo_http_*) ----
|
||||
discovery.kubernetes "pods" { role = "pod" }
|
||||
discovery.relabel "annotated" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"], regex = "true", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_phase"], regex = "Running", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"], regex = "(.+)", target_label = "__metrics_path__" }
|
||||
rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"], regex = "([^:]+)(?::\\d+)?;(\\d+)", replacement = "$1:$2", target_label = "__address__" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" }
|
||||
}
|
||||
prometheus.scrape "pods" {
|
||||
targets = discovery.relabel.annotated.output
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
scrape_interval = "15s"
|
||||
clustering { enabled = true }
|
||||
}
|
||||
|
||||
// ---- cluster-wide services: kube-state-metrics, Argo CD application controller (argocd_app_info) ----
|
||||
prometheus.scrape "kube_state_metrics" {
|
||||
targets = [{ "__address__" = "kube-state-metrics.monitoring.svc:8080", "job" = "kube-state-metrics" }]
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
scrape_interval = "15s"
|
||||
clustering { enabled = true }
|
||||
}
|
||||
prometheus.scrape "argocd" {
|
||||
targets = [{ "__address__" = "argocd-metrics.argocd.svc:8082", "job" = "argocd" }]
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
scrape_interval = "30s"
|
||||
clustering { enabled = true }
|
||||
}
|
||||
|
||||
// ---- cadvisor through the API server proxy (container CPU/memory of namespace demo) ----
|
||||
discovery.kubernetes "nodes" { role = "node" }
|
||||
discovery.relabel "cadvisor" {
|
||||
targets = discovery.kubernetes.nodes.targets
|
||||
rule { target_label = "__address__", replacement = "kubernetes.default.svc:443" }
|
||||
rule { source_labels = ["__meta_kubernetes_node_name"], regex = "(.+)", target_label = "__metrics_path__", replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" }
|
||||
rule { source_labels = ["__meta_kubernetes_node_name"], target_label = "node" }
|
||||
}
|
||||
prometheus.scrape "cadvisor" {
|
||||
targets = discovery.relabel.cadvisor.output
|
||||
scheme = "https"
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" }
|
||||
scrape_interval = "30s"
|
||||
forward_to = [prometheus.relabel.cadvisor_keep.receiver]
|
||||
clustering { enabled = true }
|
||||
}
|
||||
prometheus.relabel "cadvisor_keep" {
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
rule { source_labels = ["__name__"], regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)", action = "keep" }
|
||||
rule { source_labels = ["namespace"], regex = "demo|ingress-nginx|argocd", action = "keep" }
|
||||
rule { source_labels = ["container"], regex = "POD|", action = "drop" }
|
||||
}
|
||||
|
||||
// ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ----
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" }
|
||||
}
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pod_logs.output
|
||||
forward_to = [loki.write.loki.receiver]
|
||||
clustering { enabled = true }
|
||||
}
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
- CreateNamespace=true
|
||||
Reference in new issue
Block a user