fix: Alloy block syntax, dpkg lock wait, hitl local bind, ai-reader nodes RBAC, harness image f30af2a

- Alloy 설정 3종: rule 블록을 한 줄 쉼표 나열에서 줄바꿈 속성으로 수정(실측: k8s DaemonSet
  CrashLoopBackOff, db-replica alloy.service "expected TERMINATOR, got ,"). alloy fmt로 3종 파싱 확인.
- vm/postgres-common/alloy.sh: apt에 DPkg::Lock::Timeout=600(db-primary에서 unattended-upgrades 잠금으로 100 종료).
- vm/ai-env/compose.yaml: hitl 8081을 127.0.0.1에만 바인딩(agent는 compose 네트워크로 접근).
- workloads/rbac/ai-reader.yaml: nodes get/list/watch ClusterRole 추가(agent get_k8s(nodes) selfcheck 거부 해소).
- vm/harness/image.tag: f30af2a(POST /proposals/{token}/approve·reject 포함 러너).
This commit is contained in:
seon committed 2026-10-08 01:53:06 +09:00
1 parent 7ce2dc68fe
commit a500c3d40c
7 files changed
+142 -29

No files matched your search

+97 -21
View File
@@ -39,15 +39,48 @@ spec:
discovery.kubernetes "pods" { role = "pod" } discovery.kubernetes "pods" { role = "pod" }
discovery.relabel "annotated" { discovery.relabel "annotated" {
targets = discovery.kubernetes.pods.targets targets = discovery.kubernetes.pods.targets
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"], regex = "true", action = "keep" } rule {
rule { source_labels = ["__meta_kubernetes_pod_phase"], regex = "Running", action = "keep" } source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"], regex = "(.+)", target_label = "__metrics_path__" } regex = "true"
rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"], regex = "([^:]+)(?::\\d+)?;(\\d+)", replacement = "$1:$2", target_label = "__address__" } action = "keep"
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" } }
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" } rule {
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" } source_labels = ["__meta_kubernetes_pod_phase"]
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" } regex = "Running"
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" } action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
regex = "(.+)"
target_label = "__metrics_path__"
}
rule {
source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"]
regex = "([^:]+)(?::\\d+)?;(\\d+)"
replacement = "$1:$2"
target_label = "__address__"
}
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_label_app"]
target_label = "app"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
separator = "/"
target_label = "job"
}
} }
prometheus.scrape "pods" { prometheus.scrape "pods" {
targets = discovery.relabel.annotated.output targets = discovery.relabel.annotated.output
@@ -74,9 +107,20 @@ spec:
discovery.kubernetes "nodes" { role = "node" } discovery.kubernetes "nodes" { role = "node" }
discovery.relabel "cadvisor" { discovery.relabel "cadvisor" {
targets = discovery.kubernetes.nodes.targets targets = discovery.kubernetes.nodes.targets
rule { target_label = "__address__", replacement = "kubernetes.default.svc:443" } rule {
rule { source_labels = ["__meta_kubernetes_node_name"], regex = "(.+)", target_label = "__metrics_path__", replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" } target_label = "__address__"
rule { source_labels = ["__meta_kubernetes_node_name"], target_label = "node" } replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node"
}
} }
prometheus.scrape "cadvisor" { prometheus.scrape "cadvisor" {
targets = discovery.relabel.cadvisor.output targets = discovery.relabel.cadvisor.output
@@ -89,20 +133,52 @@ spec:
} }
prometheus.relabel "cadvisor_keep" { prometheus.relabel "cadvisor_keep" {
forward_to = [prometheus.remote_write.mimir.receiver] forward_to = [prometheus.remote_write.mimir.receiver]
rule { source_labels = ["__name__"], regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)", action = "keep" } rule {
rule { source_labels = ["namespace"], regex = "demo|ingress-nginx|argocd", action = "keep" } source_labels = ["__name__"]
rule { source_labels = ["container"], regex = "POD|", action = "drop" } regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)"
action = "keep"
}
rule {
source_labels = ["namespace"]
regex = "demo|ingress-nginx|argocd"
action = "keep"
}
rule {
source_labels = ["container"]
regex = "POD|"
action = "drop"
}
} }
// ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ---- // ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ----
discovery.relabel "pod_logs" { discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pods.targets targets = discovery.kubernetes.pods.targets
rule { source_labels = ["__meta_kubernetes_namespace"], regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring", action = "keep" } rule {
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" } source_labels = ["__meta_kubernetes_namespace"]
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" } regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring"
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" } action = "keep"
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" } }
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" } rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_pod_label_app"]
target_label = "app"
}
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
separator = "/"
target_label = "job"
}
} }
loki.source.kubernetes "pods" { loki.source.kubernetes "pods" {
targets = discovery.relabel.pod_logs.output targets = discovery.relabel.pod_logs.output
+1 -1
View File
@@ -32,7 +32,7 @@ services:
volumes: volumes:
- /opt/selfheal/ai/secrets:/run/secrets:ro - /opt/selfheal/ai/secrets:/run/secrets:ro
ports: ports:
- "8081:8081" - "127.0.0.1:8081:8081" # agent reaches hitl on the compose network; host binding only for the local selfcheck
depends_on: [rag-db] depends_on: [rag-db]
rag-db: rag-db:
# pgvector 0.8.7 on PostgreSQL 16 (Docker Hub tag checked 2026-10-08); HNSW index up to 2000 dimensions (design 6.4) # pgvector 0.8.7 on PostgreSQL 16 (Docker Hub tag checked 2026-10-08); HNSW index up to 2000 dimensions (design 6.4)
+1 -1
View File
@@ -1 +1 @@
de076a7 f30af2a
+13 -3
View File
@@ -17,9 +17,19 @@ discovery.docker "local" {
discovery.relabel "containers" { discovery.relabel "containers" {
targets = discovery.docker.local.targets targets = discovery.docker.local.targets
rule { source_labels = ["__meta_docker_container_name"], regex = "/(.*)", target_label = "container" } rule {
rule { target_label = "job", replacement = "legacy-api" } source_labels = ["__meta_docker_container_name"]
rule { target_label = "instance", replacement = sys.env("INSTANCE") } regex = "/(.*)"
target_label = "container"
}
rule {
target_label = "job"
replacement = "legacy-api"
}
rule {
target_label = "instance"
replacement = sys.env("INSTANCE")
}
} }
loki.source.docker "containers" { loki.source.docker "containers" {
+3 -2
View File
@@ -11,8 +11,9 @@ if ! command -v alloy >/dev/null; then
wget -qO /etc/apt/keyrings/grafana.asc https://apt.grafana.com/gpg-full.key wget -qO /etc/apt/keyrings/grafana.asc https://apt.grafana.com/gpg-full.key
chmod 644 /etc/apt/keyrings/grafana.asc chmod 644 /etc/apt/keyrings/grafana.asc
echo "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list echo "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list
apt-get update -qq # wait for unattended-upgrades instead of failing on the dpkg lock (db-primary 2026-10-08: "Could not get lock /var/lib/dpkg/lock-frontend")
apt-get install -y -qq alloy >/dev/null apt-get -o DPkg::Lock::Timeout=600 update -qq
apt-get -o DPkg::Lock::Timeout=600 install -y -qq alloy >/dev/null
fi fi
usermod -aG adm alloy # /var/log/postgresql/*.log is postgres:adm 0640 usermod -aG adm alloy # /var/log/postgresql/*.log is postgres:adm 0640
install -d -m 755 /etc/alloy install -d -m 755 /etc/alloy
+4 -1
View File
@@ -15,7 +15,10 @@ prometheus.scrape "db" {
prometheus.relabel "instance" { prometheus.relabel "instance" {
forward_to = [prometheus.remote_write.mimir.receiver] forward_to = [prometheus.remote_write.mimir.receiver]
rule { target_label = "instance", replacement = sys.env("INSTANCE") } rule {
target_label = "instance"
replacement = sys.env("INSTANCE")
}
} }
prometheus.remote_write "mimir" { prometheus.remote_write "mimir" {
+23
View File
@@ -17,3 +17,26 @@ subjects:
- kind: ServiceAccount - kind: ServiceAccount
name: ai-reader name: ai-reader
namespace: demo namespace: demo
---
# nodes are cluster-scoped and outside ClusterRole view; the agent's get_k8s(nodes) tool needs read access (WP-12 selfcheck 2026-10-08)
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: selfheal-ai-reader-nodes
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: selfheal-ai-reader-nodes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: selfheal-ai-reader-nodes
subjects:
- kind: ServiceAccount
name: ai-reader
namespace: demo