fix: Alloy block syntax, dpkg lock wait, hitl local bind, ai-reader nodes RBAC, harness image f30af2a
- Alloy 설정 3종: rule 블록을 한 줄 쉼표 나열에서 줄바꿈 속성으로 수정(실측: k8s DaemonSet
CrashLoopBackOff, db-replica alloy.service "expected TERMINATOR, got ,"). alloy fmt로 3종 파싱 확인.
- vm/postgres-common/alloy.sh: apt에 DPkg::Lock::Timeout=600(db-primary에서 unattended-upgrades 잠금으로 100 종료).
- vm/ai-env/compose.yaml: hitl 8081을 127.0.0.1에만 바인딩(agent는 compose 네트워크로 접근).
- workloads/rbac/ai-reader.yaml: nodes get/list/watch ClusterRole 추가(agent get_k8s(nodes) selfcheck 거부 해소).
- vm/harness/image.tag: f30af2a(POST /proposals/{token}/approve·reject 포함 러너).
This commit is contained in:
1 parent
7ce2dc68fe
commit
a500c3d40c
7 files changed
+142
-29
No files matched your search
+97
-21
@@ -39,15 +39,48 @@ spec:
|
||||
discovery.kubernetes "pods" { role = "pod" }
|
||||
discovery.relabel "annotated" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"], regex = "true", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_phase"], regex = "Running", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"], regex = "(.+)", target_label = "__metrics_path__" }
|
||||
rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"], regex = "([^:]+)(?::\\d+)?;(\\d+)", replacement = "$1:$2", target_label = "__address__" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" }
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
|
||||
regex = "true"
|
||||
action = "keep"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_phase"]
|
||||
regex = "Running"
|
||||
action = "keep"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
|
||||
regex = "(.+)"
|
||||
target_label = "__metrics_path__"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"]
|
||||
regex = "([^:]+)(?::\\d+)?;(\\d+)"
|
||||
replacement = "$1:$2"
|
||||
target_label = "__address__"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app"]
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
|
||||
separator = "/"
|
||||
target_label = "job"
|
||||
}
|
||||
}
|
||||
prometheus.scrape "pods" {
|
||||
targets = discovery.relabel.annotated.output
|
||||
@@ -74,9 +107,20 @@ spec:
|
||||
discovery.kubernetes "nodes" { role = "node" }
|
||||
discovery.relabel "cadvisor" {
|
||||
targets = discovery.kubernetes.nodes.targets
|
||||
rule { target_label = "__address__", replacement = "kubernetes.default.svc:443" }
|
||||
rule { source_labels = ["__meta_kubernetes_node_name"], regex = "(.+)", target_label = "__metrics_path__", replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" }
|
||||
rule { source_labels = ["__meta_kubernetes_node_name"], target_label = "node" }
|
||||
rule {
|
||||
target_label = "__address__"
|
||||
replacement = "kubernetes.default.svc:443"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "__metrics_path__"
|
||||
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
}
|
||||
prometheus.scrape "cadvisor" {
|
||||
targets = discovery.relabel.cadvisor.output
|
||||
@@ -89,20 +133,52 @@ spec:
|
||||
}
|
||||
prometheus.relabel "cadvisor_keep" {
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
rule { source_labels = ["__name__"], regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)", action = "keep" }
|
||||
rule { source_labels = ["namespace"], regex = "demo|ingress-nginx|argocd", action = "keep" }
|
||||
rule { source_labels = ["container"], regex = "POD|", action = "drop" }
|
||||
rule {
|
||||
source_labels = ["__name__"]
|
||||
regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)"
|
||||
action = "keep"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["namespace"]
|
||||
regex = "demo|ingress-nginx|argocd"
|
||||
action = "keep"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["container"]
|
||||
regex = "POD|"
|
||||
action = "drop"
|
||||
}
|
||||
}
|
||||
|
||||
// ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ----
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring", action = "keep" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" }
|
||||
rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" }
|
||||
rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" }
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring"
|
||||
action = "keep"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app"]
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
|
||||
separator = "/"
|
||||
target_label = "job"
|
||||
}
|
||||
}
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pod_logs.output
|
||||
|
||||
@@ -32,7 +32,7 @@ services:
|
||||
volumes:
|
||||
- /opt/selfheal/ai/secrets:/run/secrets:ro
|
||||
ports:
|
||||
- "8081:8081"
|
||||
- "127.0.0.1:8081:8081" # agent reaches hitl on the compose network; host binding only for the local selfcheck
|
||||
depends_on: [rag-db]
|
||||
rag-db:
|
||||
# pgvector 0.8.7 on PostgreSQL 16 (Docker Hub tag checked 2026-10-08); HNSW index up to 2000 dimensions (design 6.4)
|
||||
|
||||
@@ -1 +1 @@
|
||||
de076a7
|
||||
f30af2a
|
||||
@@ -17,9 +17,19 @@ discovery.docker "local" {
|
||||
|
||||
discovery.relabel "containers" {
|
||||
targets = discovery.docker.local.targets
|
||||
rule { source_labels = ["__meta_docker_container_name"], regex = "/(.*)", target_label = "container" }
|
||||
rule { target_label = "job", replacement = "legacy-api" }
|
||||
rule { target_label = "instance", replacement = sys.env("INSTANCE") }
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
target_label = "job"
|
||||
replacement = "legacy-api"
|
||||
}
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = sys.env("INSTANCE")
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "containers" {
|
||||
|
||||
@@ -11,8 +11,9 @@ if ! command -v alloy >/dev/null; then
|
||||
wget -qO /etc/apt/keyrings/grafana.asc https://apt.grafana.com/gpg-full.key
|
||||
chmod 644 /etc/apt/keyrings/grafana.asc
|
||||
echo "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq alloy >/dev/null
|
||||
# wait for unattended-upgrades instead of failing on the dpkg lock (db-primary 2026-10-08: "Could not get lock /var/lib/dpkg/lock-frontend")
|
||||
apt-get -o DPkg::Lock::Timeout=600 update -qq
|
||||
apt-get -o DPkg::Lock::Timeout=600 install -y -qq alloy >/dev/null
|
||||
fi
|
||||
usermod -aG adm alloy # /var/log/postgresql/*.log is postgres:adm 0640
|
||||
install -d -m 755 /etc/alloy
|
||||
|
||||
@@ -15,7 +15,10 @@ prometheus.scrape "db" {
|
||||
|
||||
prometheus.relabel "instance" {
|
||||
forward_to = [prometheus.remote_write.mimir.receiver]
|
||||
rule { target_label = "instance", replacement = sys.env("INSTANCE") }
|
||||
rule {
|
||||
target_label = "instance"
|
||||
replacement = sys.env("INSTANCE")
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.remote_write "mimir" {
|
||||
|
||||
@@ -17,3 +17,26 @@ subjects:
|
||||
- kind: ServiceAccount
|
||||
name: ai-reader
|
||||
namespace: demo
|
||||
---
|
||||
# nodes are cluster-scoped and outside ClusterRole view; the agent's get_k8s(nodes) tool needs read access (WP-12 selfcheck 2026-10-08)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: selfheal-ai-reader-nodes
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: selfheal-ai-reader-nodes
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: selfheal-ai-reader-nodes
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: ai-reader
|
||||
namespace: demo
|
||||
Reference in new issue
Block a user