diff --git a/apps/alloy.yaml b/apps/alloy.yaml index 484d711..3b2d845 100644 --- a/apps/alloy.yaml +++ b/apps/alloy.yaml @@ -39,15 +39,48 @@ spec: discovery.kubernetes "pods" { role = "pod" } discovery.relabel "annotated" { targets = discovery.kubernetes.pods.targets - rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"], regex = "true", action = "keep" } - rule { source_labels = ["__meta_kubernetes_pod_phase"], regex = "Running", action = "keep" } - rule { source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"], regex = "(.+)", target_label = "__metrics_path__" } - rule { source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"], regex = "([^:]+)(?::\\d+)?;(\\d+)", replacement = "$1:$2", target_label = "__address__" } - rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" } - rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" } - rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" } - rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" } - rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Running" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"] + regex = "(.+)" + target_label = "__metrics_path__" + } + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"] + regex = "([^:]+)(?::\\d+)?;(\\d+)" + replacement = "$1:$2" + target_label = "__address__" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"] + separator = "/" + target_label = "job" + } } prometheus.scrape "pods" { targets = discovery.relabel.annotated.output @@ -74,9 +107,20 @@ spec: discovery.kubernetes "nodes" { role = "node" } discovery.relabel "cadvisor" { targets = discovery.kubernetes.nodes.targets - rule { target_label = "__address__", replacement = "kubernetes.default.svc:443" } - rule { source_labels = ["__meta_kubernetes_node_name"], regex = "(.+)", target_label = "__metrics_path__", replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" } - rule { source_labels = ["__meta_kubernetes_node_name"], target_label = "node" } + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc:443" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "__metrics_path__" + replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node" + } } prometheus.scrape "cadvisor" { targets = discovery.relabel.cadvisor.output @@ -89,20 +133,52 @@ spec: } prometheus.relabel "cadvisor_keep" { forward_to = [prometheus.remote_write.mimir.receiver] - rule { source_labels = ["__name__"], regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)", action = "keep" } - rule { source_labels = ["namespace"], regex = "demo|ingress-nginx|argocd", action = "keep" } - rule { source_labels = ["container"], regex = "POD|", action = "drop" } + rule { + source_labels = ["__name__"] + regex = "container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)" + action = "keep" + } + rule { + source_labels = ["namespace"] + regex = "demo|ingress-nginx|argocd" + action = "keep" + } + rule { + source_labels = ["container"] + regex = "POD|" + action = "drop" + } } // ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ---- discovery.relabel "pod_logs" { targets = discovery.kubernetes.pods.targets - rule { source_labels = ["__meta_kubernetes_namespace"], regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring", action = "keep" } - rule { source_labels = ["__meta_kubernetes_namespace"], target_label = "namespace" } - rule { source_labels = ["__meta_kubernetes_pod_name"], target_label = "pod" } - rule { source_labels = ["__meta_kubernetes_pod_container_name"], target_label = "container" } - rule { source_labels = ["__meta_kubernetes_pod_label_app"], target_label = "app" } - rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"], separator = "/", target_label = "job" } + rule { + source_labels = ["__meta_kubernetes_namespace"] + regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"] + separator = "/" + target_label = "job" + } } loki.source.kubernetes "pods" { targets = discovery.relabel.pod_logs.output diff --git a/vm/ai-env/compose.yaml b/vm/ai-env/compose.yaml index d597930..3bbe875 100644 --- a/vm/ai-env/compose.yaml +++ b/vm/ai-env/compose.yaml @@ -32,7 +32,7 @@ services: volumes: - /opt/selfheal/ai/secrets:/run/secrets:ro ports: - - "8081:8081" + - "127.0.0.1:8081:8081" # agent reaches hitl on the compose network; host binding only for the local selfcheck depends_on: [rag-db] rag-db: # pgvector 0.8.7 on PostgreSQL 16 (Docker Hub tag checked 2026-10-08); HNSW index up to 2000 dimensions (design 6.4) diff --git a/vm/harness/image.tag b/vm/harness/image.tag index ca1c06f..098f02e 100644 --- a/vm/harness/image.tag +++ b/vm/harness/image.tag @@ -1 +1 @@ -de076a7 +f30af2a diff --git a/vm/legacy-api/config.alloy b/vm/legacy-api/config.alloy index d4ea41d..b50781d 100644 --- a/vm/legacy-api/config.alloy +++ b/vm/legacy-api/config.alloy @@ -17,9 +17,19 @@ discovery.docker "local" { discovery.relabel "containers" { targets = discovery.docker.local.targets - rule { source_labels = ["__meta_docker_container_name"], regex = "/(.*)", target_label = "container" } - rule { target_label = "job", replacement = "legacy-api" } - rule { target_label = "instance", replacement = sys.env("INSTANCE") } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + rule { + target_label = "job" + replacement = "legacy-api" + } + rule { + target_label = "instance" + replacement = sys.env("INSTANCE") + } } loki.source.docker "containers" { diff --git a/vm/postgres-common/alloy.sh b/vm/postgres-common/alloy.sh index f686cd0..0722bd3 100755 --- a/vm/postgres-common/alloy.sh +++ b/vm/postgres-common/alloy.sh @@ -11,8 +11,9 @@ if ! command -v alloy >/dev/null; then wget -qO /etc/apt/keyrings/grafana.asc https://apt.grafana.com/gpg-full.key chmod 644 /etc/apt/keyrings/grafana.asc echo "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list - apt-get update -qq - apt-get install -y -qq alloy >/dev/null + # wait for unattended-upgrades instead of failing on the dpkg lock (db-primary 2026-10-08: "Could not get lock /var/lib/dpkg/lock-frontend") + apt-get -o DPkg::Lock::Timeout=600 update -qq + apt-get -o DPkg::Lock::Timeout=600 install -y -qq alloy >/dev/null fi usermod -aG adm alloy # /var/log/postgresql/*.log is postgres:adm 0640 install -d -m 755 /etc/alloy diff --git a/vm/postgres-common/config.alloy b/vm/postgres-common/config.alloy index 5999b98..68fcf77 100644 --- a/vm/postgres-common/config.alloy +++ b/vm/postgres-common/config.alloy @@ -15,7 +15,10 @@ prometheus.scrape "db" { prometheus.relabel "instance" { forward_to = [prometheus.remote_write.mimir.receiver] - rule { target_label = "instance", replacement = sys.env("INSTANCE") } + rule { + target_label = "instance" + replacement = sys.env("INSTANCE") + } } prometheus.remote_write "mimir" { diff --git a/workloads/rbac/ai-reader.yaml b/workloads/rbac/ai-reader.yaml index 5e58281..16fccd3 100644 --- a/workloads/rbac/ai-reader.yaml +++ b/workloads/rbac/ai-reader.yaml @@ -17,3 +17,26 @@ subjects: - kind: ServiceAccount name: ai-reader namespace: demo +--- +# nodes are cluster-scoped and outside ClusterRole view; the agent's get_k8s(nodes) tool needs read access (WP-12 selfcheck 2026-10-08) +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: selfheal-ai-reader-nodes +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: selfheal-ai-reader-nodes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: selfheal-ai-reader-nodes +subjects: + - kind: ServiceAccount + name: ai-reader + namespace: demo