selfheal-lab-gitops/apps/alloy.yaml

# Grafana Alloy DaemonSet (experiment design 4.1, WP-15): pod metrics (prometheus.io/scrape annotations), cadvisor,
# kube-state-metrics, Argo CD metrics -> Mimir; pod logs of demo/argocd/ingress-nginx/chaos-mesh -> Loki.
# Clustering spreads cluster-wide scrape targets and log tailing over the DaemonSet members (one scraper per target).
# Official chart: https://github.com/grafana/alloy/tree/main/operations/helm/charts/alloy (1.13.0 = app v1.20.0, index checked 2026-10-08)
# Components: https://grafana.com/docs/alloy/latest/reference/components/
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: alloy
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://grafana.github.io/helm-charts
    chart: alloy
    targetRevision: 1.13.0
    helm:
      releaseName: alloy
      valuesObject:
        fullnameOverride: alloy
        controller:
          type: daemonset
        alloy:
          clustering:
            enabled: true
          resources:
            requests: { cpu: 50m, memory: 128Mi }
          configMap:
            content: |
              // ---- outputs (single-tenant Mimir / Loki on the observability VM, design 4.1) ----
              prometheus.remote_write "mimir" {
                endpoint { url = "http://10.20.0.10:9009/api/v1/push" }
              }
              loki.write "loki" {
                endpoint { url = "http://10.20.0.10:3100/loki/api/v1/push" }
              }

              // ---- pods with prometheus.io/scrape=true (demo api: demo_http_*) ----
              discovery.kubernetes "pods" { role = "pod" }
              discovery.relabel "annotated" {
                targets = discovery.kubernetes.pods.targets
                rule {
                  source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
                  regex = "true"
                  action = "keep"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_phase"]
                  regex = "Running"
                  action = "keep"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
                  regex = "(.+)"
                  target_label = "__metrics_path__"
                }
                rule {
                  source_labels = ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"]
                  regex = "([^:]+)(?::\\d+)?;(\\d+)"
                  replacement = "$1:$2"
                  target_label = "__address__"
                }
                rule {
                  source_labels = ["__meta_kubernetes_namespace"]
                  target_label = "namespace"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_name"]
                  target_label = "pod"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_label_app"]
                  target_label = "app"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_container_name"]
                  target_label = "container"
                }
                rule {
                  source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
                  separator = "/"
                  target_label = "job"
                }
              }
              prometheus.scrape "pods" {
                targets         = discovery.relabel.annotated.output
                forward_to      = [prometheus.remote_write.mimir.receiver]
                scrape_interval = "15s"
                clustering { enabled = true }
              }

              // ---- cluster-wide services: kube-state-metrics, Argo CD application controller (argocd_app_info) ----
              prometheus.scrape "kube_state_metrics" {
                targets         = [{ "__address__" = "kube-state-metrics.monitoring.svc:8080", "job" = "kube-state-metrics" }]
                forward_to      = [prometheus.remote_write.mimir.receiver]
                scrape_interval = "15s"
                clustering { enabled = true }
              }
              prometheus.scrape "argocd" {
                targets         = [{ "__address__" = "argocd-metrics.argocd.svc:8082", "job" = "argocd" }]
                forward_to      = [prometheus.remote_write.mimir.receiver]
                scrape_interval = "30s"
                clustering { enabled = true }
              }

              // ---- cadvisor straight from each kubelet (container CPU/memory of namespace demo) ----
              // nodes/metrics is in the chart ClusterRole, nodes/proxy is not (checked 2026-10-08), so the kubelet
              // port is scraped directly; the kubeadm kubelet serving certificate is self-signed, hence insecure_skip_verify.
              discovery.kubernetes "nodes" { role = "node" }
              discovery.relabel "cadvisor" {
                targets = discovery.kubernetes.nodes.targets
                rule {
                  target_label = "__metrics_path__"
                  replacement  = "/metrics/cadvisor"
                }
                rule {
                  source_labels = ["__meta_kubernetes_node_name"]
                  target_label  = "node"
                }
                rule {
                  target_label = "job"
                  replacement  = "cadvisor"
                }
              }
              prometheus.scrape "cadvisor" {
                targets           = discovery.relabel.cadvisor.output
                scheme            = "https"
                bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
                tls_config { insecure_skip_verify = true }
                scrape_interval   = "30s"
                forward_to        = [prometheus.relabel.cadvisor_keep.receiver]
                clustering { enabled = true }
              }
              prometheus.relabel "cadvisor_keep" {
                forward_to = [prometheus.remote_write.mimir.receiver]
                rule {
                  source_labels = ["__name__"]
                  regex         = "up|container_(cpu_usage_seconds_total|memory_working_set_bytes|oom_events_total|network_receive_bytes_total|network_transmit_bytes_total)"
                  action        = "keep"
                }
                rule {
                  source_labels = ["__name__", "namespace"]
                  regex         = "up;.*|container_.*;(demo|ingress-nginx|argocd)"
                  action        = "keep"
                }
                rule {
                  source_labels = ["container"]
                  regex         = "POD|"
                  action        = "drop"
                }
              }

              // ---- pod logs -> Loki (labels: namespace, pod, container, app, job=namespace/app) ----
              discovery.relabel "pod_logs" {
                targets = discovery.kubernetes.pods.targets
                rule {
                  source_labels = ["__meta_kubernetes_namespace"]
                  regex = "demo|argocd|ingress-nginx|chaos-mesh|monitoring"
                  action = "keep"
                }
                rule {
                  source_labels = ["__meta_kubernetes_namespace"]
                  target_label = "namespace"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_name"]
                  target_label = "pod"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_container_name"]
                  target_label = "container"
                }
                rule {
                  source_labels = ["__meta_kubernetes_pod_label_app"]
                  target_label = "app"
                }
                rule {
                  source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app"]
                  separator = "/"
                  target_label = "job"
                }
              }
              loki.source.kubernetes "pods" {
                targets    = discovery.relabel.pod_logs.output
                forward_to = [loki.write.loki.receiver]
                clustering { enabled = true }
              }
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - ServerSideApply=true
      - CreateNamespace=true