selfheal-lab-gitops/vm/observability/grafana/provisioning/alerting/rules.yaml

# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
apiVersion: 1
groups:
- orgId: 1
  name: selfheal
  folder: selfheal
  interval: 30s
  rules:
  - uid: probe-down
    title: ProbeDown
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: ALB synthetic probe failing (symptom alert, cause unknown)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: api-latency-high
    title: ApiLatencyHigh
    condition: C
    for: 3m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: demo api p95 latency above 500ms (grey zone)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))
          > bool 0.5
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: api-error-rate-high
    title: ApiErrorRateHigh
    condition: C
    for: 3m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: demo api 5xx ratio above 1% (grey zone)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m])))
          > bool 0.01
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: pod-crashlooping
    title: PodCrashLooping
    condition: C
    for: 0s
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: pod_crashloop
    annotations:
      summary: container restarted 3+ times in 5m (kubelet is handling; escalate after
        5m)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m]))
          >= bool 3
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: legacy-unhealthy
    title: LegacyUnhealthy
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: legacy_unhealthy
    annotations:
      summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling;
        escalate after 10m)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: count(up{job="legacy-api"} == 1) < bool 2
        # no series until WP-15 ships Alloy on the legacy VMs: absent -> NoData -> OK (the `or vector(0)` form fired continuously)
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: db-replication-lag
    title: DbReplicationLag
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused)
          == bool 1)
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: db-primary-down
    title: DbPrimaryDown
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: gitops-drift
    title: GitOpsDrift
    condition: C
    for: 5m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: gitops_drift
    annotations:
      summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0