# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
apiVersion: 1
groups:
- orgId: 1
  name: selfheal
  folder: selfheal
  interval: 30s
  rules:
  - uid: probe-down
    title: ProbeDown
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: ALB synthetic probe failing (symptom alert, cause unknown)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: api-latency-high
    title: ApiLatencyHigh
    condition: C
    for: 2m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: api p95 over the last minute above 500 ms for 2 minutes
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[1m]))) > bool 0.5
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: api-error-rate-high
    title: ApiErrorRateHigh
    condition: C
    for: 3m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: demo api 5xx ratio above 1% (grey zone)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m]))) > bool 0.01
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: pod-crashlooping
    title: PodCrashLooping
    condition: C
    for: 0s
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: pod_crashloop
    annotations:
      summary: container restarted 3+ times in 5m (kubelet is handling; escalate after 5m)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m])) >= bool 3
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: legacy-unhealthy
    title: LegacyUnhealthy
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: legacy_unhealthy
    annotations:
      summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling; escalate after 10m)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: count(up{job="legacy-api"} == 1) < bool 2
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: db-replication-lag
    title: DbReplicationLag
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: clamp_max((max(pg_replication_lag_seconds) > bool 60) + (max(pg_is_wal_replay_paused) == bool 1), 1)
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: db-primary-down
    title: DbPrimaryDown
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: gitops-drift
    title: GitOpsDrift
    condition: C
    for: 5m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '1'
      signature: gitops_drift
    annotations:
      summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0
  - uid: api-memory-pressure
    title: ApiMemoryPressure
    condition: C
    for: 1m
    noDataState: OK
    execErrState: Error
    labels:
      layer: '2'
    annotations:
      summary: an api container holds more than 80% of its memory limit for 1 minute (grey zone; rollout restart after approval)
    data:
    - refId: A
      relativeTimeRange:
        from: 600
        to: 0
      datasourceUid: mimir
      model:
        refId: A
        instant: true
        expr: max by (pod) (container_memory_working_set_bytes{namespace="demo",container="api"}) / on (pod) max by (pod) (kube_pod_container_resource_limits{namespace="demo",container="api",resource="memory"})
          > bool 0.8
    - refId: C
      datasourceUid: __expr__
      model:
        refId: C
        type: threshold
        expression: A
        conditions:
        - evaluator:
            type: gt
            params:
            - 0