feat: add ingress-nginx, demo-app, Chaos Mesh apps and Grafana alert rules
WP-14(Terraform 설계서 7.3, 실험·앱 설계서 3.1·4.3). - apps/ingress-nginx: baremetal 매니페스트(controller-v1.15.1)를 kustomize로 가져와 NodePort 30080/30443 고정(ALB TG와 일치). - apps/demo-app + workloads/demo-app: front(nginx-unprivileged 1.31.6, 정적 페이지 + /api/ 프록시) 2, api(demo-api:8f5378b) 3 + PDB minAvailable 2, Service, Ingress(/ front, /api api). DB_DSN은 Secret demo-db(make k8s-secrets로 생성, git 밖). - apps/chaos-mesh: Helm chart 2.8.4(containerd 소켓, dashboard 없음). - vm/observability/grafana/provisioning/alerting: contact point harness-webhook (10.20.0.11:8080/hooks/alert), 루트 정책, 규칙 8건(ProbeDown·ApiLatencyHigh· ApiErrorRateHigh·PodCrashLooping·LegacyUnhealthy·DbReplicationLag·DbPrimaryDown· GitOpsDrift, 라벨 layer/signature).
This commit is contained in:
seon committed
1 parent
907056f4f6
commit
1db68b643f
14 files changed
+511
-12
No files matched your search
@@ -0,0 +1,11 @@
|
||||
# Contact point: the harness runner webhook (experiment design 4.3). Payload is Grafana's standard webhook format.
|
||||
apiVersion: 1
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: harness-webhook
|
||||
receivers:
|
||||
- uid: harness-webhook
|
||||
type: webhook
|
||||
settings:
|
||||
url: http://10.20.0.11:8080/hooks/alert
|
||||
httpMethod: POST
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: harness-webhook
|
||||
group_by: [alertname]
|
||||
group_wait: 10s
|
||||
group_interval: 30s
|
||||
repeat_interval: 1h
|
||||
@@ -0,0 +1,266 @@
|
||||
# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
|
||||
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: selfheal
|
||||
folder: selfheal
|
||||
interval: 30s
|
||||
rules:
|
||||
- uid: probe-down
|
||||
title: ProbeDown
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: ALB synthetic probe failing (symptom alert, cause unknown)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: api-latency-high
|
||||
title: ApiLatencyHigh
|
||||
condition: C
|
||||
for: 3m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: demo api p95 latency above 500ms (grey zone)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))
|
||||
> bool 0.5
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: api-error-rate-high
|
||||
title: ApiErrorRateHigh
|
||||
condition: C
|
||||
for: 3m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: demo api 5xx ratio above 1% (grey zone)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m])))
|
||||
> bool 0.01
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: pod-crashlooping
|
||||
title: PodCrashLooping
|
||||
condition: C
|
||||
for: 0s
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: pod_crashloop
|
||||
annotations:
|
||||
summary: container restarted 3+ times in 5m (kubelet is handling; escalate after
|
||||
5m)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m]))
|
||||
>= bool 3
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: legacy-unhealthy
|
||||
title: LegacyUnhealthy
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: legacy_unhealthy
|
||||
annotations:
|
||||
summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling;
|
||||
escalate after 10m)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (count(up{job="legacy-api"} == 1) or vector(0)) < bool 2
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: db-replication-lag
|
||||
title: DbReplicationLag
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused)
|
||||
== bool 1)
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: db-primary-down
|
||||
title: DbPrimaryDown
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: gitops-drift
|
||||
title: GitOpsDrift
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: gitops_drift
|
||||
annotations:
|
||||
summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
Reference in new issue
Block a user