feat: add ingress-nginx, demo-app, Chaos Mesh apps and Grafana alert rules

WP-14(Terraform 설계서 7.3, 실험·앱 설계서 3.1·4.3).
- apps/ingress-nginx: baremetal 매니페스트(controller-v1.15.1)를 kustomize로 가져와
  NodePort 30080/30443 고정(ALB TG와 일치).
- apps/demo-app + workloads/demo-app: front(nginx-unprivileged 1.31.6, 정적 페이지 +
  /api/ 프록시) 2, api(demo-api:8f5378b) 3 + PDB minAvailable 2, Service, Ingress(/ front,
  /api api). DB_DSN은 Secret demo-db(make k8s-secrets로 생성, git 밖).
- apps/chaos-mesh: Helm chart 2.8.4(containerd 소켓, dashboard 없음).
- vm/observability/grafana/provisioning/alerting: contact point harness-webhook
  (10.20.0.11:8080/hooks/alert), 루트 정책, 규칙 8건(ProbeDown·ApiLatencyHigh·
  ApiErrorRateHigh·PodCrashLooping·LegacyUnhealthy·DbReplicationLag·DbPrimaryDown·
  GitOpsDrift, 라벨 layer/signature).
This commit is contained in:
seon committed 2026-09-24 01:35:45 +09:00
1 parent 907056f4f6
commit 1db68b643f
14 files changed
+511 -12

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
# Chaos Mesh via the official Helm chart (containerd socket). Fault injection for the container layer (experiment design 7).
# https://chaos-mesh.org/docs/production-installation-using-helm/
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: chaos-mesh
namespace: argocd
spec:
project: default
source:
repoURL: https://charts.chaos-mesh.org
chart: chaos-mesh
targetRevision: 2.8.4
helm:
releaseName: chaos-mesh
valuesObject:
chaosDaemon:
runtime: containerd
socketPath: /run/containerd/containerd.sock
dashboard:
create: false
destination:
server: https://kubernetes.default.svc
namespace: chaos-mesh
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true
+21
View File
@@ -0,0 +1,21 @@
# demo-app (front + api) in namespace demo (experiment design 3.1). DB_DSN comes from Secret demo-db (make k8s-secrets).
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: demo-app
namespace: argocd
spec:
project: default
source:
repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git
targetRevision: main
path: workloads/demo-app
destination:
server: https://kubernetes.default.svc
namespace: demo
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true
+22
View File
@@ -0,0 +1,22 @@
# ingress-nginx (baremetal/NodePort 30080). The ALB target group health check hits /healthz on this NodePort.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ingress-nginx
namespace: argocd
spec:
project: default
source:
repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git
targetRevision: main
path: workloads/ingress-nginx
destination:
server: https://kubernetes.default.svc
namespace: ingress-nginx
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true
- CreateNamespace=true
@@ -0,0 +1,11 @@
# Contact point: the harness runner webhook (experiment design 4.3). Payload is Grafana's standard webhook format.
apiVersion: 1
contactPoints:
- orgId: 1
name: harness-webhook
receivers:
- uid: harness-webhook
type: webhook
settings:
url: http://10.20.0.11:8080/hooks/alert
httpMethod: POST
@@ -0,0 +1,8 @@
apiVersion: 1
policies:
- orgId: 1
receiver: harness-webhook
group_by: [alertname]
group_wait: 10s
group_interval: 30s
repeat_interval: 1h
@@ -0,0 +1,266 @@
# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
apiVersion: 1
groups:
- orgId: 1
name: selfheal
folder: selfheal
interval: 30s
rules:
- uid: probe-down
title: ProbeDown
condition: C
for: 1m
noDataState: OK
execErrState: Error
labels:
layer: '2'
annotations:
summary: ALB synthetic probe failing (symptom alert, cause unknown)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: api-latency-high
title: ApiLatencyHigh
condition: C
for: 3m
noDataState: OK
execErrState: Error
labels:
layer: '2'
annotations:
summary: demo api p95 latency above 500ms (grey zone)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))
> bool 0.5
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: api-error-rate-high
title: ApiErrorRateHigh
condition: C
for: 3m
noDataState: OK
execErrState: Error
labels:
layer: '2'
annotations:
summary: demo api 5xx ratio above 1% (grey zone)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m])))
> bool 0.01
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: pod-crashlooping
title: PodCrashLooping
condition: C
for: 0s
noDataState: OK
execErrState: Error
labels:
layer: '1'
signature: pod_crashloop
annotations:
summary: container restarted 3+ times in 5m (kubelet is handling; escalate after
5m)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m]))
>= bool 3
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: legacy-unhealthy
title: LegacyUnhealthy
condition: C
for: 1m
noDataState: OK
execErrState: Error
labels:
layer: '1'
signature: legacy_unhealthy
annotations:
summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling;
escalate after 10m)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: (count(up{job="legacy-api"} == 1) or vector(0)) < bool 2
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: db-replication-lag
title: DbReplicationLag
condition: C
for: 1m
noDataState: OK
execErrState: Error
labels:
layer: '2'
annotations:
summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused)
== bool 1)
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: db-primary-down
title: DbPrimaryDown
condition: C
for: 1m
noDataState: OK
execErrState: Error
labels:
layer: '2'
annotations:
summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
- uid: gitops-drift
title: GitOpsDrift
condition: C
for: 5m
noDataState: OK
execErrState: Error
labels:
layer: '1'
signature: gitops_drift
annotations:
summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: mimir
model:
refId: A
instant: true
expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
- refId: C
datasourceUid: __expr__
model:
refId: C
type: threshold
expression: A
conditions:
- evaluator:
type: gt
params:
- 0
-3
View File
@@ -1,3 +0,0 @@
# workloads/alloy
Placeholder. See the root README for the work package that fills this directory.
-3
View File
@@ -1,3 +0,0 @@
# workloads/chaos-mesh
Placeholder. See the root README for the work package that fills this directory.
-3
View File
@@ -1,3 +0,0 @@
# workloads/demo-app
Placeholder. See the root README for the work package that fills this directory.
+53
View File
@@ -0,0 +1,53 @@
# api: Go service (selfheal-lab-apps/demo-app). 3 replicas, PDB minAvailable 2, probes per experiment design 3.1.
apiVersion: apps/v1
kind: Deployment
metadata:
name: api
namespace: demo
labels: { app: api }
spec:
replicas: 3
selector:
matchLabels: { app: api }
template:
metadata:
labels: { app: api }
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
prometheus.io/path: /metrics
spec:
containers:
- name: api
image: asia-northeast1-docker.pkg.dev/cms-d-414903/selfheal-public/demo-api:8f5378b
ports: [{ name: http, containerPort: 8080 }]
env:
- name: DB_DSN
valueFrom: { secretKeyRef: { name: demo-db, key: DB_DSN } }
- { name: LEGACY_URL, value: "http://10.20.0.100:8080" }
- { name: LEGACY_TIMEOUT, value: "2s" }
livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 }
readinessProbe: { httpGet: { path: /readyz, port: http }, periodSeconds: 5, failureThreshold: 2 }
resources:
requests: { cpu: 100m, memory: 128Mi }
limits: { memory: 256Mi }
securityContext: { allowPrivilegeEscalation: false, runAsNonRoot: true }
---
apiVersion: v1
kind: Service
metadata:
name: api
namespace: demo
spec:
selector: { app: api }
ports: [{ name: http, port: 8080, targetPort: http }]
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: api
namespace: demo
spec:
minAvailable: 2
selector:
matchLabels: { app: api }
+65
View File
@@ -0,0 +1,65 @@
# front: nginx (unprivileged) serving a static page and proxying /api/ to the api Service.
apiVersion: v1
kind: ConfigMap
metadata:
name: front-nginx
namespace: demo
data:
default.conf: |
server {
listen 8080;
server_name _;
location = /healthz { add_header Content-Type text/plain; return 200 'ok'; }
location /api/ {
proxy_pass http://api:8080/api/;
proxy_connect_timeout 2s;
proxy_read_timeout 5s;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / { root /usr/share/nginx/html; index index.html; }
}
index.html: |
<!doctype html><html lang="ja"><meta charset="utf-8"><title>selfheal demo</title>
<body style="font-family:system-ui;margin:2rem"><h1>selfheal demo front</h1>
<p>container layer (AWS k8s) -&gt; VPN -&gt; VM layer (GCP legacy-api, PostgreSQL)</p>
<ul><li><a href="/api/orders">/api/orders</a></li><li><a href="/api/price/apple">/api/price/apple</a></li><li><a href="/api/readyz">/api/readyz</a></li></ul>
</body></html>
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: front
namespace: demo
labels: { app: front }
spec:
replicas: 2
selector:
matchLabels: { app: front }
template:
metadata:
labels: { app: front }
spec:
containers:
- name: nginx
image: nginxinc/nginx-unprivileged:1.31.6-alpine
ports: [{ name: http, containerPort: 8080 }]
volumeMounts:
- { name: conf, mountPath: /etc/nginx/conf.d/default.conf, subPath: default.conf, readOnly: true }
- { name: conf, mountPath: /usr/share/nginx/html/index.html, subPath: index.html, readOnly: true }
livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 }
readinessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 5, failureThreshold: 2 }
resources:
requests: { cpu: 50m, memory: 32Mi }
limits: { memory: 64Mi }
volumes:
- name: conf
configMap: { name: front-nginx }
---
apiVersion: v1
kind: Service
metadata:
name: front
namespace: demo
spec:
selector: { app: front }
ports: [{ name: http, port: 8080, targetPort: http }]
+17
View File
@@ -0,0 +1,17 @@
# Host-agnostic Ingress (reached through the ALB DNS name): / -> front, /api -> api.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: demo
namespace: demo
spec:
ingressClassName: nginx
rules:
- http:
paths:
- path: /api
pathType: Prefix
backend: { service: { name: api, port: { name: http } } }
- path: /
pathType: Prefix
backend: { service: { name: front, port: { name: http } } }
-3
View File
@@ -1,3 +0,0 @@
# workloads/ingress-nginx
Placeholder. See the root README for the work package that fills this directory.
@@ -0,0 +1,18 @@
# ingress-nginx baremetal (NodePort) provider manifest, pinned; NodePorts fixed so the ALB target group (30080) matches.
# https://kubernetes.github.io/ingress-nginx/deploy/baremetal/
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.15.1/deploy/static/provider/baremetal/deploy.yaml
patches:
- target:
kind: Service
name: ingress-nginx-controller
namespace: ingress-nginx
patch: |-
- op: add
path: /spec/ports/0/nodePort
value: 30080
- op: add
path: /spec/ports/1/nodePort
value: 30443