feat: add ingress-nginx, demo-app, Chaos Mesh apps and Grafana alert rules
WP-14(Terraform 설계서 7.3, 실험·앱 설계서 3.1·4.3). - apps/ingress-nginx: baremetal 매니페스트(controller-v1.15.1)를 kustomize로 가져와 NodePort 30080/30443 고정(ALB TG와 일치). - apps/demo-app + workloads/demo-app: front(nginx-unprivileged 1.31.6, 정적 페이지 + /api/ 프록시) 2, api(demo-api:8f5378b) 3 + PDB minAvailable 2, Service, Ingress(/ front, /api api). DB_DSN은 Secret demo-db(make k8s-secrets로 생성, git 밖). - apps/chaos-mesh: Helm chart 2.8.4(containerd 소켓, dashboard 없음). - vm/observability/grafana/provisioning/alerting: contact point harness-webhook (10.20.0.11:8080/hooks/alert), 루트 정책, 규칙 8건(ProbeDown·ApiLatencyHigh· ApiErrorRateHigh·PodCrashLooping·LegacyUnhealthy·DbReplicationLag·DbPrimaryDown· GitOpsDrift, 라벨 layer/signature).
This commit is contained in:
seon committed
1 parent
907056f4f6
commit
1db68b643f
14 files changed
+511
-12
No files matched your search
@@ -0,0 +1,30 @@
|
||||
# Chaos Mesh via the official Helm chart (containerd socket). Fault injection for the container layer (experiment design 7).
|
||||
# https://chaos-mesh.org/docs/production-installation-using-helm/
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: chaos-mesh
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://charts.chaos-mesh.org
|
||||
chart: chaos-mesh
|
||||
targetRevision: 2.8.4
|
||||
helm:
|
||||
releaseName: chaos-mesh
|
||||
valuesObject:
|
||||
chaosDaemon:
|
||||
runtime: containerd
|
||||
socketPath: /run/containerd/containerd.sock
|
||||
dashboard:
|
||||
create: false
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: chaos-mesh
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,21 @@
|
||||
# demo-app (front + api) in namespace demo (experiment design 3.1). DB_DSN comes from Secret demo-db (make k8s-secrets).
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: demo-app
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git
|
||||
targetRevision: main
|
||||
path: workloads/demo-app
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: demo
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,22 @@
|
||||
# ingress-nginx (baremetal/NodePort 30080). The ALB target group health check hits /healthz on this NodePort.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: ingress-nginx
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git
|
||||
targetRevision: main
|
||||
path: workloads/ingress-nginx
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ingress-nginx
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
- CreateNamespace=true
|
||||
@@ -0,0 +1,11 @@
|
||||
# Contact point: the harness runner webhook (experiment design 4.3). Payload is Grafana's standard webhook format.
|
||||
apiVersion: 1
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: harness-webhook
|
||||
receivers:
|
||||
- uid: harness-webhook
|
||||
type: webhook
|
||||
settings:
|
||||
url: http://10.20.0.11:8080/hooks/alert
|
||||
httpMethod: POST
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: harness-webhook
|
||||
group_by: [alertname]
|
||||
group_wait: 10s
|
||||
group_interval: 30s
|
||||
repeat_interval: 1h
|
||||
@@ -0,0 +1,266 @@
|
||||
# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are
|
||||
# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD.
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: selfheal
|
||||
folder: selfheal
|
||||
interval: 30s
|
||||
rules:
|
||||
- uid: probe-down
|
||||
title: ProbeDown
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: ALB synthetic probe failing (symptom alert, cause unknown)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(probe_success{job="blackbox",target="alb"}) == bool 0
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: api-latency-high
|
||||
title: ApiLatencyHigh
|
||||
condition: C
|
||||
for: 3m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: demo api p95 latency above 500ms (grey zone)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m])))
|
||||
> bool 0.5
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: api-error-rate-high
|
||||
title: ApiErrorRateHigh
|
||||
condition: C
|
||||
for: 3m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: demo api 5xx ratio above 1% (grey zone)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m])))
|
||||
> bool 0.01
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: pod-crashlooping
|
||||
title: PodCrashLooping
|
||||
condition: C
|
||||
for: 0s
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: pod_crashloop
|
||||
annotations:
|
||||
summary: container restarted 3+ times in 5m (kubelet is handling; escalate after
|
||||
5m)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m]))
|
||||
>= bool 3
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: legacy-unhealthy
|
||||
title: LegacyUnhealthy
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: legacy_unhealthy
|
||||
annotations:
|
||||
summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling;
|
||||
escalate after 10m)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (count(up{job="legacy-api"} == 1) or vector(0)) < bool 2
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: db-replication-lag
|
||||
title: DbReplicationLag
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused)
|
||||
== bool 1)
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: db-primary-down
|
||||
title: DbPrimaryDown
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '2'
|
||||
annotations:
|
||||
summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
- uid: gitops-drift
|
||||
title: GitOpsDrift
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
layer: '1'
|
||||
signature: gitops_drift
|
||||
annotations:
|
||||
summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded)
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: mimir
|
||||
model:
|
||||
refId: A
|
||||
instant: true
|
||||
expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: gt
|
||||
params:
|
||||
- 0
|
||||
@@ -1,3 +0,0 @@
|
||||
# workloads/alloy
|
||||
|
||||
Placeholder. See the root README for the work package that fills this directory.
|
||||
@@ -1,3 +0,0 @@
|
||||
# workloads/chaos-mesh
|
||||
|
||||
Placeholder. See the root README for the work package that fills this directory.
|
||||
@@ -1,3 +0,0 @@
|
||||
# workloads/demo-app
|
||||
|
||||
Placeholder. See the root README for the work package that fills this directory.
|
||||
@@ -0,0 +1,53 @@
|
||||
# api: Go service (selfheal-lab-apps/demo-app). 3 replicas, PDB minAvailable 2, probes per experiment design 3.1.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: api
|
||||
namespace: demo
|
||||
labels: { app: api }
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels: { app: api }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: api }
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
prometheus.io/path: /metrics
|
||||
spec:
|
||||
containers:
|
||||
- name: api
|
||||
image: asia-northeast1-docker.pkg.dev/cms-d-414903/selfheal-public/demo-api:8f5378b
|
||||
ports: [{ name: http, containerPort: 8080 }]
|
||||
env:
|
||||
- name: DB_DSN
|
||||
valueFrom: { secretKeyRef: { name: demo-db, key: DB_DSN } }
|
||||
- { name: LEGACY_URL, value: "http://10.20.0.100:8080" }
|
||||
- { name: LEGACY_TIMEOUT, value: "2s" }
|
||||
livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 }
|
||||
readinessProbe: { httpGet: { path: /readyz, port: http }, periodSeconds: 5, failureThreshold: 2 }
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 128Mi }
|
||||
limits: { memory: 256Mi }
|
||||
securityContext: { allowPrivilegeEscalation: false, runAsNonRoot: true }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api
|
||||
namespace: demo
|
||||
spec:
|
||||
selector: { app: api }
|
||||
ports: [{ name: http, port: 8080, targetPort: http }]
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: api
|
||||
namespace: demo
|
||||
spec:
|
||||
minAvailable: 2
|
||||
selector:
|
||||
matchLabels: { app: api }
|
||||
@@ -0,0 +1,65 @@
|
||||
# front: nginx (unprivileged) serving a static page and proxying /api/ to the api Service.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: front-nginx
|
||||
namespace: demo
|
||||
data:
|
||||
default.conf: |
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
location = /healthz { add_header Content-Type text/plain; return 200 'ok'; }
|
||||
location /api/ {
|
||||
proxy_pass http://api:8080/api/;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
location / { root /usr/share/nginx/html; index index.html; }
|
||||
}
|
||||
index.html: |
|
||||
<!doctype html><html lang="ja"><meta charset="utf-8"><title>selfheal demo</title>
|
||||
<body style="font-family:system-ui;margin:2rem"><h1>selfheal demo front</h1>
|
||||
<p>container layer (AWS k8s) -> VPN -> VM layer (GCP legacy-api, PostgreSQL)</p>
|
||||
<ul><li><a href="/api/orders">/api/orders</a></li><li><a href="/api/price/apple">/api/price/apple</a></li><li><a href="/api/readyz">/api/readyz</a></li></ul>
|
||||
</body></html>
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: front
|
||||
namespace: demo
|
||||
labels: { app: front }
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels: { app: front }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: front }
|
||||
spec:
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginxinc/nginx-unprivileged:1.31.6-alpine
|
||||
ports: [{ name: http, containerPort: 8080 }]
|
||||
volumeMounts:
|
||||
- { name: conf, mountPath: /etc/nginx/conf.d/default.conf, subPath: default.conf, readOnly: true }
|
||||
- { name: conf, mountPath: /usr/share/nginx/html/index.html, subPath: index.html, readOnly: true }
|
||||
livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 }
|
||||
readinessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 5, failureThreshold: 2 }
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 32Mi }
|
||||
limits: { memory: 64Mi }
|
||||
volumes:
|
||||
- name: conf
|
||||
configMap: { name: front-nginx }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: front
|
||||
namespace: demo
|
||||
spec:
|
||||
selector: { app: front }
|
||||
ports: [{ name: http, port: 8080, targetPort: http }]
|
||||
@@ -0,0 +1,17 @@
|
||||
# Host-agnostic Ingress (reached through the ALB DNS name): / -> front, /api -> api.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: demo
|
||||
namespace: demo
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- http:
|
||||
paths:
|
||||
- path: /api
|
||||
pathType: Prefix
|
||||
backend: { service: { name: api, port: { name: http } } }
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend: { service: { name: front, port: { name: http } } }
|
||||
@@ -1,3 +0,0 @@
|
||||
# workloads/ingress-nginx
|
||||
|
||||
Placeholder. See the root README for the work package that fills this directory.
|
||||
@@ -0,0 +1,18 @@
|
||||
# ingress-nginx baremetal (NodePort) provider manifest, pinned; NodePorts fixed so the ALB target group (30080) matches.
|
||||
# https://kubernetes.github.io/ingress-nginx/deploy/baremetal/
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.15.1/deploy/static/provider/baremetal/deploy.yaml
|
||||
patches:
|
||||
- target:
|
||||
kind: Service
|
||||
name: ingress-nginx-controller
|
||||
namespace: ingress-nginx
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/ports/0/nodePort
|
||||
value: 30080
|
||||
- op: add
|
||||
path: /spec/ports/1/nodePort
|
||||
value: 30443
|
||||
Reference in new issue
Block a user