diff --git a/apps/chaos-mesh.yaml b/apps/chaos-mesh.yaml new file mode 100644 index 0000000..115cc9b --- /dev/null +++ b/apps/chaos-mesh.yaml @@ -0,0 +1,30 @@ +# Chaos Mesh via the official Helm chart (containerd socket). Fault injection for the container layer (experiment design 7). +# https://chaos-mesh.org/docs/production-installation-using-helm/ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: chaos-mesh + namespace: argocd +spec: + project: default + source: + repoURL: https://charts.chaos-mesh.org + chart: chaos-mesh + targetRevision: 2.8.4 + helm: + releaseName: chaos-mesh + valuesObject: + chaosDaemon: + runtime: containerd + socketPath: /run/containerd/containerd.sock + dashboard: + create: false + destination: + server: https://kubernetes.default.svc + namespace: chaos-mesh + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/apps/demo-app.yaml b/apps/demo-app.yaml new file mode 100644 index 0000000..1a597af --- /dev/null +++ b/apps/demo-app.yaml @@ -0,0 +1,21 @@ +# demo-app (front + api) in namespace demo (experiment design 3.1). DB_DSN comes from Secret demo-db (make k8s-secrets). +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: demo-app + namespace: argocd +spec: + project: default + source: + repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git + targetRevision: main + path: workloads/demo-app + destination: + server: https://kubernetes.default.svc + namespace: demo + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/apps/ingress-nginx.yaml b/apps/ingress-nginx.yaml new file mode 100644 index 0000000..48f82d7 --- /dev/null +++ b/apps/ingress-nginx.yaml @@ -0,0 +1,22 @@ +# ingress-nginx (baremetal/NodePort 30080). The ALB target group health check hits /healthz on this NodePort. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ingress-nginx + namespace: argocd +spec: + project: default + source: + repoURL: https://git.seonology.com/selfheal-lab/selfheal-lab-gitops.git + targetRevision: main + path: workloads/ingress-nginx + destination: + server: https://kubernetes.default.svc + namespace: ingress-nginx + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true + - CreateNamespace=true diff --git a/vm/observability/grafana/provisioning/alerting/contact-points.yaml b/vm/observability/grafana/provisioning/alerting/contact-points.yaml new file mode 100644 index 0000000..9bdb1f3 --- /dev/null +++ b/vm/observability/grafana/provisioning/alerting/contact-points.yaml @@ -0,0 +1,11 @@ +# Contact point: the harness runner webhook (experiment design 4.3). Payload is Grafana's standard webhook format. +apiVersion: 1 +contactPoints: + - orgId: 1 + name: harness-webhook + receivers: + - uid: harness-webhook + type: webhook + settings: + url: http://10.20.0.11:8080/hooks/alert + httpMethod: POST diff --git a/vm/observability/grafana/provisioning/alerting/policies.yaml b/vm/observability/grafana/provisioning/alerting/policies.yaml new file mode 100644 index 0000000..ab5544d --- /dev/null +++ b/vm/observability/grafana/provisioning/alerting/policies.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +policies: + - orgId: 1 + receiver: harness-webhook + group_by: [alertname] + group_wait: 10s + group_interval: 30s + repeat_interval: 1h diff --git a/vm/observability/grafana/provisioning/alerting/rules.yaml b/vm/observability/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..3fed79b --- /dev/null +++ b/vm/observability/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,266 @@ +# 8 alert rules of experiment design 4.3 (labels layer / signature drive the harness routing). Metric names are +# the ones emitted by demo-app, blackbox (harness), Alloy/kube-state-metrics, postgres exporter and Argo CD. +apiVersion: 1 +groups: +- orgId: 1 + name: selfheal + folder: selfheal + interval: 30s + rules: + - uid: probe-down + title: ProbeDown + condition: C + for: 1m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: ALB synthetic probe failing (symptom alert, cause unknown) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: max(probe_success{job="blackbox",target="alb"}) == bool 0 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: api-latency-high + title: ApiLatencyHigh + condition: C + for: 3m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: demo api p95 latency above 500ms (grey zone) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket{route=~"orders|price"}[5m]))) + > bool 0.5 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: api-error-rate-high + title: ApiErrorRateHigh + condition: C + for: 3m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: demo api 5xx ratio above 1% (grey zone) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: (sum(rate(demo_http_requests_total{status=~"5.."}[5m])) / sum(rate(demo_http_requests_total[5m]))) + > bool 0.01 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: pod-crashlooping + title: PodCrashLooping + condition: C + for: 0s + noDataState: OK + execErrState: Error + labels: + layer: '1' + signature: pod_crashloop + annotations: + summary: container restarted 3+ times in 5m (kubelet is handling; escalate after + 5m) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: max by (pod) (increase(kube_pod_container_status_restarts_total{namespace="demo"}[5m])) + >= bool 3 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: legacy-unhealthy + title: LegacyUnhealthy + condition: C + for: 1m + noDataState: OK + execErrState: Error + labels: + layer: '1' + signature: legacy_unhealthy + annotations: + summary: fewer than 2 healthy legacy-api instances (MIG autohealing is handling; + escalate after 10m) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: (count(up{job="legacy-api"} == 1) or vector(0)) < bool 2 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: db-replication-lag + title: DbReplicationLag + condition: C + for: 1m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: replica lag above 60s or WAL replay paused (grey zone, small SLO impact) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: (max(pg_replication_lag_seconds) > bool 60) or (max(pg_is_wal_replay_paused) + == bool 1) + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: db-primary-down + title: DbPrimaryDown + condition: C + for: 1m + noDataState: OK + execErrState: Error + labels: + layer: '2' + annotations: + summary: db-primary 5432 unreachable (layer 2 diagnosis, layer 3 expected) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: max(probe_success{job="blackbox",target="db-primary"}) == bool 0 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 + - uid: gitops-drift + title: GitOpsDrift + condition: C + for: 5m + noDataState: OK + execErrState: Error + labels: + layer: '1' + signature: gitops_drift + annotations: + summary: Argo CD application OutOfSync for 5m (selfHeal is handling; recorded) + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: mimir + model: + refId: A + instant: true + expr: max(argocd_app_info{sync_status!="Synced"}) == bool 1 + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: + type: gt + params: + - 0 diff --git a/workloads/alloy/README.md b/workloads/alloy/README.md deleted file mode 100644 index 9118888..0000000 --- a/workloads/alloy/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# workloads/alloy - -Placeholder. See the root README for the work package that fills this directory. diff --git a/workloads/chaos-mesh/README.md b/workloads/chaos-mesh/README.md deleted file mode 100644 index e4e8051..0000000 --- a/workloads/chaos-mesh/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# workloads/chaos-mesh - -Placeholder. See the root README for the work package that fills this directory. diff --git a/workloads/demo-app/README.md b/workloads/demo-app/README.md deleted file mode 100644 index 87af39b..0000000 --- a/workloads/demo-app/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# workloads/demo-app - -Placeholder. See the root README for the work package that fills this directory. diff --git a/workloads/demo-app/api.yaml b/workloads/demo-app/api.yaml new file mode 100644 index 0000000..5fdf5e9 --- /dev/null +++ b/workloads/demo-app/api.yaml @@ -0,0 +1,53 @@ +# api: Go service (selfheal-lab-apps/demo-app). 3 replicas, PDB minAvailable 2, probes per experiment design 3.1. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: demo + labels: { app: api } +spec: + replicas: 3 + selector: + matchLabels: { app: api } + template: + metadata: + labels: { app: api } + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: /metrics + spec: + containers: + - name: api + image: asia-northeast1-docker.pkg.dev/cms-d-414903/selfheal-public/demo-api:8f5378b + ports: [{ name: http, containerPort: 8080 }] + env: + - name: DB_DSN + valueFrom: { secretKeyRef: { name: demo-db, key: DB_DSN } } + - { name: LEGACY_URL, value: "http://10.20.0.100:8080" } + - { name: LEGACY_TIMEOUT, value: "2s" } + livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 } + readinessProbe: { httpGet: { path: /readyz, port: http }, periodSeconds: 5, failureThreshold: 2 } + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { memory: 256Mi } + securityContext: { allowPrivilegeEscalation: false, runAsNonRoot: true } +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: demo +spec: + selector: { app: api } + ports: [{ name: http, port: 8080, targetPort: http }] +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: api + namespace: demo +spec: + minAvailable: 2 + selector: + matchLabels: { app: api } diff --git a/workloads/demo-app/front.yaml b/workloads/demo-app/front.yaml new file mode 100644 index 0000000..d475a37 --- /dev/null +++ b/workloads/demo-app/front.yaml @@ -0,0 +1,65 @@ +# front: nginx (unprivileged) serving a static page and proxying /api/ to the api Service. +apiVersion: v1 +kind: ConfigMap +metadata: + name: front-nginx + namespace: demo +data: + default.conf: | + server { + listen 8080; + server_name _; + location = /healthz { add_header Content-Type text/plain; return 200 'ok'; } + location /api/ { + proxy_pass http://api:8080/api/; + proxy_connect_timeout 2s; + proxy_read_timeout 5s; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location / { root /usr/share/nginx/html; index index.html; } + } + index.html: | + selfheal demo +

selfheal demo front

+

container layer (AWS k8s) -> VPN -> VM layer (GCP legacy-api, PostgreSQL)

+ + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: front + namespace: demo + labels: { app: front } +spec: + replicas: 2 + selector: + matchLabels: { app: front } + template: + metadata: + labels: { app: front } + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged:1.31.6-alpine + ports: [{ name: http, containerPort: 8080 }] + volumeMounts: + - { name: conf, mountPath: /etc/nginx/conf.d/default.conf, subPath: default.conf, readOnly: true } + - { name: conf, mountPath: /usr/share/nginx/html/index.html, subPath: index.html, readOnly: true } + livenessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 10, failureThreshold: 3 } + readinessProbe: { httpGet: { path: /healthz, port: http }, periodSeconds: 5, failureThreshold: 2 } + resources: + requests: { cpu: 50m, memory: 32Mi } + limits: { memory: 64Mi } + volumes: + - name: conf + configMap: { name: front-nginx } +--- +apiVersion: v1 +kind: Service +metadata: + name: front + namespace: demo +spec: + selector: { app: front } + ports: [{ name: http, port: 8080, targetPort: http }] diff --git a/workloads/demo-app/ingress.yaml b/workloads/demo-app/ingress.yaml new file mode 100644 index 0000000..f4f7996 --- /dev/null +++ b/workloads/demo-app/ingress.yaml @@ -0,0 +1,17 @@ +# Host-agnostic Ingress (reached through the ALB DNS name): / -> front, /api -> api. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: demo + namespace: demo +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /api + pathType: Prefix + backend: { service: { name: api, port: { name: http } } } + - path: / + pathType: Prefix + backend: { service: { name: front, port: { name: http } } } diff --git a/workloads/ingress-nginx/README.md b/workloads/ingress-nginx/README.md deleted file mode 100644 index 0f3da4e..0000000 --- a/workloads/ingress-nginx/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# workloads/ingress-nginx - -Placeholder. See the root README for the work package that fills this directory. diff --git a/workloads/ingress-nginx/kustomization.yaml b/workloads/ingress-nginx/kustomization.yaml new file mode 100644 index 0000000..a4ef331 --- /dev/null +++ b/workloads/ingress-nginx/kustomization.yaml @@ -0,0 +1,18 @@ +# ingress-nginx baremetal (NodePort) provider manifest, pinned; NodePorts fixed so the ALB target group (30080) matches. +# https://kubernetes.github.io/ingress-nginx/deploy/baremetal/ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.15.1/deploy/static/provider/baremetal/deploy.yaml +patches: + - target: + kind: Service + name: ingress-nginx-controller + namespace: ingress-nginx + patch: |- + - op: add + path: /spec/ports/0/nodePort + value: 30080 + - op: add + path: /spec/ports/1/nodePort + value: 30443