205 lines
7.7 KiB
Go
205 lines
7.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Catalogue of design 2.3. Exec = the harness can carry it out; otherwise it is a platform-internal (layer 1) or
|
|
// human-only (layer 3) action that the harness records but does not perform.
|
|
type CatalogEntry struct {
|
|
ActionID string `json:"action_id"`
|
|
SLOImpact string `json:"slo_impact"` // small | large
|
|
Reversible bool `json:"reversible"`
|
|
Layer string `json:"layer"`
|
|
Exec bool `json:"harness_executes"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
var catalog = []CatalogEntry{
|
|
{"k8s.container_restart", "small", true, "1", false, "kubelet restarts the container on liveness failure"},
|
|
{"k8s.pod_recreate", "small", true, "1", false, "ReplicaSet recreates the pod"},
|
|
{"gitops.self_heal", "small", true, "1", false, "Argo CD selfHeal restores the Git state"},
|
|
{"mig.autoheal_recreate", "small", true, "1", false, "MIG autohealing recreates the instance"},
|
|
{"vm.service_restart", "small", true, "1", false, "compose restart:always / systemd Restart=on-failure"},
|
|
{"k8s.rollout_restart", "small", true, "1/2", true, "kubectl rollout restart deployment/<name>"},
|
|
{"k8s.rollout_undo", "large", true, "2", true, "kubectl rollout undo deployment/<name>"},
|
|
{"k8s.scale", "small", true, "2", true, "change replicas"},
|
|
{"k8s.node_drain", "large", true, "2", true, "cordon + drain a worker (undo: uncordon)"},
|
|
{"k8s.node_uncordon", "small", true, "2", true, "uncordon a worker (revert of k8s.node_drain)"},
|
|
{"gce.instance_reset", "large", true, "2", true, "gcloud compute instances reset <pet vm>"},
|
|
{"mig.recreate_instance", "small", true, "2", true, "recreate one legacy MIG instance"},
|
|
{"mig.resize", "small", true, "2", true, "MIG target_size (restore 2 before the session ends)"},
|
|
{"db.replay_resume", "small", true, "2", true, "SELECT pg_wal_replay_resume() on db-replica"},
|
|
{"net.route_change", "large", true, "2", false, "route/firewall change through Terraform"},
|
|
{"db.promote", "large", false, "3", false, "pg_promote() + DSN switch (human)"},
|
|
{"db.data_fix", "large", false, "3", false, "data fix SQL / restore from backup (human)"},
|
|
{"tf.instance_lifecycle", "large", false, "3", false, "instance create/delete/spec change through Terraform (human)"},
|
|
{"escalate", "-", true, "3", false, "no catalogue action fits; hand over to a human"},
|
|
}
|
|
|
|
func catalogEntry(id string) *CatalogEntry {
|
|
for i := range catalog {
|
|
if catalog[i].ActionID == id {
|
|
return &catalog[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *App) handleCatalog(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, catalog) }
|
|
|
|
type ActionRequest struct {
|
|
ActionID string `json:"action_id"`
|
|
Target map[string]any `json:"target"`
|
|
Token string `json:"token"`
|
|
RunID *string `json:"run_id"`
|
|
ScenarioID *string `json:"scenario_id"`
|
|
Note string `json:"note"`
|
|
}
|
|
|
|
// POST /actions/execute (design 5.2): token = human token (layer 3) or an approved proposal token (layer 2).
|
|
func (a *App) handleActionExecute(w http.ResponseWriter, r *http.Request) {
|
|
var req ActionRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeErr(w, 400, err)
|
|
return
|
|
}
|
|
ce := catalogEntry(req.ActionID)
|
|
if ce == nil {
|
|
writeErr(w, 400, fmt.Errorf("action_id %q is not in the catalogue", req.ActionID))
|
|
return
|
|
}
|
|
actor, err := a.authorize(r.Context(), req)
|
|
if err != nil {
|
|
writeErr(w, 401, err)
|
|
return
|
|
}
|
|
runID, sid := req.RunID, req.ScenarioID
|
|
if cur := a.runs.current(); cur != nil && runID == nil {
|
|
runID = &cur.RunID
|
|
if cur.ScenarioID != "" {
|
|
sid = &cur.ScenarioID
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
defer cancel()
|
|
if !ce.Exec {
|
|
a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "recorded_only": true, "note": req.Note, "layer": ce.Layer})
|
|
mActs.WithLabelValues(req.ActionID, actor, "recorded").Inc()
|
|
if actor == "ai-approved" {
|
|
_ = a.store.MarkProposalExecuted(ctx, req.Token)
|
|
}
|
|
writeJSON(w, 202, map[string]any{"action_id": req.ActionID, "recorded_only": true, "layer": ce.Layer, "note": "not executed by the harness (platform-internal or human-only); recorded in events"})
|
|
return
|
|
}
|
|
res, err := a.executeAction(ctx, req.ActionID, req.Target, actor, runID, sid)
|
|
result := "ok"
|
|
if err != nil {
|
|
result = "error"
|
|
}
|
|
mActs.WithLabelValues(req.ActionID, actor, result).Inc()
|
|
a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "result": res, "error": errStr(err), "note": req.Note})
|
|
if req.Token != "" && actor == "ai-approved" {
|
|
_ = a.store.MarkProposalExecuted(ctx, req.Token)
|
|
}
|
|
if err != nil {
|
|
writeJSON(w, 500, map[string]any{"action_id": req.ActionID, "actor": actor, "error": err.Error(), "result": res})
|
|
return
|
|
}
|
|
writeJSON(w, 200, map[string]any{"action_id": req.ActionID, "actor": actor, "result": res})
|
|
}
|
|
|
|
func (a *App) authorize(ctx context.Context, req ActionRequest) (string, error) {
|
|
if req.Token == "" {
|
|
return "", errors.New("token required")
|
|
}
|
|
if human, err := readFileTrim(a.cfg.HumanTokFile); err == nil && subtle.ConstantTimeCompare([]byte(human), []byte(req.Token)) == 1 {
|
|
return "human", nil
|
|
}
|
|
p, err := a.store.GetProposal(ctx, req.Token)
|
|
if err != nil {
|
|
return "", errors.New("invalid token")
|
|
}
|
|
switch {
|
|
case p.ApprovedAt == nil:
|
|
return "", errors.New("proposal not approved")
|
|
case p.ExecutedAt != nil:
|
|
return "", errors.New("proposal already executed")
|
|
case time.Now().After(p.ExpiresAt):
|
|
return "", errors.New("proposal expired")
|
|
case p.ActionID != req.ActionID:
|
|
return "", fmt.Errorf("token was approved for %s, not %s", p.ActionID, req.ActionID)
|
|
}
|
|
return "ai-approved", nil
|
|
}
|
|
|
|
func str(m map[string]any, k string) string {
|
|
v, _ := m[k].(string)
|
|
return v
|
|
}
|
|
|
|
func num(m map[string]any, k string) int64 {
|
|
switch v := m[k].(type) {
|
|
case float64:
|
|
return int64(v)
|
|
case int:
|
|
return int64(v)
|
|
case int64:
|
|
return v
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// executeAction performs one executable catalogue action.
|
|
func (a *App) executeAction(ctx context.Context, id string, t map[string]any, actor string, runID, sid *string) (map[string]any, error) {
|
|
if t == nil {
|
|
t = map[string]any{}
|
|
}
|
|
ns := str(t, "namespace")
|
|
if ns == "" {
|
|
ns = "demo"
|
|
}
|
|
switch id {
|
|
case "k8s.rollout_restart":
|
|
return map[string]any{"deployment": str(t, "deployment")}, a.rolloutRestart(ctx, ns, str(t, "deployment"))
|
|
case "k8s.rollout_undo":
|
|
return map[string]any{"deployment": str(t, "deployment")}, a.rolloutUndo(ctx, ns, str(t, "deployment"))
|
|
case "k8s.scale":
|
|
return map[string]any{"deployment": str(t, "deployment"), "replicas": num(t, "replicas")}, a.scaleDeployment(ctx, ns, str(t, "deployment"), int32(num(t, "replicas")))
|
|
case "k8s.node_drain":
|
|
n, err := a.drain(ctx, str(t, "node"))
|
|
return map[string]any{"node": str(t, "node"), "evicted": n}, err
|
|
case "k8s.node_uncordon":
|
|
return map[string]any{"node": str(t, "node")}, a.cordon(ctx, str(t, "node"), false)
|
|
case "gce.instance_reset":
|
|
return a.gcpOp(ctx, "instance_reset", str(t, "instance"), "", 0)
|
|
case "mig.recreate_instance":
|
|
return a.gcpOp(ctx, "mig_recreate", str(t, "instance"), migName(t), 0)
|
|
case "mig.resize":
|
|
return a.gcpOp(ctx, "mig_resize", "", migName(t), num(t, "size"))
|
|
case "db.replay_resume":
|
|
host := str(t, "host")
|
|
if host == "" {
|
|
host = "db-replica"
|
|
}
|
|
out, err := a.psqlExec(ctx, host, "postgres", "SELECT pg_wal_replay_resume()")
|
|
return map[string]any{"host": host, "output": out}, err
|
|
case "escalate":
|
|
return map[string]any{"escalated": true}, nil
|
|
}
|
|
return nil, fmt.Errorf("action %s is not executable by the harness", id)
|
|
}
|
|
|
|
func migName(t map[string]any) string {
|
|
if m := str(t, "mig"); m != "" {
|
|
return m
|
|
}
|
|
return env("LEGACY_MIG", "selfheal-legacy")
|
|
}
|