selfheal-lab-apps/harness/actions.go

package main

import (
	"context"
	"crypto/subtle"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"time"
)

// Catalogue of design 2.3. Exec = the harness can carry it out; otherwise it is a platform-internal (layer 1) or
// human-only (layer 3) action that the harness records but does not perform.
type CatalogEntry struct {
	ActionID    string `json:"action_id"`
	SLOImpact   string `json:"slo_impact"` // small | large
	Reversible  bool   `json:"reversible"`
	Layer       string `json:"layer"`
	Exec        bool   `json:"harness_executes"`
	Description string `json:"description"`
}

var catalog = []CatalogEntry{
	{"k8s.container_restart", "small", true, "1", false, "kubelet restarts the container on liveness failure"},
	{"k8s.pod_recreate", "small", true, "1", false, "ReplicaSet recreates the pod"},
	{"gitops.self_heal", "small", true, "1", false, "Argo CD selfHeal restores the Git state"},
	{"mig.autoheal_recreate", "small", true, "1", false, "MIG autohealing recreates the instance"},
	{"vm.service_restart", "small", true, "1", false, "compose restart:always / systemd Restart=on-failure"},
	{"k8s.rollout_restart", "small", true, "1/2", true, "kubectl rollout restart deployment/<name>"},
	{"k8s.rollout_undo", "large", true, "2", true, "kubectl rollout undo deployment/<name>"},
	{"k8s.scale", "small", true, "2", true, "change replicas"},
	{"k8s.node_drain", "large", true, "2", true, "cordon + drain a worker (undo: uncordon)"},
	{"k8s.node_uncordon", "small", true, "2", true, "uncordon a worker (revert of k8s.node_drain)"},
	{"gce.instance_reset", "large", true, "2", true, "gcloud compute instances reset <pet vm>"},
	{"mig.recreate_instance", "small", true, "2", true, "recreate one legacy MIG instance"},
	{"mig.resize", "small", true, "2", true, "MIG target_size (restore 2 before the session ends)"},
	{"db.replay_resume", "small", true, "2", true, "SELECT pg_wal_replay_resume() on db-replica"},
	{"net.route_change", "large", true, "2", false, "route/firewall change through Terraform"},
	{"db.promote", "large", false, "3", false, "pg_promote() + DSN switch (human)"},
	{"db.data_fix", "large", false, "3", false, "data fix SQL / restore from backup (human)"},
	{"tf.instance_lifecycle", "large", false, "3", false, "instance create/delete/spec change through Terraform (human)"},
	{"escalate", "-", true, "3", false, "no catalogue action fits; hand over to a human"},
}

func catalogEntry(id string) *CatalogEntry {
	for i := range catalog {
		if catalog[i].ActionID == id {
			return &catalog[i]
		}
	}
	return nil
}

func (a *App) handleCatalog(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, catalog) }

type ActionRequest struct {
	ActionID   string         `json:"action_id"`
	Target     map[string]any `json:"target"`
	Token      string         `json:"token"`
	RunID      *string        `json:"run_id"`
	ScenarioID *string        `json:"scenario_id"`
	Note       string         `json:"note"`
}

// POST /actions/execute (design 5.2): token = human token (layer 3) or an approved proposal token (layer 2).
func (a *App) handleActionExecute(w http.ResponseWriter, r *http.Request) {
	var req ActionRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeErr(w, 400, err)
		return
	}
	ce := catalogEntry(req.ActionID)
	if ce == nil {
		writeErr(w, 400, fmt.Errorf("action_id %q is not in the catalogue", req.ActionID))
		return
	}
	actor, err := a.authorize(r.Context(), req)
	if err != nil {
		writeErr(w, 401, err)
		return
	}
	runID, sid := req.RunID, req.ScenarioID
	if cur := a.runs.current(); cur != nil && runID == nil {
		runID = &cur.RunID
		if cur.ScenarioID != "" {
			sid = &cur.ScenarioID
		}
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()
	if !ce.Exec {
		a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "recorded_only": true, "note": req.Note, "layer": ce.Layer})
		mActs.WithLabelValues(req.ActionID, actor, "recorded").Inc()
		if actor == "ai-approved" {
			_ = a.store.MarkProposalExecuted(ctx, req.Token)
		}
		writeJSON(w, 202, map[string]any{"action_id": req.ActionID, "recorded_only": true, "layer": ce.Layer, "note": "not executed by the harness (platform-internal or human-only); recorded in events"})
		return
	}
	res, err := a.executeAction(ctx, req.ActionID, req.Target, actor, runID, sid)
	result := "ok"
	if err != nil {
		result = "error"
	}
	mActs.WithLabelValues(req.ActionID, actor, result).Inc()
	a.store.Event(ctx, runID, sid, "action", actor, map[string]any{"action_id": req.ActionID, "target": req.Target, "result": res, "error": errStr(err), "note": req.Note})
	if req.Token != "" && actor == "ai-approved" {
		_ = a.store.MarkProposalExecuted(ctx, req.Token)
	}
	if err != nil {
		writeJSON(w, 500, map[string]any{"action_id": req.ActionID, "actor": actor, "error": err.Error(), "result": res})
		return
	}
	writeJSON(w, 200, map[string]any{"action_id": req.ActionID, "actor": actor, "result": res})
}

func (a *App) authorize(ctx context.Context, req ActionRequest) (string, error) {
	if req.Token == "" {
		return "", errors.New("token required")
	}
	if human, err := readFileTrim(a.cfg.HumanTokFile); err == nil && subtle.ConstantTimeCompare([]byte(human), []byte(req.Token)) == 1 {
		return "human", nil
	}
	p, err := a.store.GetProposal(ctx, req.Token)
	if err != nil {
		return "", errors.New("invalid token")
	}
	switch {
	case p.RejectedAt != nil:
		return "", errors.New("proposal rejected")
	case p.ApprovedAt == nil:
		return "", errors.New("proposal not approved")
	case p.ExecutedAt != nil:
		return "", errors.New("proposal already executed")
	case time.Now().After(p.ExpiresAt):
		return "", errors.New("proposal expired")
	case p.ActionID != req.ActionID:
		return "", fmt.Errorf("token was approved for %s, not %s", p.ActionID, req.ActionID)
	}
	return "ai-approved", nil
}

func str(m map[string]any, k string) string {
	v, _ := m[k].(string)
	return v
}

func num(m map[string]any, k string) int64 {
	switch v := m[k].(type) {
	case float64:
		return int64(v)
	case int:
		return int64(v)
	case int64:
		return v
	}
	return 0
}

// executeAction performs one executable catalogue action.
func (a *App) executeAction(ctx context.Context, id string, t map[string]any, actor string, runID, sid *string) (map[string]any, error) {
	if t == nil {
		t = map[string]any{}
	}
	ns := str(t, "namespace")
	if ns == "" {
		ns = "demo"
	}
	switch id {
	case "k8s.rollout_restart":
		return map[string]any{"deployment": str(t, "deployment")}, a.rolloutRestart(ctx, ns, str(t, "deployment"))
	case "k8s.rollout_undo":
		return map[string]any{"deployment": str(t, "deployment")}, a.rolloutUndo(ctx, ns, str(t, "deployment"))
	case "k8s.scale":
		return map[string]any{"deployment": str(t, "deployment"), "replicas": num(t, "replicas")}, a.scaleDeployment(ctx, ns, str(t, "deployment"), int32(num(t, "replicas")))
	case "k8s.node_drain":
		n, err := a.drain(ctx, str(t, "node"))
		return map[string]any{"node": str(t, "node"), "evicted": n}, err
	case "k8s.node_uncordon":
		return map[string]any{"node": str(t, "node")}, a.cordon(ctx, str(t, "node"), false)
	case "gce.instance_reset":
		return a.gcpOp(ctx, "instance_reset", str(t, "instance"), "", 0)
	case "mig.recreate_instance":
		return a.gcpOp(ctx, "mig_recreate", str(t, "instance"), migName(t), 0)
	case "mig.resize":
		return a.gcpOp(ctx, "mig_resize", "", migName(t), num(t, "size"))
	case "db.replay_resume":
		host := str(t, "host")
		if host == "" {
			host = "db-replica"
		}
		out, err := a.psqlExec(ctx, host, "postgres", "SELECT pg_wal_replay_resume()")
		return map[string]any{"host": host, "output": out}, err
	case "escalate":
		return map[string]any{"escalated": true}, nil
	}
	return nil, fmt.Errorf("action %s is not executable by the harness", id)
}

func migName(t map[string]any) string {
	if m := str(t, "mig"); m != "" {
		return m
	}
	return env("LEGACY_MIG", "selfheal-legacy")
}