package agent import ( "context" "encoding/json" "fmt" "log" "strings" "time" "github.com/selfheal-lab/selfheal-lab-apps/ai/internal/jev" "github.com/selfheal-lab/selfheal-lab-apps/ai/internal/rag" ) // Shadow gates (2026-10-08): two System One (Jev) judgments run next to the LLM diagnosis and are only recorded. // Triage runs before the LLM sees the alert (same pre-fetched samples the LLM gets), Review runs on the finished // diagnosis (evidence + runbook of the proposed action). Neither changes the disposition; the campaign report compares // the answers with the LLM output and the scenario ground truth to decide whether an enforce mode is worth it. // ShadowRecord is one gate call, saved to ai.jev_shadow and echoed in the Result for the harness event payload. type ShadowRecord struct { Gate string `json:"gate"` Model string `json:"model,omitempty"` Answers map[string]any `json:"answers,omitempty"` LatencyMs int64 `json:"latency_ms"` InputTokens int `json:"input_tokens,omitempty"` Error string `json:"error,omitempty"` state any questions map[string]jev.Question } // Label descriptions (system.md label hints) used as Choice criteria; the model reads these, not the enum names alone. var labelHints = map[string]string{ "api_pod_killed": "one api pod was killed; the ReplicaSet recreates it (platform recovers)", "api_pod_failure": "api pods crash-looping or failing readiness; restart counts rising", "memory_pressure_api": "api pods under memory stress; latency up, OOMKilled possible", "network_latency_api_to_db": "the api to database path is slow (p95 up, no restarts, the database itself is healthy)", "gitops_drift": "an Argo CD application is OutOfSync; selfHeal restores it (platform recovers)", "legacy_process_killed": "the legacy-api container was killed on a MIG instance; compose restarts it (platform recovers)", "legacy_instance_lost": "a legacy MIG instance was deleted or stopped; autohealing recreates it (platform recovers)", "network_latency_legacy": "the ILB to legacy path is slow or lossy", "replication_paused": "db-replica WAL replay is paused (pg_is_wal_replay_paused = 1, lag rising)", "db_primary_down": "db-primary is unreachable on 5432 (probe db-primary = 0)", "schema_mismatch": "application errors from a schema or data change on the database (5xx with SQL errors in logs)", "transient_partition": "a short network partition that already cleared", "transient_loss_legacy": "short packet loss on the legacy path that already cleared", "unknown": "the evidence does not support any of the other labels", } var actionHints = map[string]string{ "none": "no action: the platform already recovers or the symptom cleared", "escalate": "hand to a human: irreversible action needed, no catalogue action fits, or evidence is insufficient", "k8s.rollout_restart": "restart the api or front pods one by one (degraded pods without a configuration change)", "k8s.rollout_undo": "roll a deployment back to the previous revision (the newest revision fails, the previous one was healthy)", "k8s.scale": "add api replicas (capacity problem: pods Running, CPU or memory saturated, no crashes)", "k8s.node_drain": "drain one unhealthy worker node (NotReady or kubelet errors on that node only)", "k8s.node_uncordon": "uncordon a worker node after a drain", "gce.instance_reset": "reset a hung pet VM (status RUNNING but no SSH, metrics or docker); never db-primary", "mig.recreate_instance": "recreate one legacy-api MIG instance that serves errors and was not replaced by autohealing", "mig.resize": "set the legacy MIG back to 2 instances", "db.replay_resume": "resume WAL replay on db-replica when pg_is_wal_replay_paused = 1", "net.route_change": "operator changes a route or firewall through Terraform (BGP sessions down, routes missing on one side; not transient)", } func (a *Agent) shadowOn() bool { return a.jev != nil } // ShadowTriage asks Jev about the raw alert plus the pre-fetched samples (the same material the LLM user message // carries), before the LLM diagnosis starts. Returns nil when shadow mode is off. func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string) *ShadowRecord { if !a.shadowOn() { return nil } labels := map[string]string{} for _, l := range a.enums.Labels { labels[l] = labelHints[l] } labels["unknown"] = labelHints["unknown"] rec := &ShadowRecord{Gate: "triage"} rec.state = map[string]any{ "alert": compactAlert(req.Alert), "now": a.nowSummary(ctx), "samples_last_10m": prefetched, "recent_error_logs": a.recentErrorLogs(ctx), "slo": "availability: probe_success = 1; latency: api p95 < 0.5 s; errors: 5xx ratio < 1%", } rec.questions = map[string]jev.Question{ "label": {Type: "choice", Instructions: "Which root cause label best explains `alert`, `now`, `samples_last_10m` and `recent_error_logs`?", Criteria: labels}, "recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.slo_now_ok` true, meaning the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?", Criteria: map[string]string{"true": "now.slo_now_ok is true: the symptom is gone in the newest samples", "false": "now.slo_now_ok is false, or the values are missing"}}, "needs_investigation": {Type: "noul", Instructions: "To choose a safe remediation action, would an operator need evidence that is not in this state (more logs, Kubernetes objects, VM status, database replication status)?", Criteria: map[string]string{"true": "more evidence is needed before any action", "false": "this state alone is enough to decide the action"}}, "irreversible_needed": {Type: "noul", Instructions: "Would fixing the root cause shown in this state require an irreversible or human-only operation: promoting the database replica, repairing a database schema or data, or starting or recreating a stopped VM?", Criteria: map[string]string{"true": "the fix is irreversible or human-only (database promotion, schema or data repair, VM lifecycle)", "false": "the platform recovers by itself or a reversible catalogue action is enough"}}, } a.ask(ctx, rec) return rec } // nowSummary computes the newest values and the SLO verdict in code (Jev reads text literally and should not compare // numbers or read the tail of a series: jev-1.13 jaggedness notes). func (a *Agent) nowSummary(ctx context.Context) map[string]any { q := func(promql string) any { v, err := a.tools.Instant(ctx, promql) if err != nil || v != v { // NaN return nil } return v } probeALB := q(`probe_success{target="alb"}`) probeDB := q(`probe_success{target="db-primary"}`) probeMin1m := q(`min_over_time(probe_success{target="alb"}[1m])`) probeMin3m := q(`min_over_time(probe_success{target="alb"}[3m])`) p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[1m])))`) errRatio := q(`sum(rate(demo_http_requests_total{status=~"5.."}[1m])) / sum(rate(demo_http_requests_total[1m]))`) restarts := q(`sum(increase(kube_pod_container_status_restarts_total{namespace="demo"}[10m]))`) legacyUp := q(`count(up{job="legacy-api"} == 1)`) f := func(v any) (float64, bool) { x, ok := v.(float64) return x, ok } pa, okA := f(probeALB) pv, okP := f(p95) ev, okE := f(errRatio) probeOK := okA && pa >= 1 latOK := !okP || pv < 0.5 // no traffic = inside SLO (same rule as the harness judge) errOK := !okE || ev < 0.01 return map[string]any{ "probe_alb_now": probeALB, "probe_db_primary_now": probeDB, "probe_alb_min_1m": probeMin1m, "probe_alb_min_3m": probeMin3m, "api_p95_1m_s": p95, "api_error_ratio_1m": errRatio, "api_restarts_10m": restarts, "legacy_instances_up": legacyUp, "probe_ok_now": probeOK, "latency_ok_now": latOK, "errors_ok_now": errOK, "slo_now_ok": probeOK && latOK && errOK, } } // recentErrorLogs returns a short, newest-first excerpt of api error lines (5 minutes, 8 lines) for the triage state. func (a *Agent) recentErrorLogs(ctx context.Context) string { out := a.tools.Call(ctx, "query_logs", `{"logql":"{namespace=\"demo\", app=\"api\"} |~ \"(?i)error|timeout|refused|SQLSTATE\"","range_minutes":5,"limit":8}`) if len(out) > 1800 { out = out[:1800] + "..." } return out } // ShadowReview asks Jev to check the finished diagnosis against its evidence and the runbook of the proposed action. // It runs for every diagnosis (also none / escalate) so the comparison covers the whole campaign. func (a *Agent) ShadowReview(ctx context.Context, req Request, res *Result) *ShadowRecord { if !a.shadowOn() || res == nil { return nil } d := res.Diagnosis runbook := "" if q := strings.TrimSpace(d.ProposedAction); q != "" && q != "none" { if hits, err := a.tools.SearchRunbooks(ctx, q, 2); err == nil { if hs, ok := hits.([]rag.Hit); ok { var parts []string for _, h := range hs { parts = append(parts, "["+h.Source+"] "+h.Chunk) } runbook = strings.Join(parts, "\n\n") } } } actions := map[string]string{"none": actionHints["none"], "escalate": actionHints["escalate"]} for _, id := range a.enums.Actions { actions[id] = actionHints[id] } rec := &ShadowRecord{Gate: "review"} rec.state = map[string]any{ "alert": compactAlert(req.Alert), "diagnosis": map[string]any{"root_cause_label": d.RootCauseLabel, "confidence": d.Confidence, "evidence": d.Evidence, "rationale": d.Rationale, "proposed_action": d.ProposedAction, "target": d.Target, "harness_target": targetFor(d.ProposedAction, d.Target)}, "runbook": runbook, } catalogue := d.ProposedAction != "" && d.ProposedAction != "none" && d.ProposedAction != "escalate" rec.questions = map[string]jev.Question{ "label_supported": {Type: "noul", Instructions: "Do the items in `diagnosis.evidence` support `diagnosis.root_cause_label`?", Criteria: map[string]string{"true": "the evidence items show the condition that the label describes", "false": "the evidence is about something else, or too thin to support the label"}}, "runbook_conditions_met": {Type: "noul", Instructions: "Does `diagnosis.evidence` satisfy the conditions written after 'When' or '使う場面' in `runbook` for `diagnosis.proposed_action`? If `runbook` is empty, answer false.", Criteria: map[string]string{"true": "each condition in the runbook is backed by a quoted evidence item", "false": "a condition is not backed by evidence, the evidence contradicts it, or the runbook is empty"}}, "best_action": {Type: "choice", Instructions: "Given `diagnosis.evidence`, `diagnosis.rationale` and `runbook`, which catalogue action should be proposed? Prefer the smallest action; choose escalate for irreversible changes or weak evidence; choose none when the platform already recovers.", Criteria: actions}, } if catalogue { rec.questions["target_valid"] = jev.Question{Type: "noul", Instructions: "Is `diagnosis.target` a concrete, non-empty object name that the harness can act on for `diagnosis.proposed_action` (a deployment such as api or demo/api, a node name such as ip-10-10-10-11, a VM name such as selfheal-db-replica, or a legacy instance name such as legacy-xxxx)?", Criteria: map[string]string{"true": "the target names one existing object of the kind the action needs", "false": "the target is empty, generic, or of the wrong kind"}} } a.ask(ctx, rec) return rec } // Enforce applies the two review rules when JEV_MODE=enforce (2026-10-08 campaign-2 report, 3.3). It only ever downgrades // a catalogue proposal to escalate; none and escalate are never changed. The decision is recorded in res.Jev["enforced"]. func (a *Agent) Enforce(res *Result, rev *ShadowRecord) { if !a.jevEnforce || res == nil || rev == nil || rev.Answers == nil { return } d := &res.Diagnosis if d.ProposedAction == "" || d.ProposedAction == "none" || d.ProposedAction == "escalate" { return } var rule, reason string if ans, ok := rev.Answers["best_action"].(jev.Answer); ok && ans.Choice != "" && ans.Choice != d.ProposedAction && ans.Confidence != nil && *ans.Confidence >= 0.9 { rule, reason = "best_action_disagrees", fmt.Sprintf("jev best_action=%s (confidence %.2f) != proposed %s", ans.Choice, *ans.Confidence, d.ProposedAction) } if rule == "" { if ans, ok := rev.Answers["target_valid"].(jev.Answer); ok && ans.Noul != nil && *ans.Noul < 0.5 { rule, reason = "target_invalid", fmt.Sprintf("jev target_valid=%.2f for target %q", *ans.Noul, d.Target) } } if rule == "" { return } log.Printf("jev enforce: %s: %s -> escalate", rule, reason) if res.Jev == nil { res.Jev = map[string]any{} } res.Jev["enforced"] = map[string]any{"rule": rule, "reason": reason, "from": d.ProposedAction, "to": "escalate"} d.Rationale = fmt.Sprintf("%s [jev %s: %s]", d.Rationale, rule, reason) d.ProposedAction = "escalate" } func (a *Agent) ask(ctx context.Context, rec *ShadowRecord) { c, cancel := context.WithTimeout(ctx, 12*time.Second) defer cancel() t0 := time.Now() resp, err := a.jev.Ask(c, rec.state, rec.questions) rec.LatencyMs = time.Since(t0).Milliseconds() if err != nil { rec.Error = err.Error() if err != jev.ErrNoKey { log.Printf("jev %s: %v", rec.Gate, err) } return } rec.Model = resp.Model rec.InputTokens = resp.Usage.InputTokens rec.Answers = map[string]any{} for k, v := range resp.Answers { rec.Answers[k] = v } } // SaveShadow stores the gate records with the session id and the LLM's own decision for the later comparison. func (a *Agent) SaveShadow(ctx context.Context, req Request, sessionID string, res *Result, recs ...*ShadowRecord) { if a.store == nil { return } var llm map[string]any if res != nil { llm = map[string]any{"root_cause_label": res.RootCauseLabel, "confidence": res.Confidence, "proposed_action": res.ProposedAction, "target": res.Target, "disposition": res.Disposition} } for _, r := range recs { if r == nil { continue } row := rag.JevShadow{SessionID: sessionID, RunID: req.RunID, ScenarioID: req.ScenarioID, Gate: r.Gate, Model: r.Model, State: r.state, Questions: r.questions, Answers: r.Answers, LLM: llm, LatencyMs: r.LatencyMs, InputTokens: r.InputTokens, Error: r.Error} if err := a.store.SaveJev(ctx, row); err != nil { log.Printf("jev shadow save (%s): %v", r.Gate, err) } } } // compactAlert keeps the fields a judgment needs (name, labels, annotations, status, times) and drops Grafana noise. func compactAlert(alert map[string]any) map[string]any { out := map[string]any{} for _, k := range []string{"alertname", "status", "labels", "annotations", "startsAt", "endsAt", "valueString"} { if v, ok := alert[k]; ok { out[k] = v } } if len(out) == 0 { b, _ := json.Marshal(alert) out["raw"] = fmt.Sprintf("%.1500s", string(b)) } return out }