feat(ai): redesign the Jev triage state and add JEV_MODE=enforce review rules

第 2 回キャンペーンの突き合わせ結果を反映した。triage の state にコードで計算した
最新値の要約(now: プローブ、1 分 p95、1 分エラー率、再起動数、slo_now_ok)と
直近 5 分のエラーログ抜粋を加え、recovering は slo_now_ok を読むだけにした。
layer の choice を irreversible_needed の noul に置き換え、target_valid は
カタログ action のときだけ尋ねる。JEV_MODE=enforce では review の 2 ルール
(best_action 不一致かつ確信度 0.9 以上、target_valid 0.5 未満)で提案を
escalate に格下げし、判断を res.jev.enforced に記録する。既定は shadow のまま。
This commit is contained in:
seon committed 2026-10-08 17:45:28 +09:00
1 parent 795be156d9
commit 13797e5741
5 files changed
+178 -30

No files matched your search

+12 -11
View File
@@ -36,15 +36,16 @@ type Config struct {
}
type Agent struct {
cfg Config
oai *openai.Client
tools *tools.Runner
harness *harness.Client
store *rag.Store
enums diag.Enums
promptV string
http *http.Client
jev *jev.Client // nil = shadow gates off
cfg Config
oai *openai.Client
tools *tools.Runner
harness *harness.Client
store *rag.Store
enums diag.Enums
promptV string
http *http.Client
jev *jev.Client // nil = shadow gates off
jevEnforce bool // JEV_MODE=enforce: review rules may downgrade a proposal to escalate
}
func New(cfg Config, oai *openai.Client, tr *tools.Runner, hc *harness.Client, store *rag.Store, enums diag.Enums, promptVersion string) *Agent {
@@ -63,8 +64,8 @@ func New(cfg Config, oai *openai.Client, tr *tools.Runner, hc *harness.Client, s
return &Agent{cfg: cfg, oai: oai, tools: tr, harness: hc, store: store, enums: enums, promptV: promptVersion, http: &http.Client{Timeout: 30 * time.Second}}
}
// EnableShadow turns the Jev shadow gates on (JEV_MODE=shadow). Answers are recorded only.
func (a *Agent) EnableShadow(c *jev.Client) { a.jev = c }
// EnableShadow turns the Jev gates on (JEV_MODE=shadow records only; enforce also applies the review rules).
func (a *Agent) EnableShadow(c *jev.Client, enforce bool) { a.jev, a.jevEnforce = c, enforce }
// Request is the harness -> AI VM contract (harness alerts.go diagnose()).
type Request struct {
+50
View File
@@ -0,0 +1,50 @@
package agent
import (
"testing"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/jev"
)
func f(v float64) *float64 { return &v }
func TestEnforceRules(t *testing.T) {
a := &Agent{jevEnforce: true}
// rule A: best_action disagrees with high confidence -> escalate
res := &Result{}
res.ProposedAction, res.Target = "net.route_change", ""
rev := &ShadowRecord{Gate: "review", Answers: map[string]any{"best_action": jev.Answer{Type: "choice", Choice: "escalate", Confidence: f(0.95)}}}
a.Enforce(res, rev)
if res.ProposedAction != "escalate" || res.Jev["enforced"].(map[string]any)["rule"] != "best_action_disagrees" {
t.Fatalf("rule A: %+v %v", res.Diagnosis, res.Jev)
}
// rule B: target invalid -> escalate
res = &Result{}
res.ProposedAction, res.Target = "k8s.rollout_restart", ""
rev = &ShadowRecord{Gate: "review", Answers: map[string]any{"best_action": jev.Answer{Type: "choice", Choice: "k8s.rollout_restart", Confidence: f(0.9)}, "target_valid": jev.Answer{Type: "noul", Noul: f(0.2)}}}
a.Enforce(res, rev)
if res.ProposedAction != "escalate" || res.Jev["enforced"].(map[string]any)["rule"] != "target_invalid" {
t.Fatalf("rule B: %+v %v", res.Diagnosis, res.Jev)
}
// agreement with valid target -> unchanged; none/escalate -> untouched; shadow mode -> untouched
res = &Result{}
res.ProposedAction, res.Target = "k8s.rollout_restart", "api"
rev = &ShadowRecord{Gate: "review", Answers: map[string]any{"best_action": jev.Answer{Choice: "k8s.rollout_restart", Confidence: f(0.99)}, "target_valid": jev.Answer{Noul: f(0.9)}}}
a.Enforce(res, rev)
if res.ProposedAction != "k8s.rollout_restart" || res.Jev != nil {
t.Fatalf("agreement: %+v", res.Diagnosis)
}
res = &Result{}
res.ProposedAction = "none"
a.Enforce(res, &ShadowRecord{Answers: map[string]any{"best_action": jev.Answer{Choice: "escalate", Confidence: f(0.99)}}})
if res.ProposedAction != "none" {
t.Fatalf("none must not change")
}
s := &Agent{jevEnforce: false}
res = &Result{}
res.ProposedAction = "k8s.scale"
s.Enforce(res, &ShadowRecord{Answers: map[string]any{"best_action": jev.Answer{Choice: "escalate", Confidence: f(0.99)}}})
if res.ProposedAction != "k8s.scale" {
t.Fatalf("shadow must not change")
}
}
+91 -14
View File
@@ -77,26 +77,69 @@ func (a *Agent) ShadowTriage(ctx context.Context, req Request, prefetched string
labels["unknown"] = labelHints["unknown"]
rec := &ShadowRecord{Gate: "triage"}
rec.state = map[string]any{
"alert": compactAlert(req.Alert),
"samples_last_10m": prefetched,
"slo": "availability: probe_success = 1; latency: api p95 < 0.5 s; errors: 5xx ratio < 1%",
"alert": compactAlert(req.Alert),
"now": a.nowSummary(ctx),
"samples_last_10m": prefetched,
"recent_error_logs": a.recentErrorLogs(ctx),
"slo": "availability: probe_success = 1; latency: api p95 < 0.5 s; errors: 5xx ratio < 1%",
}
rec.questions = map[string]jev.Question{
"label": {Type: "choice", Instructions: "Which root cause label best explains `alert` together with `samples_last_10m`?", Criteria: labels},
"recovering": {Type: "noul", Instructions: "Looking only at the newest values in `samples_last_10m`, is the symptom already gone: probe_success back to 1 and api_p95_latency_s and api_error_ratio inside `slo`?",
Criteria: map[string]string{"true": "the newest samples are inside the SLO again", "false": "the newest samples still show the failure, or the samples are missing"}},
"needs_investigation": {Type: "noul", Instructions: "To choose a safe remediation action, would an operator need evidence that is not in this state (pod logs, Kubernetes objects, VM status, database replication status)?",
"label": {Type: "choice", Instructions: "Which root cause label best explains `alert`, `now`, `samples_last_10m` and `recent_error_logs`?", Criteria: labels},
"recovering": {Type: "noul", Instructions: "Look only at `now`: is `now.slo_now_ok` true, meaning the newest probe, the 1-minute p95 and the 1-minute error ratio are all inside `slo` right now?",
Criteria: map[string]string{"true": "now.slo_now_ok is true: the symptom is gone in the newest samples", "false": "now.slo_now_ok is false, or the values are missing"}},
"needs_investigation": {Type: "noul", Instructions: "To choose a safe remediation action, would an operator need evidence that is not in this state (more logs, Kubernetes objects, VM status, database replication status)?",
Criteria: map[string]string{"true": "more evidence is needed before any action", "false": "this state alone is enough to decide the action"}},
"layer": {Type: "choice", Instructions: "Which recovery layer handles this alert?", Criteria: map[string]string{
"layer1": "the platform recovers by itself (ReplicaSet, Argo CD selfHeal, MIG autohealing, compose restart); no action needed",
"layer2": "a reversible catalogue action with human approval is needed",
"layer3": "an irreversible or human-only action is needed (database promotion, data fix, instance lifecycle)",
}},
"irreversible_needed": {Type: "noul", Instructions: "Would fixing the root cause shown in this state require an irreversible or human-only operation: promoting the database replica, repairing a database schema or data, or starting or recreating a stopped VM?",
Criteria: map[string]string{"true": "the fix is irreversible or human-only (database promotion, schema or data repair, VM lifecycle)", "false": "the platform recovers by itself or a reversible catalogue action is enough"}},
}
a.ask(ctx, rec)
return rec
}
// nowSummary computes the newest values and the SLO verdict in code (Jev reads text literally and should not compare
// numbers or read the tail of a series: jev-1.13 jaggedness notes).
func (a *Agent) nowSummary(ctx context.Context) map[string]any {
q := func(promql string) any {
v, err := a.tools.Instant(ctx, promql)
if err != nil || v != v { // NaN
return nil
}
return v
}
probeALB := q(`probe_success{target="alb"}`)
probeDB := q(`probe_success{target="db-primary"}`)
probeMin1m := q(`min_over_time(probe_success{target="alb"}[1m])`)
probeMin3m := q(`min_over_time(probe_success{target="alb"}[3m])`)
p95 := q(`histogram_quantile(0.95, sum by (le) (rate(demo_http_request_duration_seconds_bucket[1m])))`)
errRatio := q(`sum(rate(demo_http_requests_total{status=~"5.."}[1m])) / sum(rate(demo_http_requests_total[1m]))`)
restarts := q(`sum(increase(kube_pod_container_status_restarts_total{namespace="demo"}[10m]))`)
legacyUp := q(`count(up{job="legacy-api"} == 1)`)
f := func(v any) (float64, bool) {
x, ok := v.(float64)
return x, ok
}
pa, okA := f(probeALB)
pv, okP := f(p95)
ev, okE := f(errRatio)
probeOK := okA && pa >= 1
latOK := !okP || pv < 0.5 // no traffic = inside SLO (same rule as the harness judge)
errOK := !okE || ev < 0.01
return map[string]any{
"probe_alb_now": probeALB, "probe_db_primary_now": probeDB, "probe_alb_min_1m": probeMin1m, "probe_alb_min_3m": probeMin3m,
"api_p95_1m_s": p95, "api_error_ratio_1m": errRatio, "api_restarts_10m": restarts, "legacy_instances_up": legacyUp,
"probe_ok_now": probeOK, "latency_ok_now": latOK, "errors_ok_now": errOK, "slo_now_ok": probeOK && latOK && errOK,
}
}
// recentErrorLogs returns a short, newest-first excerpt of api error lines (5 minutes, 8 lines) for the triage state.
func (a *Agent) recentErrorLogs(ctx context.Context) string {
out := a.tools.Call(ctx, "query_logs", `{"logql":"{namespace=\"demo\", app=\"api\"} |~ \"(?i)error|timeout|refused|SQLSTATE\"","range_minutes":5,"limit":8}`)
if len(out) > 1800 {
out = out[:1800] + "..."
}
return out
}
// ShadowReview asks Jev to check the finished diagnosis against its evidence and the runbook of the proposed action.
// It runs for every diagnosis (also none / escalate) so the comparison covers the whole campaign.
func (a *Agent) ShadowReview(ctx context.Context, req Request, res *Result) *ShadowRecord {
@@ -127,19 +170,53 @@ func (a *Agent) ShadowReview(ctx context.Context, req Request, res *Result) *Sha
"rationale": d.Rationale, "proposed_action": d.ProposedAction, "target": d.Target, "harness_target": targetFor(d.ProposedAction, d.Target)},
"runbook": runbook,
}
catalogue := d.ProposedAction != "" && d.ProposedAction != "none" && d.ProposedAction != "escalate"
rec.questions = map[string]jev.Question{
"label_supported": {Type: "noul", Instructions: "Do the items in `diagnosis.evidence` support `diagnosis.root_cause_label`?",
Criteria: map[string]string{"true": "the evidence items show the condition that the label describes", "false": "the evidence is about something else, or too thin to support the label"}},
"runbook_conditions_met": {Type: "noul", Instructions: "Does `diagnosis.evidence` satisfy the conditions written after 'When' or '使う場面' in `runbook` for `diagnosis.proposed_action`? If `runbook` is empty, answer false.",
Criteria: map[string]string{"true": "each condition in the runbook is backed by a quoted evidence item", "false": "a condition is not backed by evidence, the evidence contradicts it, or the runbook is empty"}},
"target_valid": {Type: "noul", Instructions: "Is `diagnosis.target` a concrete, non-empty object name that the harness can act on for `diagnosis.proposed_action` (a deployment such as api or demo/api, a node name such as ip-10-10-10-11, a VM name such as selfheal-db-replica, or a legacy instance name such as legacy-xxxx)? For proposed_action none or escalate, answer true.",
Criteria: map[string]string{"true": "the target names one existing object of the kind the action needs", "false": "the target is empty, generic, or of the wrong kind"}},
"best_action": {Type: "choice", Instructions: "Given `diagnosis.evidence`, `diagnosis.rationale` and `runbook`, which catalogue action should be proposed? Prefer the smallest action; choose escalate for irreversible changes or weak evidence; choose none when the platform already recovers.", Criteria: actions},
}
if catalogue {
rec.questions["target_valid"] = jev.Question{Type: "noul", Instructions: "Is `diagnosis.target` a concrete, non-empty object name that the harness can act on for `diagnosis.proposed_action` (a deployment such as api or demo/api, a node name such as ip-10-10-10-11, a VM name such as selfheal-db-replica, or a legacy instance name such as legacy-xxxx)?",
Criteria: map[string]string{"true": "the target names one existing object of the kind the action needs", "false": "the target is empty, generic, or of the wrong kind"}}
}
a.ask(ctx, rec)
return rec
}
// Enforce applies the two review rules when JEV_MODE=enforce (2026-10-08 campaign-2 report, 3.3). It only ever downgrades
// a catalogue proposal to escalate; none and escalate are never changed. The decision is recorded in res.Jev["enforced"].
func (a *Agent) Enforce(res *Result, rev *ShadowRecord) {
if !a.jevEnforce || res == nil || rev == nil || rev.Answers == nil {
return
}
d := &res.Diagnosis
if d.ProposedAction == "" || d.ProposedAction == "none" || d.ProposedAction == "escalate" {
return
}
var rule, reason string
if ans, ok := rev.Answers["best_action"].(jev.Answer); ok && ans.Choice != "" && ans.Choice != d.ProposedAction && ans.Confidence != nil && *ans.Confidence >= 0.9 {
rule, reason = "best_action_disagrees", fmt.Sprintf("jev best_action=%s (confidence %.2f) != proposed %s", ans.Choice, *ans.Confidence, d.ProposedAction)
}
if rule == "" {
if ans, ok := rev.Answers["target_valid"].(jev.Answer); ok && ans.Noul != nil && *ans.Noul < 0.5 {
rule, reason = "target_invalid", fmt.Sprintf("jev target_valid=%.2f for target %q", *ans.Noul, d.Target)
}
}
if rule == "" {
return
}
log.Printf("jev enforce: %s: %s -> escalate", rule, reason)
if res.Jev == nil {
res.Jev = map[string]any{}
}
res.Jev["enforced"] = map[string]any{"rule": rule, "reason": reason, "from": d.ProposedAction, "to": "escalate"}
d.Rationale = fmt.Sprintf("%s [jev %s: %s]", d.Rationale, rule, reason)
d.ProposedAction = "escalate"
}
func (a *Agent) ask(ctx context.Context, rec *ShadowRecord) {
c, cancel := context.WithTimeout(ctx, 12*time.Second)
defer cancel()