diff --git a/ai/cmd/agent/main.go b/ai/cmd/agent/main.go index b275567..1efaee0 100644 --- a/ai/cmd/agent/main.go +++ b/ai/cmd/agent/main.go @@ -122,9 +122,17 @@ func main() { // external-* model ids (reference runs, 2026-10-16): each loop turn is served to an outside client over // GET /external/next and answered through POST /external/reply (internal/external). The diagnosis deadline is // EXTERNAL_DIAGNOSE_TIMEOUT_S (the harness waits a little longer for external-* models), one turn at most - // EXTERNAL_TURN_TIMEOUT_S. The endpoints are reachable only inside the VPC / over the IAP SSH tunnel. - ext := external.New(time.Duration(envInt("EXTERNAL_TURN_TIMEOUT_S", 1200)) * time.Second) - ag.SetExternal(ext) + // EXTERNAL_TURN_TIMEOUT_S. Off unless EXTERNAL_ADDR is set; the endpoints live on that separate listener, which + // the VM compose file maps to 127.0.0.1 only (reachable through the IAP SSH tunnel, never from the VPC): a reply + // is a diagnosis and, under auto-approval, an executed action. + var ext *external.Responder + if addr := env("EXTERNAL_ADDR", ""); addr != "" { + ext = external.New(time.Duration(envInt("EXTERNAL_TURN_TIMEOUT_S", 1200)) * time.Second) + ag.SetExternal(ext) + esrv := &http.Server{Addr: addr, Handler: logging(ext.Handler()), ReadHeaderTimeout: 10 * time.Second} + go func() { log.Fatal(esrv.ListenAndServe()) }() + log.Printf("external responder: listening on %s", addr) + } externalDiagnoseTimeout := time.Duration(envInt("EXTERNAL_DIAGNOSE_TIMEOUT_S", 1500)) * time.Second jevMode := env("JEV_MODE", "off") // off | shadow | enforce (shadow records Jev answers next to the diagnosis; enforce also applies the two review rules) var jc *jev.Client @@ -166,7 +174,11 @@ func main() { if f := anth.Federation(); f != nil { out["anthropic_federation"] = f.Probe(c, anth.BaseURL) // one real token exchange; the Console history page shows the deny reason on failure } - out["external_responder"] = ext.Status() + if ext != nil { + out["external_responder"] = ext.Status() + } else { + out["external_responder"] = "off" + } out["jev_mode"] = jevMode if jc != nil { out["jev_key_present"] = jc.KeyPresent() @@ -222,8 +234,6 @@ func main() { ag.SaveShadow(context.Background(), req, res.SessionID, res, tri, rev) writeJSON(w, 200, res) }) - mux.HandleFunc("GET /external/next", ext.Next) - mux.HandleFunc("POST /external/reply", ext.ReplyHandler) srv := &http.Server{Addr: env("AGENT_ADDR", ":8080"), Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second} log.Fatal(srv.ListenAndServe()) } diff --git a/ai/internal/external/responder.go b/ai/internal/external/responder.go index 8238c8f..ab0cc37 100644 --- a/ai/internal/external/responder.go +++ b/ai/internal/external/responder.go @@ -3,10 +3,18 @@ // GET /external/next (long poll); the client answers with POST /external/reply. Used for reference measurements with // a model that has no API path (2026-10-16: Claude Mythos 5.1 driven by a Claude Code subagent). Token usage is not // known for such turns and is reported as zero; reports mark those runs as reference records. +// +// A reply becomes a diagnosis and, with auto-approval, an executed action, so whoever can answer a turn controls the +// lab: the endpoints are served on their own listener (EXTERNAL_ADDR, off by default; the VM maps it to 127.0.0.1 so +// only the IAP SSH tunnel reaches it), turn ids are random, and every reply must carry the per-turn reply_token that +// only the client who took the turn received. package external import ( "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -24,7 +32,8 @@ const maxConvCached = 64 // on the first turn, so it is carried along), the tool definitions and whether the final submit_diagnosis is forced. type Turn struct { TurnID string `json:"turn_id"` - Session string `json:"session"` // first response id of the conversation ("" on the first turn) + ReplyToken string `json:"reply_token"` // random, handed out with the turn, required by /external/reply + Session string `json:"session"` // first response id of the conversation ("" on the first turn) Model string `json:"model"` Turn int `json:"turn"` // 1-based index within the conversation System string `json:"system"` @@ -38,9 +47,10 @@ type Turn struct { // Reply is the client's answer: function calls (name + arguments) and/or assistant text. type Reply struct { - TurnID string `json:"turn_id"` - Output []replyItem `json:"output"` - Error string `json:"error,omitempty"` // client gave up on this turn + TurnID string `json:"turn_id"` + ReplyToken string `json:"reply_token"` + Output []replyItem `json:"output"` + Error string `json:"error,omitempty"` // client gave up on this turn } type replyItem struct { @@ -115,11 +125,11 @@ func (r *Responder) CreateResponse(ctx context.Context, req openai.Request) (*op if m, ok := req.ToolChoice.(map[string]any); ok && m["type"] == "function" { forced = true } + id, tok := "ext_"+randHex(8), randHex(16) r.mu.Lock() r.seq++ - id := fmt.Sprintf("ext_%d", r.seq) now := time.Now() - p := &pending{turn: Turn{TurnID: id, Session: hist.first, Model: req.Model, Turn: hist.turn + 1, System: hist.system, Input: hist.items, Tools: req.Tools, + p := &pending{turn: Turn{TurnID: id, ReplyToken: tok, Session: hist.first, Model: req.Model, Turn: hist.turn + 1, System: hist.system, Input: hist.items, Tools: req.Tools, ForceSubmit: forced, MaxOutputTokens: req.MaxOutputTokens, IssuedAt: now, DeadlineAt: now.Add(r.TurnTimeout)}, reply: make(chan Reply, 1)} r.queue = append(r.queue, p) r.lastTurn = now @@ -269,6 +279,11 @@ func (r *Responder) ReplyHandler(w http.ResponseWriter, req *http.Request) { } r.mu.Lock() p, ok := r.taken[rep.TurnID] + if ok && subtle.ConstantTimeCompare([]byte(p.turn.ReplyToken), []byte(rep.ReplyToken)) != 1 { + r.mu.Unlock() + writeJSON(w, 403, map[string]string{"error": "reply_token does not match turn " + rep.TurnID}) + return + } if ok { delete(r.taken, rep.TurnID) } @@ -295,6 +310,22 @@ func (r *Responder) Status() map[string]any { return out } +func randHex(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + panic("crypto/rand: " + err.Error()) + } + return hex.EncodeToString(b) +} + +// Handler serves the two endpoints; mount it on the dedicated external listener only. +func (r *Responder) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /external/next", r.Next) + mux.HandleFunc("POST /external/reply", r.ReplyHandler) + return mux +} + func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) diff --git a/ai/internal/external/responder_test.go b/ai/internal/external/responder_test.go index 06cd2d2..d9c9cfb 100644 --- a/ai/internal/external/responder_test.go +++ b/ai/internal/external/responder_test.go @@ -18,10 +18,7 @@ import ( // the forced-submit flag; the reply's arguments may be a JSON string or object. func TestTurnsOverHTTP(t *testing.T) { r := New(5 * time.Second) - mux := http.NewServeMux() - mux.HandleFunc("GET /external/next", r.Next) - mux.HandleFunc("POST /external/reply", r.ReplyHandler) - srv := httptest.NewServer(mux) + srv := httptest.NewServer(r.Handler()) defer srv.Close() tools := []openai.Tool{{Type: "function", Strict: true, Name: "get_probe_status", Parameters: map[string]any{"type": "object"}}} type res struct { @@ -35,14 +32,18 @@ func TestTurnsOverHTTP(t *testing.T) { ch <- res{resp, err} }() turn := poll(t, srv.URL) - if turn.TurnID != "ext_1" || turn.Session != "" || turn.Turn != 1 || turn.System != "SYS" || len(turn.Input) != 1 || turn.Input[0].Content != "alert" || len(turn.Tools) != 1 || turn.ForceSubmit || turn.MaxOutputTokens != 4000 { + if !strings.HasPrefix(turn.TurnID, "ext_") || len(turn.TurnID) != 20 || len(turn.ReplyToken) != 32 || turn.Session != "" || turn.Turn != 1 || turn.System != "SYS" || len(turn.Input) != 1 || turn.Input[0].Content != "alert" || len(turn.Tools) != 1 || turn.ForceSubmit || turn.MaxOutputTokens != 4000 { t.Fatalf("turn 1: %+v", turn) } // second poll while nothing is queued -> 204 within the wait if code := get(t, srv.URL+"/external/next?wait=0"); code != 204 { t.Fatalf("empty queue must answer 204, got %d", code) } - code, body := post(t, srv.URL+"/external/reply", `{"turn_id":"ext_1","output":[{"type":"message","text":"checking"},{"type":"function_call","name":"get_probe_status","arguments":"{}"}]}`) + // a reply without the turn's token is refused (the turn stays open) + if code, _ := post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn.TurnID+`","reply_token":"wrong","output":[{"type":"message","text":"x"}]}`); code != 403 { + t.Fatalf("wrong token must be 403, got %d", code) + } + code, body := post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn.TurnID+`","reply_token":"`+turn.ReplyToken+`","output":[{"type":"message","text":"checking"},{"type":"function_call","name":"get_probe_status","arguments":"{}"}]}`) if code != 200 { t.Fatalf("reply 1: %d %s", code, body) } @@ -51,7 +52,7 @@ func TestTurnsOverHTTP(t *testing.T) { t.Fatal(r1.err) } fcs := r1.resp.FunctionCalls() - if r1.resp.ID != "ext_1" || len(fcs) != 1 || fcs[0].Name != "get_probe_status" || fcs[0].Arguments != "{}" || fcs[0].CallID == "" || r1.resp.Usage.TotalTokens != 0 { + if r1.resp.ID != turn.TurnID || len(fcs) != 1 || fcs[0].Name != "get_probe_status" || fcs[0].Arguments != "{}" || fcs[0].CallID == "" || r1.resp.Usage.TotalTokens != 0 { t.Fatalf("response 1: %+v", r1.resp) } if !strings.Contains(r1.resp.Text(), "checking") { @@ -59,20 +60,20 @@ func TestTurnsOverHTTP(t *testing.T) { } go func() { - resp, err := r.CreateResponse(context.Background(), openai.Request{Model: "external-claude-mythos-5-1", Tools: tools, PreviousResponseID: "ext_1", + resp, err := r.CreateResponse(context.Background(), openai.Request{Model: "external-claude-mythos-5-1", Tools: tools, PreviousResponseID: turn.TurnID, ToolChoice: map[string]any{"type": "function", "name": "submit_diagnosis"}, Input: []openai.Item{{Type: "function_call_output", CallID: fcs[0].CallID, Output: `{"alb":1}`}}}) ch <- res{resp, err} }() turn2 := poll(t, srv.URL) - if turn2.TurnID != "ext_2" || turn2.Session != "ext_1" || turn2.Turn != 2 || turn2.System != "SYS" || !turn2.ForceSubmit { + if turn2.TurnID == turn.TurnID || turn2.Session != turn.TurnID || turn2.Turn != 2 || turn2.System != "SYS" || !turn2.ForceSubmit { t.Fatalf("turn 2 header: %+v", turn2) } // history: user alert / assistant text / assistant function_call / function_call_output if len(turn2.Input) != 4 || turn2.Input[2].Type != "function_call" || turn2.Input[2].CallID != fcs[0].CallID || turn2.Input[3].Type != "function_call_output" || turn2.Input[3].Output != `{"alb":1}` { t.Fatalf("turn 2 history: %s", mustJSON(turn2.Input)) } - code, body = post(t, srv.URL+"/external/reply", `{"turn_id":"ext_2","output":[{"type":"function_call","name":"submit_diagnosis","arguments":{"root_cause_label":"x","confidence":0.9}}]}`) + code, body = post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn2.TurnID+`","reply_token":"`+turn2.ReplyToken+`","output":[{"type":"function_call","name":"submit_diagnosis","arguments":{"root_cause_label":"x","confidence":0.9}}]}`) if code != 200 { t.Fatalf("reply 2: %d %s", code, body) } @@ -84,7 +85,7 @@ func TestTurnsOverHTTP(t *testing.T) { t.Fatalf("response 2: %+v", r2.resp.Output) } // an answered turn cannot be answered twice; malformed replies are refused before touching the queue - if code, _ := post(t, srv.URL+"/external/reply", `{"turn_id":"ext_2","output":[{"type":"message","text":"again"}]}`); code != 404 { + if code, _ := post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn2.TurnID+`","reply_token":"`+turn2.ReplyToken+`","output":[{"type":"message","text":"again"}]}`); code != 404 { t.Fatalf("second reply must be 404, got %d", code) } if code, _ := post(t, srv.URL+"/external/reply", `{"turn_id":"ext_9","output":[{"type":"function_call","name":"x","arguments":"[1]"}]}`); code != 400 { @@ -118,10 +119,7 @@ func TestDeadlinesAndErrors(t *testing.T) { } // error reply r2 := New(5 * time.Second) - mux := http.NewServeMux() - mux.HandleFunc("GET /external/next", r2.Next) - mux.HandleFunc("POST /external/reply", r2.ReplyHandler) - srv := httptest.NewServer(mux) + srv := httptest.NewServer(r2.Handler()) defer srv.Close() ch := make(chan error, 1) go func() { @@ -129,7 +127,7 @@ func TestDeadlinesAndErrors(t *testing.T) { ch <- err }() turn := poll(t, srv.URL) - post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn.TurnID+`","error":"gave up"}`) + post(t, srv.URL+"/external/reply", `{"turn_id":"`+turn.TurnID+`","reply_token":"`+turn.ReplyToken+`","error":"gave up"}`) if err := <-ch; err == nil || !strings.Contains(err.Error(), "gave up") { t.Fatalf("expected client error, got %v", err) }