feat(ai): external responder for external-* model ids (reference runs)

모델 id가 external- 로 시작하면 ai-agent 가 LLM API 를 부르지 않고 각 턴(시스템
프롬프트·대화 이력·도구 정의·강제 제출 여부)을 GET /external/next 로 내어주고
POST /external/reply 로 답을 받는다. 진단 루프·프롬프트·도구·턴 상한은 다른 모델과
동일하므로, API 경로가 없는 모델(Claude Mythos 5.1 을 Claude Code 서브에이전트가
구동)의 참고 측정에 쓴다. 토큰 사용량은 알 수 없어 0 으로 기록한다.

external-* 진단은 EXTERNAL_DIAGNOSE_TIMEOUT_S(기본 1500초), 한 턴은
EXTERNAL_TURN_TIMEOUT_S(기본 1200초)까지 기다리고, 하네스는 같은 모델에 26분을
기다린다. /selfcheck 에 external_responder 상태(대기 턴·마지막 폴링 시각)를 추가.
This commit is contained in:
seon committed 2026-10-16 09:05:12 +09:00
1 parent 85d99324a1
commit ed3d41d399
8 files changed
+585 -6

No files matched your search

+26 -1
View File
@@ -7,12 +7,14 @@ import (
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/agent"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/anthropic"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/diag"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/external"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/harness"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/jev"
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/openai"
@@ -20,6 +22,15 @@ import (
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/tools"
)
func envInt(k string, d int) int {
if v := os.Getenv(k); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return d
}
func env(k, d string) string {
if v := os.Getenv(k); v != "" {
return v
@@ -108,6 +119,13 @@ func main() {
})
ag.SetAnthropic(anth)
log.Printf("anthropic auth: %s", anth.AuthMode())
// external-* model ids (reference runs, 2026-10-16): each loop turn is served to an outside client over
// GET /external/next and answered through POST /external/reply (internal/external). The diagnosis deadline is
// EXTERNAL_DIAGNOSE_TIMEOUT_S (the harness waits a little longer for external-* models), one turn at most
// EXTERNAL_TURN_TIMEOUT_S. The endpoints are reachable only inside the VPC / over the IAP SSH tunnel.
ext := external.New(time.Duration(envInt("EXTERNAL_TURN_TIMEOUT_S", 1200)) * time.Second)
ag.SetExternal(ext)
externalDiagnoseTimeout := time.Duration(envInt("EXTERNAL_DIAGNOSE_TIMEOUT_S", 1500)) * time.Second
jevMode := env("JEV_MODE", "off") // off | shadow | enforce (shadow records Jev answers next to the diagnosis; enforce also applies the two review rules)
var jc *jev.Client
if jevMode == "shadow" || jevMode == "enforce" {
@@ -148,6 +166,7 @@ func main() {
if f := anth.Federation(); f != nil {
out["anthropic_federation"] = f.Probe(c, anth.BaseURL) // one real token exchange; the Console history page shows the deny reason on failure
}
out["external_responder"] = ext.Status()
out["jev_mode"] = jevMode
if jc != nil {
out["jev_key_present"] = jc.KeyPresent()
@@ -180,7 +199,11 @@ func main() {
}
inflight++
defer func() { inflight-- }()
c, cancel := context.WithTimeout(context.Background(), 170*time.Second) // harness waits 3 minutes
timeout := 170 * time.Second // harness waits 3 minutes
if agent.IsExternal(req.Model) {
timeout = externalDiagnoseTimeout
}
c, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
prefetched := ag.Prefetch(c)
tri := ag.ShadowTriage(c, req, prefetched) // nil unless JEV_MODE=shadow; recorded only
@@ -199,6 +222,8 @@ func main() {
ag.SaveShadow(context.Background(), req, res.SessionID, res, tri, rev)
writeJSON(w, 200, res)
})
mux.HandleFunc("GET /external/next", ext.Next)
mux.HandleFunc("POST /external/reply", ext.ReplyHandler)
srv := &http.Server{Addr: env("AGENT_ADDR", ":8080"), Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second}
log.Fatal(srv.ListenAndServe())
}