feat: per-run model selection, usage in diagnosis results and a Claude Messages API provider
캠페인마다 진단 모델을 고를 수 있게 한다. 하네스는 runs.model 열을 추가해 POST /runs의 model을 저장하고, 활성 런의 model을 /diagnose 요청에 실어 보낸다. 에이전트는 요청의 model을 우선 사용하고(없으면 OPENAI_MODEL 기본값), 모델 id가 claude-로 시작하면 internal/anthropic(Claude Messages API 어댑터)로 라우팅한다. 어댑터는 Responses API 항목(message/function_call/function_call_output)을 system·tools(input_schema)·tool_use/ tool_result로 변환하고 previous_response_id 체인을 클라이언트 쪽에서 재현하며, Claude 5.5가 강제 tool_choice를 거부하므로 강제 호출은 auto + 지시문으로 바꾼다. 키는 ANTHROPIC_KEY_FILE(/run/secrets/anthropic-api-key)에서 읽고 비어 있으면 명확한 오류를 낸다. Result에 usage(입력·출력·합계 토큰)를 추가해 하네스 diagnosis 이벤트에 비용 근거를 남긴다.
This commit is contained in:
seon committed
1 parent
fa9c08c778
commit
53ff243183
8 files changed
+492
-12
No files matched your search
@@ -0,0 +1,118 @@
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/openai"
|
||||
)
|
||||
|
||||
// Two turns through a fake Messages API: the first asks for a tool, the second (with previous_response_id and a
|
||||
// forced submit) must carry the tool_result, the assistant tool_use block, the nudge text and an auto tool_choice.
|
||||
func TestTranslateTurns(t *testing.T) {
|
||||
var bodies []map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/messages" || r.Header.Get("x-api-key") != "k1" || r.Header.Get("anthropic-version") != "2023-06-01" {
|
||||
t.Errorf("bad request line/headers: %s %v", r.URL.Path, r.Header)
|
||||
}
|
||||
var b map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&b)
|
||||
bodies = append(bodies, b)
|
||||
switch len(bodies) {
|
||||
case 1:
|
||||
w.Write([]byte(`{"id":"msg_1","type":"message","model":"claude-sonnet-5-5","stop_reason":"tool_use","content":[{"type":"text","text":"checking"},{"type":"tool_use","id":"toolu_1","name":"get_probe_status","input":{}}],"usage":{"input_tokens":100,"output_tokens":20,"cache_creation_input_tokens":5,"cache_read_input_tokens":0}}`))
|
||||
default:
|
||||
w.Write([]byte(`{"id":"msg_2","type":"message","model":"claude-sonnet-5-5","stop_reason":"tool_use","content":[{"type":"tool_use","id":"toolu_2","name":"submit_diagnosis","input":{"root_cause_label":"x","confidence":0.9}}],"usage":{"input_tokens":200,"output_tokens":30}}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := New(srv.URL, func() (string, error) { return "k1", nil })
|
||||
pf := false
|
||||
tools := []openai.Tool{{Type: "function", Strict: true, Name: "get_probe_status", Description: "probe", Parameters: map[string]any{"type": "object", "properties": map[string]any{}}}}
|
||||
r1, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-sonnet-5-5", MaxOutputTokens: 1234, ParallelToolCalls: &pf, Tools: tools, ToolChoice: "auto",
|
||||
Input: []openai.Item{{Type: "message", Role: "system", Content: "SYS"}, {Type: "message", Role: "user", Content: "alert"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fcs := r1.FunctionCalls()
|
||||
if r1.ID != "msg_1" || len(fcs) != 1 || fcs[0].CallID != "toolu_1" || fcs[0].Name != "get_probe_status" || fcs[0].Arguments != "{}" {
|
||||
t.Fatalf("turn 1 translation: %+v", r1)
|
||||
}
|
||||
if r1.Usage.InputTokens != 105 || r1.Usage.OutputTokens != 20 || r1.Usage.TotalTokens != 125 {
|
||||
t.Fatalf("usage: %+v", r1.Usage)
|
||||
}
|
||||
b1 := bodies[0]
|
||||
if b1["system"] != "SYS" || b1["max_tokens"].(float64) != 1234 || b1["model"] != "claude-sonnet-5-5" {
|
||||
t.Fatalf("body 1: %v", b1)
|
||||
}
|
||||
tl := b1["tools"].([]any)[0].(map[string]any)
|
||||
if tl["name"] != "get_probe_status" || tl["input_schema"] == nil || tl["strict"] != true {
|
||||
t.Fatalf("tool translation: %v", tl)
|
||||
}
|
||||
if tc := b1["tool_choice"].(map[string]any); tc["type"] != "auto" || tc["disable_parallel_tool_use"] != true {
|
||||
t.Fatalf("tool_choice 1: %v", tc)
|
||||
}
|
||||
|
||||
r2, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-sonnet-5-5", Tools: tools, PreviousResponseID: "msg_1",
|
||||
ToolChoice: map[string]any{"type": "function", "name": "submit_diagnosis"},
|
||||
Input: []openai.Item{{Type: "function_call_output", CallID: "toolu_1", Output: `{"alb":1}`}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := r2.FunctionCalls(); len(got) != 1 || got[0].Name != "submit_diagnosis" || !strings.Contains(got[0].Arguments, `"root_cause_label":"x"`) {
|
||||
t.Fatalf("turn 2 translation: %+v", r2.Output)
|
||||
}
|
||||
b2 := bodies[1]
|
||||
msgs := b2["messages"].([]any)
|
||||
if len(msgs) != 3 { // user(alert) / assistant(text+tool_use) / user(tool_result+nudge)
|
||||
t.Fatalf("messages: %v", msgs)
|
||||
}
|
||||
as := msgs[1].(map[string]any)
|
||||
if as["role"] != "assistant" || !strings.Contains(mustJSON(as["content"]), `"tool_use"`) {
|
||||
t.Fatalf("assistant turn not replayed: %v", as)
|
||||
}
|
||||
last := msgs[2].(map[string]any)["content"].([]any)
|
||||
if tr := last[0].(map[string]any); tr["type"] != "tool_result" || tr["tool_use_id"] != "toolu_1" || tr["content"] != `{"alb":1}` {
|
||||
t.Fatalf("tool_result: %v", tr)
|
||||
}
|
||||
if nd := last[1].(map[string]any); nd["type"] != "text" || !strings.Contains(nd["text"].(string), "submit_diagnosis") {
|
||||
t.Fatalf("forced nudge missing: %v", nd)
|
||||
}
|
||||
if tc := b2["tool_choice"].(map[string]any); tc["type"] != "auto" {
|
||||
t.Fatalf("forced tool_choice must become auto for Claude 5.5: %v", tc)
|
||||
}
|
||||
if b2["system"] != "SYS" {
|
||||
t.Fatalf("system prompt must be carried on the chained turn, got %v", b2["system"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyKeyAndErrors(t *testing.T) {
|
||||
c := New("http://127.0.0.1:1", func() (string, error) { return "", nil })
|
||||
if c.KeyPresent() {
|
||||
t.Fatal("empty key reported present")
|
||||
}
|
||||
_, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", Input: []openai.Item{{Type: "message", Role: "user", Content: "x"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "anthropic api key is empty") {
|
||||
t.Fatalf("expected empty-key error, got %v", err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(400)
|
||||
w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"tool_choice: type \"tool\" and \"any\" are not supported for this model."}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c2 := New(srv.URL, func() (string, error) { return "k", nil })
|
||||
_, err = c2.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", Input: []openai.Item{{Type: "message", Role: "user", Content: "x"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "invalid_request_error") {
|
||||
t.Fatalf("expected 400 error, got %v", err)
|
||||
}
|
||||
_, err = c2.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", PreviousResponseID: "nope", Input: []openai.Item{{Type: "function_call_output", CallID: "a", Output: "b"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown previous_response_id") {
|
||||
t.Fatalf("expected unknown id error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
|
||||
Reference in new issue
Block a user