feat: per-run model selection, usage in diagnosis results and a Claude Messages API provider
캠페인마다 진단 모델을 고를 수 있게 한다. 하네스는 runs.model 열을 추가해 POST /runs의 model을 저장하고, 활성 런의 model을 /diagnose 요청에 실어 보낸다. 에이전트는 요청의 model을 우선 사용하고(없으면 OPENAI_MODEL 기본값), 모델 id가 claude-로 시작하면 internal/anthropic(Claude Messages API 어댑터)로 라우팅한다. 어댑터는 Responses API 항목(message/function_call/function_call_output)을 system·tools(input_schema)·tool_use/ tool_result로 변환하고 previous_response_id 체인을 클라이언트 쪽에서 재현하며, Claude 5.5가 강제 tool_choice를 거부하므로 강제 호출은 auto + 지시문으로 바꾼다. 키는 ANTHROPIC_KEY_FILE(/run/secrets/anthropic-api-key)에서 읽고 비어 있으면 명확한 오류를 낸다. Result에 usage(입력·출력·합계 토큰)를 추가해 하네스 diagnosis 이벤트에 비용 근거를 남긴다.
This commit is contained in:
seon committed
1 parent
fa9c08c778
commit
53ff243183
8 files changed
+492
-12
No files matched your search
@@ -0,0 +1,262 @@
|
||||
// Package anthropic adapts the Claude Messages API to the small Responses-API surface the agent loop uses
|
||||
// (openai.Request / openai.Response with message, function_call and function_call_output items), so one loop
|
||||
// serves both providers. The agent picks this client for model ids starting with "claude-".
|
||||
//
|
||||
// Wire format (Messages API reference, anthropic-version 2023-06-01): system prompt as the top-level `system`
|
||||
// field, tools as {name, description, input_schema, strict}, tool calls as `tool_use` content blocks answered by
|
||||
// `tool_result` blocks in the next user message. The Responses-API `previous_response_id` chain is emulated
|
||||
// client-side: every response id maps to the full message list up to and including that assistant turn.
|
||||
//
|
||||
// Claude Opus 5.5 / Sonnet 5.5 reject forced tool use (tool_choice type "tool"/"any" -> 400), so a forced
|
||||
// tool_choice from the loop becomes {type:"auto"} plus a one-line instruction in the user turn.
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/openai"
|
||||
)
|
||||
|
||||
const (
|
||||
apiVersion = "2023-06-01"
|
||||
maxConvCached = 64 // response ids kept for previous_response_id chaining (one diagnosis uses <= MaxTurns)
|
||||
defaultTokens = 4000
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
BaseURL string
|
||||
HTTP *http.Client
|
||||
keyFn func() (string, error)
|
||||
|
||||
mu sync.Mutex
|
||||
conv map[string]conversation // response id -> messages (and system prompt) up to and including that assistant turn
|
||||
order []string // insertion order for eviction
|
||||
seq int
|
||||
}
|
||||
|
||||
type conversation struct {
|
||||
system string
|
||||
msgs []message
|
||||
}
|
||||
|
||||
func New(baseURL string, keyFn func() (string, error)) *Client {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://api.anthropic.com"
|
||||
}
|
||||
return &Client{BaseURL: strings.TrimRight(baseURL, "/"), HTTP: &http.Client{Timeout: 150 * time.Second}, keyFn: keyFn, conv: map[string]conversation{}}
|
||||
}
|
||||
|
||||
// KeyPresent reports whether the key file has a non-empty value (the secret may not be registered yet).
|
||||
func (c *Client) KeyPresent() bool {
|
||||
k, err := c.keyFn()
|
||||
return err == nil && k != ""
|
||||
}
|
||||
|
||||
type message struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"` // string or []block
|
||||
}
|
||||
|
||||
type block struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ID string `json:"id,omitempty"` // tool_use
|
||||
Name string `json:"name,omitempty"` // tool_use
|
||||
Input json.RawMessage `json:"input,omitempty"` // tool_use
|
||||
ToolUseID string `json:"tool_use_id,omitempty"` // tool_result
|
||||
Content string `json:"content,omitempty"` // tool_result
|
||||
}
|
||||
|
||||
type tool struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
InputSchema map[string]any `json:"input_schema"`
|
||||
Strict bool `json:"strict,omitempty"`
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Model string `json:"model"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
System string `json:"system,omitempty"`
|
||||
Messages []message `json:"messages"`
|
||||
Tools []tool `json:"tools,omitempty"`
|
||||
ToolChoice map[string]any `json:"tool_choice,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Model string `json:"model"`
|
||||
StopReason string `json:"stop_reason"`
|
||||
Content []block `json:"content"`
|
||||
Usage struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
|
||||
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
||||
} `json:"usage"`
|
||||
Error *struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// forcedNudge is appended to the user turn when the loop asked for a forced submit_diagnosis call.
|
||||
const forcedNudge = "You must call the submit_diagnosis tool now with your final answer; do not call any other tool."
|
||||
|
||||
// CreateResponse translates one loop turn to a Messages API call and the answer back to Responses-API items.
|
||||
func (c *Client) CreateResponse(ctx context.Context, req openai.Request) (*openai.Response, error) {
|
||||
key, err := c.keyFn()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("anthropic key: %w", err)
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("anthropic api key is empty (secret selfheal-anthropic-api-key not registered yet)")
|
||||
}
|
||||
var msgs []message
|
||||
var system []string
|
||||
if req.PreviousResponseID != "" {
|
||||
c.mu.Lock()
|
||||
prev, ok := c.conv[req.PreviousResponseID]
|
||||
c.mu.Unlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("anthropic: unknown previous_response_id %q", req.PreviousResponseID)
|
||||
}
|
||||
msgs = append(msgs, prev.msgs...)
|
||||
if prev.system != "" {
|
||||
system = append(system, prev.system) // the loop sends the system message only on the first turn
|
||||
}
|
||||
}
|
||||
var userBlocks []block
|
||||
flush := func() {
|
||||
if len(userBlocks) == 0 {
|
||||
return
|
||||
}
|
||||
msgs = append(msgs, message{Role: "user", Content: userBlocks})
|
||||
userBlocks = nil
|
||||
}
|
||||
for _, it := range req.Input {
|
||||
switch it.Type {
|
||||
case "message":
|
||||
text, _ := it.Content.(string)
|
||||
switch it.Role {
|
||||
case "system":
|
||||
system = append(system, text)
|
||||
case "assistant":
|
||||
flush()
|
||||
msgs = append(msgs, message{Role: "assistant", Content: []block{{Type: "text", Text: text}}})
|
||||
default:
|
||||
userBlocks = append(userBlocks, block{Type: "text", Text: text})
|
||||
}
|
||||
case "function_call_output":
|
||||
userBlocks = append(userBlocks, block{Type: "tool_result", ToolUseID: it.CallID, Content: it.Output})
|
||||
}
|
||||
}
|
||||
forced := false
|
||||
if m, ok := req.ToolChoice.(map[string]any); ok && m["type"] == "function" {
|
||||
forced = true
|
||||
}
|
||||
if forced {
|
||||
userBlocks = append(userBlocks, block{Type: "text", Text: forcedNudge})
|
||||
}
|
||||
flush()
|
||||
if len(msgs) == 0 {
|
||||
return nil, fmt.Errorf("anthropic: no messages to send")
|
||||
}
|
||||
body := request{Model: req.Model, MaxTokens: req.MaxOutputTokens, System: strings.Join(system, "\n\n"), Messages: msgs}
|
||||
if body.MaxTokens <= 0 {
|
||||
body.MaxTokens = defaultTokens
|
||||
}
|
||||
for _, t := range req.Tools {
|
||||
body.Tools = append(body.Tools, tool{Name: t.Name, Description: t.Description, InputSchema: t.Parameters, Strict: t.Strict})
|
||||
}
|
||||
if len(body.Tools) > 0 {
|
||||
tc := map[string]any{"type": "auto"}
|
||||
if req.ParallelToolCalls != nil && !*req.ParallelToolCalls {
|
||||
tc["disable_parallel_tool_use"] = true
|
||||
}
|
||||
body.ToolChoice = tc
|
||||
}
|
||||
|
||||
b, _ := json.Marshal(body)
|
||||
hreq, err := http.NewRequestWithContext(ctx, "POST", c.BaseURL+"/v1/messages", bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hreq.Header.Set("x-api-key", key)
|
||||
hreq.Header.Set("anthropic-version", apiVersion)
|
||||
hreq.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.HTTP.Do(hreq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rb, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
var r response
|
||||
_ = json.Unmarshal(rb, &r)
|
||||
if resp.StatusCode/100 != 2 {
|
||||
msg := truncate(string(rb), 600)
|
||||
if r.Error != nil {
|
||||
msg = r.Error.Type + ": " + r.Error.Message
|
||||
}
|
||||
return nil, fmt.Errorf("anthropic /v1/messages: status %d: %s", resp.StatusCode, msg)
|
||||
}
|
||||
if r.Error != nil {
|
||||
return nil, fmt.Errorf("anthropic response error: %s", r.Error.Message)
|
||||
}
|
||||
|
||||
out := &openai.Response{ID: r.ID, Status: "completed", Model: r.Model}
|
||||
if out.ID == "" {
|
||||
c.mu.Lock()
|
||||
c.seq++
|
||||
out.ID = fmt.Sprintf("msg_local_%d", c.seq)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
var text strings.Builder
|
||||
for _, bl := range r.Content {
|
||||
switch bl.Type {
|
||||
case "text":
|
||||
text.WriteString(bl.Text)
|
||||
case "tool_use":
|
||||
args := string(bl.Input)
|
||||
if args == "" {
|
||||
args = "{}"
|
||||
}
|
||||
out.Output = append(out.Output, openai.Item{Type: "function_call", CallID: bl.ID, Name: bl.Name, Arguments: args, ID: bl.ID, Status: "completed"})
|
||||
}
|
||||
}
|
||||
if text.Len() > 0 {
|
||||
out.Output = append([]openai.Item{{Type: "message", Role: "assistant", Content: []any{map[string]any{"type": "output_text", "text": text.String()}}}}, out.Output...)
|
||||
}
|
||||
// input tokens include cache writes/reads so the sum matches what is billed
|
||||
out.Usage.InputTokens = r.Usage.InputTokens + r.Usage.CacheCreationInputTokens + r.Usage.CacheReadInputTokens
|
||||
out.Usage.OutputTokens = r.Usage.OutputTokens
|
||||
out.Usage.TotalTokens = out.Usage.InputTokens + out.Usage.OutputTokens
|
||||
|
||||
// remember the conversation for the next turn (assistant content kept verbatim so tool_use ids line up)
|
||||
full := append(append([]message{}, msgs...), message{Role: "assistant", Content: r.Content})
|
||||
c.mu.Lock()
|
||||
c.conv[out.ID] = conversation{system: body.System, msgs: full}
|
||||
c.order = append(c.order, out.ID)
|
||||
for len(c.order) > maxConvCached {
|
||||
delete(c.conv, c.order[0])
|
||||
c.order = c.order[1:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "..."
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/openai"
|
||||
)
|
||||
|
||||
// Two turns through a fake Messages API: the first asks for a tool, the second (with previous_response_id and a
|
||||
// forced submit) must carry the tool_result, the assistant tool_use block, the nudge text and an auto tool_choice.
|
||||
func TestTranslateTurns(t *testing.T) {
|
||||
var bodies []map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/messages" || r.Header.Get("x-api-key") != "k1" || r.Header.Get("anthropic-version") != "2023-06-01" {
|
||||
t.Errorf("bad request line/headers: %s %v", r.URL.Path, r.Header)
|
||||
}
|
||||
var b map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&b)
|
||||
bodies = append(bodies, b)
|
||||
switch len(bodies) {
|
||||
case 1:
|
||||
w.Write([]byte(`{"id":"msg_1","type":"message","model":"claude-sonnet-5-5","stop_reason":"tool_use","content":[{"type":"text","text":"checking"},{"type":"tool_use","id":"toolu_1","name":"get_probe_status","input":{}}],"usage":{"input_tokens":100,"output_tokens":20,"cache_creation_input_tokens":5,"cache_read_input_tokens":0}}`))
|
||||
default:
|
||||
w.Write([]byte(`{"id":"msg_2","type":"message","model":"claude-sonnet-5-5","stop_reason":"tool_use","content":[{"type":"tool_use","id":"toolu_2","name":"submit_diagnosis","input":{"root_cause_label":"x","confidence":0.9}}],"usage":{"input_tokens":200,"output_tokens":30}}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := New(srv.URL, func() (string, error) { return "k1", nil })
|
||||
pf := false
|
||||
tools := []openai.Tool{{Type: "function", Strict: true, Name: "get_probe_status", Description: "probe", Parameters: map[string]any{"type": "object", "properties": map[string]any{}}}}
|
||||
r1, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-sonnet-5-5", MaxOutputTokens: 1234, ParallelToolCalls: &pf, Tools: tools, ToolChoice: "auto",
|
||||
Input: []openai.Item{{Type: "message", Role: "system", Content: "SYS"}, {Type: "message", Role: "user", Content: "alert"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fcs := r1.FunctionCalls()
|
||||
if r1.ID != "msg_1" || len(fcs) != 1 || fcs[0].CallID != "toolu_1" || fcs[0].Name != "get_probe_status" || fcs[0].Arguments != "{}" {
|
||||
t.Fatalf("turn 1 translation: %+v", r1)
|
||||
}
|
||||
if r1.Usage.InputTokens != 105 || r1.Usage.OutputTokens != 20 || r1.Usage.TotalTokens != 125 {
|
||||
t.Fatalf("usage: %+v", r1.Usage)
|
||||
}
|
||||
b1 := bodies[0]
|
||||
if b1["system"] != "SYS" || b1["max_tokens"].(float64) != 1234 || b1["model"] != "claude-sonnet-5-5" {
|
||||
t.Fatalf("body 1: %v", b1)
|
||||
}
|
||||
tl := b1["tools"].([]any)[0].(map[string]any)
|
||||
if tl["name"] != "get_probe_status" || tl["input_schema"] == nil || tl["strict"] != true {
|
||||
t.Fatalf("tool translation: %v", tl)
|
||||
}
|
||||
if tc := b1["tool_choice"].(map[string]any); tc["type"] != "auto" || tc["disable_parallel_tool_use"] != true {
|
||||
t.Fatalf("tool_choice 1: %v", tc)
|
||||
}
|
||||
|
||||
r2, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-sonnet-5-5", Tools: tools, PreviousResponseID: "msg_1",
|
||||
ToolChoice: map[string]any{"type": "function", "name": "submit_diagnosis"},
|
||||
Input: []openai.Item{{Type: "function_call_output", CallID: "toolu_1", Output: `{"alb":1}`}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := r2.FunctionCalls(); len(got) != 1 || got[0].Name != "submit_diagnosis" || !strings.Contains(got[0].Arguments, `"root_cause_label":"x"`) {
|
||||
t.Fatalf("turn 2 translation: %+v", r2.Output)
|
||||
}
|
||||
b2 := bodies[1]
|
||||
msgs := b2["messages"].([]any)
|
||||
if len(msgs) != 3 { // user(alert) / assistant(text+tool_use) / user(tool_result+nudge)
|
||||
t.Fatalf("messages: %v", msgs)
|
||||
}
|
||||
as := msgs[1].(map[string]any)
|
||||
if as["role"] != "assistant" || !strings.Contains(mustJSON(as["content"]), `"tool_use"`) {
|
||||
t.Fatalf("assistant turn not replayed: %v", as)
|
||||
}
|
||||
last := msgs[2].(map[string]any)["content"].([]any)
|
||||
if tr := last[0].(map[string]any); tr["type"] != "tool_result" || tr["tool_use_id"] != "toolu_1" || tr["content"] != `{"alb":1}` {
|
||||
t.Fatalf("tool_result: %v", tr)
|
||||
}
|
||||
if nd := last[1].(map[string]any); nd["type"] != "text" || !strings.Contains(nd["text"].(string), "submit_diagnosis") {
|
||||
t.Fatalf("forced nudge missing: %v", nd)
|
||||
}
|
||||
if tc := b2["tool_choice"].(map[string]any); tc["type"] != "auto" {
|
||||
t.Fatalf("forced tool_choice must become auto for Claude 5.5: %v", tc)
|
||||
}
|
||||
if b2["system"] != "SYS" {
|
||||
t.Fatalf("system prompt must be carried on the chained turn, got %v", b2["system"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyKeyAndErrors(t *testing.T) {
|
||||
c := New("http://127.0.0.1:1", func() (string, error) { return "", nil })
|
||||
if c.KeyPresent() {
|
||||
t.Fatal("empty key reported present")
|
||||
}
|
||||
_, err := c.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", Input: []openai.Item{{Type: "message", Role: "user", Content: "x"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "anthropic api key is empty") {
|
||||
t.Fatalf("expected empty-key error, got %v", err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(400)
|
||||
w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"tool_choice: type \"tool\" and \"any\" are not supported for this model."}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c2 := New(srv.URL, func() (string, error) { return "k", nil })
|
||||
_, err = c2.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", Input: []openai.Item{{Type: "message", Role: "user", Content: "x"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "invalid_request_error") {
|
||||
t.Fatalf("expected 400 error, got %v", err)
|
||||
}
|
||||
_, err = c2.CreateResponse(context.Background(), openai.Request{Model: "claude-haiku-5-5", PreviousResponseID: "nope", Input: []openai.Item{{Type: "function_call_output", CallID: "a", Output: "b"}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown previous_response_id") {
|
||||
t.Fatalf("expected unknown id error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
|
||||
Reference in new issue
Block a user