Files
seon f8e0aa9122 feat: alert dedupe, proposal state API, HITL mirrors API approvals, Jev recovering reads the alert condition
第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
  /diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
  出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
  全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
  メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
  直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
  recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
  障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
  route フィルタを付け、判定と同じ式にそろえる。
2026-10-09 08:33:41 +09:00

144 lines
4.6 KiB
Go

// Package harness talks to the harness runner (design 5.2): proposals, approval, execution, catalogue, scenarios.
package harness
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
type Client struct {
BaseURL string // http://10.20.0.11:8080
HTTP *http.Client
}
func New(base string) *Client {
return &Client{BaseURL: strings.TrimRight(base, "/"), HTTP: &http.Client{Timeout: 11 * time.Minute}}
}
func (c *Client) call(ctx context.Context, method, path string, body any, out any) (int, error) {
var rd io.Reader
if body != nil {
b, _ := json.Marshal(body)
rd = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, rd)
if err != nil {
return 0, err
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.HTTP.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
rb, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if out != nil && len(rb) > 0 {
if err := json.Unmarshal(rb, out); err != nil {
return resp.StatusCode, fmt.Errorf("harness %s %s: status %d, body not JSON: %s", method, path, resp.StatusCode, truncate(string(rb), 300))
}
}
if resp.StatusCode/100 != 2 {
return resp.StatusCode, fmt.Errorf("harness %s %s: status %d: %s", method, path, resp.StatusCode, truncate(string(rb), 300))
}
return resp.StatusCode, nil
}
type Proposal struct {
Token string `json:"token"`
ExpiresIn float64 `json:"expires_in_s"`
Approved bool `json:"approved"`
}
// ProposalState is GET /proposals/{token}: the decision as the harness knows it (approval may come through the API).
type ProposalState struct {
Token string `json:"token"`
ActionID string `json:"action_id"`
Approved bool `json:"approved"`
ApprovedAt *string `json:"approved_at"`
ApprovedBy *string `json:"approved_by"`
Executed bool `json:"executed"`
ExecutedAt *string `json:"executed_at"`
ExpiresAt string `json:"expires_at"`
}
// GetProposal reads the harness-side state of a proposal.
func (c *Client) GetProposal(ctx context.Context, token string) (*ProposalState, error) {
var out ProposalState
if _, err := c.call(ctx, "GET", "/proposals/"+token, nil, &out); err != nil {
return nil, err
}
return &out, nil
}
// Propose registers a layer-2 proposal (POST /proposals); the harness records the proposal event and returns the token.
func (c *Client) Propose(ctx context.Context, runID, scenarioID *string, actionID string, target, diagnosis map[string]any, ttl time.Duration) (*Proposal, error) {
body := map[string]any{"run_id": runID, "scenario_id": scenarioID, "action_id": actionID, "target": target, "diagnosis": diagnosis}
if ttl > 0 {
body["ttl_seconds"] = int(ttl.Seconds())
}
var p Proposal
if _, err := c.call(ctx, "POST", "/proposals", body, &p); err != nil {
return nil, err
}
return &p, nil
}
// Approve marks the proposal approved (POST /proposals/{token}/approve); the harness records the approval event.
func (c *Client) Approve(ctx context.Context, token, by string) error {
_, err := c.call(ctx, "POST", "/proposals/"+token+"/approve", map[string]any{"approved_by": by}, nil)
return err
}
// Reject records the rejection (POST /proposals/{token}/reject).
func (c *Client) Reject(ctx context.Context, token, by, reason string) error {
_, err := c.call(ctx, "POST", "/proposals/"+token+"/reject", map[string]any{"rejected_by": by, "reason": reason}, nil)
return err
}
// Execute runs the catalogue action with the approved token (POST /actions/execute). The body is returned as-is for the
// Discord message; a non-2xx status is an error with the body text.
func (c *Client) Execute(ctx context.Context, token, actionID string, target map[string]any, runID, scenarioID *string, note string) (map[string]any, error) {
body := map[string]any{"action_id": actionID, "target": target, "token": token, "run_id": runID, "scenario_id": scenarioID, "note": note}
var out map[string]any
_, err := c.call(ctx, "POST", "/actions/execute", body, &out)
return out, err
}
type CatalogEntry struct {
ActionID string `json:"action_id"`
Layer string `json:"layer"`
Exec bool `json:"harness_executes"`
}
func (c *Client) Catalog(ctx context.Context) ([]CatalogEntry, error) {
var out []CatalogEntry
_, err := c.call(ctx, "GET", "/actions/catalog", nil, &out)
return out, err
}
type Scenario struct {
ID string `json:"scenario_id"`
Label string `json:"ground_truth_label"`
}
func (c *Client) Scenarios(ctx context.Context) ([]Scenario, error) {
var out []Scenario
_, err := c.call(ctx, "GET", "/scenarios", nil, &out)
return out, err
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "..."
}