第 4 回キャンペーン(2026-10-08)の 3 つの穴を塞ぐ。
- ハーネス: 同じアラート名が ALERT_DEDUPE_S(既定 300 秒)以内に再着したら
/diagnose を送らず note(dedupe) に記録する。Grafana は Pod ごとに 1 件ずつ
出し 30 秒後にグループを再送するため、1 障害が診断 4 件・提案 4 件になっていた。
全件 resolved になれば窓をリセットする。
- ハーネス: GET /proposals/{token} で承認・実行状態を返す。
- hitl: 60 秒ごとに未決の提案をハーネスへ照会し、API で承認された件は
メッセージを「ハーネス API で承認」に書き換えてタイマーを外す。期限切れ処理の
直前にも同じ照会を行う。
- AI: Jev 選別の now に alert_condition_now / alert_condition_clear /
recovered_now を追加し、recovering は recovered_now を読む。SLO に触れない
障害(replay 停止、メモリ圧迫)で 0.98 を返していた。now の p95 と 5xx 比率に
route フィルタを付け、判定と同じ式にそろえる。
144 lines
4.6 KiB
Go
144 lines
4.6 KiB
Go
// Package harness talks to the harness runner (design 5.2): proposals, approval, execution, catalogue, scenarios.
|
|
package harness
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type Client struct {
|
|
BaseURL string // http://10.20.0.11:8080
|
|
HTTP *http.Client
|
|
}
|
|
|
|
func New(base string) *Client {
|
|
return &Client{BaseURL: strings.TrimRight(base, "/"), HTTP: &http.Client{Timeout: 11 * time.Minute}}
|
|
}
|
|
|
|
func (c *Client) call(ctx context.Context, method, path string, body any, out any) (int, error) {
|
|
var rd io.Reader
|
|
if body != nil {
|
|
b, _ := json.Marshal(body)
|
|
rd = bytes.NewReader(b)
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, rd)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := c.HTTP.Do(req)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer resp.Body.Close()
|
|
rb, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if out != nil && len(rb) > 0 {
|
|
if err := json.Unmarshal(rb, out); err != nil {
|
|
return resp.StatusCode, fmt.Errorf("harness %s %s: status %d, body not JSON: %s", method, path, resp.StatusCode, truncate(string(rb), 300))
|
|
}
|
|
}
|
|
if resp.StatusCode/100 != 2 {
|
|
return resp.StatusCode, fmt.Errorf("harness %s %s: status %d: %s", method, path, resp.StatusCode, truncate(string(rb), 300))
|
|
}
|
|
return resp.StatusCode, nil
|
|
}
|
|
|
|
type Proposal struct {
|
|
Token string `json:"token"`
|
|
ExpiresIn float64 `json:"expires_in_s"`
|
|
Approved bool `json:"approved"`
|
|
}
|
|
|
|
// ProposalState is GET /proposals/{token}: the decision as the harness knows it (approval may come through the API).
|
|
type ProposalState struct {
|
|
Token string `json:"token"`
|
|
ActionID string `json:"action_id"`
|
|
Approved bool `json:"approved"`
|
|
ApprovedAt *string `json:"approved_at"`
|
|
ApprovedBy *string `json:"approved_by"`
|
|
Executed bool `json:"executed"`
|
|
ExecutedAt *string `json:"executed_at"`
|
|
ExpiresAt string `json:"expires_at"`
|
|
}
|
|
|
|
// GetProposal reads the harness-side state of a proposal.
|
|
func (c *Client) GetProposal(ctx context.Context, token string) (*ProposalState, error) {
|
|
var out ProposalState
|
|
if _, err := c.call(ctx, "GET", "/proposals/"+token, nil, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
return &out, nil
|
|
}
|
|
|
|
// Propose registers a layer-2 proposal (POST /proposals); the harness records the proposal event and returns the token.
|
|
func (c *Client) Propose(ctx context.Context, runID, scenarioID *string, actionID string, target, diagnosis map[string]any, ttl time.Duration) (*Proposal, error) {
|
|
body := map[string]any{"run_id": runID, "scenario_id": scenarioID, "action_id": actionID, "target": target, "diagnosis": diagnosis}
|
|
if ttl > 0 {
|
|
body["ttl_seconds"] = int(ttl.Seconds())
|
|
}
|
|
var p Proposal
|
|
if _, err := c.call(ctx, "POST", "/proposals", body, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
return &p, nil
|
|
}
|
|
|
|
// Approve marks the proposal approved (POST /proposals/{token}/approve); the harness records the approval event.
|
|
func (c *Client) Approve(ctx context.Context, token, by string) error {
|
|
_, err := c.call(ctx, "POST", "/proposals/"+token+"/approve", map[string]any{"approved_by": by}, nil)
|
|
return err
|
|
}
|
|
|
|
// Reject records the rejection (POST /proposals/{token}/reject).
|
|
func (c *Client) Reject(ctx context.Context, token, by, reason string) error {
|
|
_, err := c.call(ctx, "POST", "/proposals/"+token+"/reject", map[string]any{"rejected_by": by, "reason": reason}, nil)
|
|
return err
|
|
}
|
|
|
|
// Execute runs the catalogue action with the approved token (POST /actions/execute). The body is returned as-is for the
|
|
// Discord message; a non-2xx status is an error with the body text.
|
|
func (c *Client) Execute(ctx context.Context, token, actionID string, target map[string]any, runID, scenarioID *string, note string) (map[string]any, error) {
|
|
body := map[string]any{"action_id": actionID, "target": target, "token": token, "run_id": runID, "scenario_id": scenarioID, "note": note}
|
|
var out map[string]any
|
|
_, err := c.call(ctx, "POST", "/actions/execute", body, &out)
|
|
return out, err
|
|
}
|
|
|
|
type CatalogEntry struct {
|
|
ActionID string `json:"action_id"`
|
|
Layer string `json:"layer"`
|
|
Exec bool `json:"harness_executes"`
|
|
}
|
|
|
|
func (c *Client) Catalog(ctx context.Context) ([]CatalogEntry, error) {
|
|
var out []CatalogEntry
|
|
_, err := c.call(ctx, "GET", "/actions/catalog", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
type Scenario struct {
|
|
ID string `json:"scenario_id"`
|
|
Label string `json:"ground_truth_label"`
|
|
}
|
|
|
|
func (c *Client) Scenarios(ctx context.Context) ([]Scenario, error) {
|
|
var out []Scenario
|
|
_, err := c.call(ctx, "GET", "/scenarios", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n] + "..."
|
|
}
|