feat(ai): add AI VM services (diagnosis agent, pgvector RAG, Discord HITL bot) and harness approval endpoints

실험 설계서 6장(WP-12)의 AI VM 2층 구현. - cmd/agent: 하네스 /diagnose 수신 → OpenAI Responses API 함수 호출 루프(읽기 전용 도구 7종: query_metrics, query_logs, get_probe_status, get_k8s, get_gcp_instances, get_aws_state, search_runbooks) → strict 스키마 submit_diagnosis(6.3) → 하네스 /proposals 등록 + HITL 게시, 또는 신뢰도 0.5 미만·escalate 시 3층 큐 에스컬레이션. 세션·프롬프트 버전·러너북 벡터는 pgvector(ai 스키마)에 저장. 라벨·조치 enum은 하네스 /scenarios·/actions/catalog에서 갱신. - cmd/hitl: Discord Gateway 봇(intents GUILDS만). 승인/거부 버튼(custom_id approve:/reject:<token>), 3초 내 DEFERRED_UPDATE_MESSAGE, 승인 시 하네스 승인 → /actions/execute → 원 메시지 편집, 15분 무응답 시 에스컬레이션 채널로 이동. 재기동 시 미결 제안 타이머 복원. - harness: POST /proposals/{token}/approve, /reject 추가. 제안은 진단 시점에 등록하고 승인은 버튼 시점에 기록해 MTTA(승인-제안)가 실제 대기 시간이 되도록 함. OpenAI·Discord 토큰은 Secret Manager 파일로만 읽고 코드·저장소에 두지 않는다.
2026-10-08 01:36:10 +09:00
1 parent de076a79e4
commit f30af2a4c3
@@ -0,0 +1,193 @@
+// agent: AI VM diagnosis service (experiment design 6). HTTP on :8080. /diagnose is called by the harness.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/agent"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/diag"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/harness"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/openai"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/rag"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/tools"
+)
+
+func env(k, d string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return d
+}
+
+func readFile(p string) string {
+	b, err := os.ReadFile(p)
+	if err != nil {
+		log.Fatalf("read %s: %v", p, err)
+	}
+	return string(b)
+}
+
+func keyReader(path string) func() (string, error) {
+	return func() (string, error) {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+}
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+	ctx := context.Background()
+	promptDir := env("PROMPT_DIR", "/etc/ai/prompts")
+	oai := openai.New(env("OPENAI_BASE_URL", ""), keyReader(env("OPENAI_KEY_FILE", "/run/secrets/openai-api-key")))
+	store, err := rag.New(ctx, env("RAG_DSN", "postgres://ai:ai@rag-db:5432/ai?sslmode=disable"))
+	if err != nil {
+		log.Fatalf("rag: %v", err)
+	}
+	system := readFile(promptDir + "/system.md")
+	pv, err := store.UpsertPrompt(ctx, system)
+	if err != nil {
+		log.Fatalf("prompt version: %v", err)
+	}
+	hc := harness.New(env("HARNESS_URL", "http://10.20.0.11:8080"))
+	enums := diag.Enums{Labels: diag.DefaultLabels, Actions: diag.DefaultActions}
+	if sc, err := hc.Scenarios(ctx); err == nil && len(sc) > 0 {
+		seen := map[string]bool{}
+		enums.Labels = nil
+		for _, s := range sc {
+			if s.Label != "" && !seen[s.Label] {
+				seen[s.Label] = true
+				enums.Labels = append(enums.Labels, s.Label)
+			}
+		}
+	} else if err != nil {
+		log.Printf("harness scenarios unavailable, using built-in labels: %v", err)
+	}
+	if cat, err := hc.Catalog(ctx); err == nil && len(cat) > 0 {
+		enums.Actions = nil
+		for _, c := range cat {
+			if strings.Contains(c.Layer, "2") && c.ActionID != "escalate" {
+				enums.Actions = append(enums.Actions, c.ActionID)
+			}
+		}
+	} else if err != nil {
+		log.Printf("harness catalog unavailable, using built-in actions: %v", err)
+	}
+	tcfg := tools.Config{
+		MimirURL: env("MIMIR_URL", "http://10.20.0.10:9009/prometheus"), LokiURL: env("LOKI_URL", "http://10.20.0.10:3100"),
+		K8sAPI: env("K8S_API", "https://10.10.0.10:6443"), K8sTokenFile: env("K8S_TOKEN_FILE", "/run/secrets/k8s-token"), K8sCAFile: env("K8S_CA_FILE", "/run/secrets/k8s-ca.crt"),
+		AWSRoleARN: env("AWS_ROLE_ARN", ""), AWSRegion: env("AWS_REGION", "ap-northeast-1"), GCPProject: env("GCP_PROJECT", ""),
+		EmbedModel: env("EMBED_MODEL", "text-embedding-3-small"),
+	}
+	tr := tools.New(tcfg, store, oai)
+	acfg := agent.Config{
+		Model: env("OPENAI_MODEL", "gpt-6.1-sol"), SystemPrompt: system,
+		Topology: readFile(promptDir + "/topology.md"), SLO: readFile(promptDir + "/slo.md"),
+		HitlURL: env("HITL_URL", "http://hitl:8081"),
+	}
+	ag := agent.New(acfg, oai, tr, hc, store, enums, pv)
+	log.Printf("agent up: model=%s prompt=%s labels=%d actions=%d", acfg.Model, pv, len(enums.Labels), len(enums.Actions))
+
+	// runbook ingest (needs the OpenAI key; retried in the background until the key exists)
+	go func() {
+		dir := env("RUNBOOK_DIR", "/etc/ai/runbooks")
+		for {
+			added, removed, err := store.IngestDir(ctx, dir, oai, tcfg.EmbedModel)
+			if err == nil {
+				n, _ := store.DocCount(ctx)
+				log.Printf("runbooks: +%d -%d, %d chunks stored", added, removed, n)
+				return
+			}
+			log.Printf("runbook ingest deferred: %v", err)
+			time.Sleep(5 * time.Minute)
+		}
+	}()
+
+	var inflight int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
+	mux.HandleFunc("GET /selfcheck", func(w http.ResponseWriter, r *http.Request) {
+		c, cancel := context.WithTimeout(r.Context(), 40*time.Second)
+		defer cancel()
+		out := map[string]any{"model": acfg.Model, "prompt_version": pv, "labels": enums.Labels, "actions": enums.Actions}
+		key, kerr := keyReader(env("OPENAI_KEY_FILE", "/run/secrets/openai-api-key"))()
+		out["openai_key_present"] = kerr == nil && key != ""
+		out["rag_db"] = errText(store.Ping(c))
+		n, _ := store.DocCount(c)
+		out["rag_chunks"] = n
+		out["mimir"] = tr.Call(c, "get_probe_status", "{}")
+		out["k8s"] = short(tr.Call(c, "get_k8s", `{"kind":"nodes","namespace":""}`))
+		out["gcp"] = short(tr.Call(c, "get_gcp_instances", "{}"))
+		arn, aerr := tr.AWSCallerIdentity(c)
+		out["aws"] = map[string]any{"arn": arn, "error": errText(aerr)}
+		if _, err := hc.Catalog(c); err != nil {
+			out["harness"] = err.Error()
+		} else {
+			out["harness"] = "ok"
+		}
+		writeJSON(w, 200, out)
+	})
+	mux.HandleFunc("POST /diagnose", func(w http.ResponseWriter, r *http.Request) {
+		var req agent.Request
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			writeJSON(w, 400, map[string]string{"error": err.Error()})
+			return
+		}
+		if req.Alert == nil {
+			writeJSON(w, 400, map[string]string{"error": "alert required"})
+			return
+		}
+		inflight++
+		defer func() { inflight-- }()
+		c, cancel := context.WithTimeout(context.Background(), 170*time.Second) // harness waits 3 minutes
+		defer cancel()
+		res, err := ag.Diagnose(c, req)
+		if err != nil {
+			writeJSON(w, 503, map[string]string{"error": err.Error()})
+			return
+		}
+		ag.Dispose(c, req, res)
+		writeJSON(w, 200, res)
+	})
+	srv := &http.Server{Addr: env("AGENT_ADDR", ":8080"), Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second}
+	log.Fatal(srv.ListenAndServe())
+}
+
+func errText(err error) string {
+	if err == nil {
+		return "ok"
+	}
+	return err.Error()
+}
+
+func short(s string) string {
+	if len(s) > 400 {
+		return s[:400] + "..."
+	}
+	return s
+}
+
+func writeJSON(w http.ResponseWriter, code int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	_ = enc.Encode(v)
+}
+
+func logging(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/healthz" {
+			log.Printf("%s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
+		}
+		h.ServeHTTP(w, r)
+	})
+}
@@ -0,0 +1,126 @@
+// hitl: Discord approval bot of the AI VM (experiment design 6.5). HTTP on :8081 for the agent; Gateway for buttons.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/bwmarrin/discordgo"
+
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/harness"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/hitl"
+	"github.com/selfheal-lab/selfheal-lab-apps/ai/internal/rag"
+)
+
+func env(k, d string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return d
+}
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+	ctx := context.Background()
+	store, err := rag.New(ctx, env("RAG_DSN", "postgres://ai:ai@rag-db:5432/ai?sslmode=disable"))
+	if err != nil {
+		log.Fatalf("rag: %v", err)
+	}
+	cfg := hitl.Config{GuildID: env("DISCORD_GUILD_ID", ""), ApprovalsChannel: env("DISCORD_APPROVALS_CHANNEL_ID", ""), EscalationChannel: env("DISCORD_ESCALATION_CHANNEL_ID", ""), ApproverRoleID: env("DISCORD_APPROVER_ROLE_ID", "")}
+	tokB, err := os.ReadFile(env("DISCORD_TOKEN_FILE", "/run/secrets/discord-bot-token"))
+	token := strings.TrimSpace(string(tokB))
+	connected := false
+	var bot *hitl.Bot
+	hc := harness.New(env("HARNESS_URL", "http://10.20.0.11:8080"))
+	var dg *discordgo.Session
+	if err != nil || token == "" {
+		log.Printf("discord token missing (%v); running degraded: /post and /escalate return 503 until the secret exists", err)
+	} else {
+		dg, err = discordgo.New("Bot " + token)
+		if err != nil {
+			log.Fatalf("discord session: %v", err)
+		}
+		// Minimum intent set (design 11.7): GUILDS only. INTERACTION_CREATE carries no intent requirement; no privileged intents.
+		dg.Identify.Intents = discordgo.IntentsGuilds
+		bot = hitl.New(cfg, hitl.Session{S: dg}, hc, store)
+		dg.AddHandler(bot.HandleInteraction)
+		dg.AddHandler(func(s *discordgo.Session, r *discordgo.Ready) {
+			log.Printf("discord ready: user=%s guilds=%d", r.User.Username, len(r.Guilds))
+		})
+		if err := dg.Open(); err != nil {
+			log.Fatalf("discord open: %v", err)
+		}
+		connected = true
+		bot.RearmPending(ctx)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
+	mux.HandleFunc("GET /selfcheck", func(w http.ResponseWriter, r *http.Request) {
+		out := map[string]any{"discord_connected": connected, "guild": cfg.GuildID, "approvals_channel": cfg.ApprovalsChannel, "escalation_channel": cfg.EscalationChannel, "approver_role": cfg.ApproverRoleID}
+		if dg != nil {
+			if ch, err := dg.Channel(cfg.ApprovalsChannel); err == nil {
+				out["approvals_channel_name"] = ch.Name
+			} else {
+				out["approvals_channel_error"] = err.Error()
+			}
+		}
+		pend, _ := store.Pending(r.Context())
+		out["pending"] = len(pend)
+		writeJSON(w, 200, out)
+	})
+	mux.HandleFunc("POST /post", func(w http.ResponseWriter, r *http.Request) {
+		if !connected {
+			writeJSON(w, 503, map[string]string{"error": "discord not connected"})
+			return
+		}
+		var req hitl.PostRequest
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Token == "" {
+			writeJSON(w, 400, map[string]string{"error": "bad request"})
+			return
+		}
+		if err := bot.Post(r.Context(), req); err != nil {
+			writeJSON(w, 500, map[string]string{"error": err.Error()})
+			return
+		}
+		writeJSON(w, 201, map[string]string{"status": "posted"})
+	})
+	mux.HandleFunc("POST /escalate", func(w http.ResponseWriter, r *http.Request) {
+		if !connected {
+			writeJSON(w, 503, map[string]string{"error": "discord not connected"})
+			return
+		}
+		var req hitl.EscalateRequest
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			writeJSON(w, 400, map[string]string{"error": "bad request"})
+			return
+		}
+		if err := bot.Escalate(r.Context(), req); err != nil {
+			writeJSON(w, 500, map[string]string{"error": err.Error()})
+			return
+		}
+		writeJSON(w, 201, map[string]string{"status": "escalated"})
+	})
+	srv := &http.Server{Addr: env("HITL_ADDR", ":8081"), Handler: mux, ReadHeaderTimeout: 10 * time.Second}
+	go func() { log.Fatal(srv.ListenAndServe()) }()
+	log.Printf("hitl up: connected=%v", connected)
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+	<-sig
+	if dg != nil {
+		_ = dg.Close()
+	}
+}
+
+func writeJSON(w http.ResponseWriter, code int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_ = json.NewEncoder(w).Encode(v)
+}