From fdd776c9aa1a715eeaf4c171635513260b428144 Mon Sep 17 00:00:00 2001 From: seonNoh Date: Mon, 19 Oct 2026 01:10:28 +0900 Subject: [PATCH] fix(harness): retry a failed revert step three times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-10-18 m2 배치에서 X-M2 복원 SSH 가 connection reset 으로 한 번 실패해 레거시 인스턴스에 netem loss 50% qdisc 가 남았고, 이후 14런의 V-G1·X-M2 주입이 막혔다. 복원 실패는 장애를 환경에 남기므로 15초 간격으로 최대 3회 다시 시도하고, 시도마다 이벤트를 남긴다. 3회 모두 실패하면 장애가 남아 있을 수 있다는 note 를 기록한다. --- harness/run.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/harness/run.go b/harness/run.go index fee15f9..2d07c24 100644 --- a/harness/run.go +++ b/harness/run.go @@ -279,11 +279,22 @@ func (m *RunManager) runScenario(ctx context.Context, runID string, sc *Scenario aborted := ctx.Err() != nil // revert (always, even on abort, unless the scenario is irreversible by design or skip_revert) if !skipRevert && sc.Revert.Method != "none" { - rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) - rres, rerr := a.runStep(rvCtx, sc, sc.Revert, runID, params) - cancelRv() + // a failed revert leaves the fault in place (2026-10-18: one SSH connection reset at the X-M2 revert left a + // netem qdisc on a legacy instance and broke the next 14 runs of two scenarios), so retry a few times + var rres map[string]any + var rerr error + for attempt := 1; attempt <= 3; attempt++ { + rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) + rres, rerr = a.runStep(rvCtx, sc, sc.Revert, runID, params) + cancelRv() + if rerr == nil { + break + } + a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "attempt": attempt, "error": rerr.Error(), "result": rres}) + time.Sleep(15 * time.Second) + } if rerr != nil { - a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres}) + a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres, "note": "revert failed after 3 attempts; the fault may still be in place"}) } else { a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "result": rres}) }