fix(harness): retry a failed revert step three times

2026-10-18 m2 배치에서 X-M2 복원 SSH 가 connection reset 으로 한 번 실패해 레거시
인스턴스에 netem loss 50% qdisc 가 남았고, 이후 14런의 V-G1·X-M2 주입이 막혔다.
복원 실패는 장애를 환경에 남기므로 15초 간격으로 최대 3회 다시 시도하고, 시도마다
이벤트를 남긴다. 3회 모두 실패하면 장애가 남아 있을 수 있다는 note 를 기록한다.
This commit is contained in:
seon committed 2026-10-19 01:10:28 +09:00
1 parent 2a2f91a32c
commit fdd776c9aa
1 file changed
+15 -4
+15 -4
View File
@@ -279,11 +279,22 @@ func (m *RunManager) runScenario(ctx context.Context, runID string, sc *Scenario
aborted := ctx.Err() != nil aborted := ctx.Err() != nil
// revert (always, even on abort, unless the scenario is irreversible by design or skip_revert) // revert (always, even on abort, unless the scenario is irreversible by design or skip_revert)
if !skipRevert && sc.Revert.Method != "none" { if !skipRevert && sc.Revert.Method != "none" {
rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) // a failed revert leaves the fault in place (2026-10-18: one SSH connection reset at the X-M2 revert left a
rres, rerr := a.runStep(rvCtx, sc, sc.Revert, runID, params) // netem qdisc on a legacy instance and broke the next 14 runs of two scenarios), so retry a few times
cancelRv() var rres map[string]any
var rerr error
for attempt := 1; attempt <= 3; attempt++ {
rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute)
rres, rerr = a.runStep(rvCtx, sc, sc.Revert, runID, params)
cancelRv()
if rerr == nil {
break
}
a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "attempt": attempt, "error": rerr.Error(), "result": rres})
time.Sleep(15 * time.Second)
}
if rerr != nil { if rerr != nil {
a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres}) a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres, "note": "revert failed after 3 attempts; the fault may still be in place"})
} else { } else {
a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "result": rres}) a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "result": rres})
} }