diff --git a/harness/run.go b/harness/run.go index fee15f9..2d07c24 100644 --- a/harness/run.go +++ b/harness/run.go @@ -279,11 +279,22 @@ func (m *RunManager) runScenario(ctx context.Context, runID string, sc *Scenario aborted := ctx.Err() != nil // revert (always, even on abort, unless the scenario is irreversible by design or skip_revert) if !skipRevert && sc.Revert.Method != "none" { - rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) - rres, rerr := a.runStep(rvCtx, sc, sc.Revert, runID, params) - cancelRv() + // a failed revert leaves the fault in place (2026-10-18: one SSH connection reset at the X-M2 revert left a + // netem qdisc on a legacy instance and broke the next 14 runs of two scenarios), so retry a few times + var rres map[string]any + var rerr error + for attempt := 1; attempt <= 3; attempt++ { + rvCtx, cancelRv := context.WithTimeout(bg, 10*time.Minute) + rres, rerr = a.runStep(rvCtx, sc, sc.Revert, runID, params) + cancelRv() + if rerr == nil { + break + } + a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "attempt": attempt, "error": rerr.Error(), "result": rres}) + time.Sleep(15 * time.Second) + } if rerr != nil { - a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres}) + a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "error": rerr.Error(), "result": rres, "note": "revert failed after 3 attempts; the fault may still be in place"}) } else { a.store.Event(bg, &runID, &sid, "note", "runner", map[string]any{"phase": "revert", "result": rres}) }