using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime; /// /// Tests for 's state machine + backoff. Each /// scenario drives the supervisor with controllable reopen/replay callbacks and /// observes the resulting state transitions. /// public sealed class ReconnectSupervisorTests { private static ReconnectOptions FastBackoff() => new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(20)); /// /// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff /// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is /// actually running, not for a whole backoff. /// [Fact] public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff() { var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var reopens = 0; using var sup = new ReconnectSupervisor( reopen: async _ => { if (Interlocked.Increment(ref reopens) == 1) { reopenEntered.TrySetResult(); await releaseReopen.Task; throw new IOException("gateway still down"); } }, replay: _ => Task.CompletedTask, backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt"); sup.ReportTransportFailure(new IOException("drop")); await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start"); sup.AttemptInProgress.ShouldBeTrue("a reopen is running"); releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag"); sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists"); } /// /// #865 review: a transport failure reported WHILE the replay runs (e.g. the restarted event stream is /// refused a round trip later) must fail that attempt — not go Healthy over a dead transport. The /// supervisor retries (with backoff) and goes Healthy only after a clean replay. Without the fix the /// first replay's success went Healthy and replay ran once. /// [Fact] public async Task A_failure_reported_during_the_replay_fails_that_attempt() { ReconnectSupervisor? self = null; var replays = 0; using var sup = new ReconnectSupervisor( reopen: _ => Task.CompletedTask, replay: _ => { if (Interlocked.Increment(ref replays) == 1) { self!.ReportTransportFailure(new IOException("stream refused during replay")); } return Task.CompletedTask; }, options: FastBackoff()); self = sup; sup.ReportTransportFailure(new IOException("drop")); await TestWait.HealthyAsync(sup, "the second, clean replay must recover"); Volatile.Read(ref replays).ShouldBe(2, "the faulted replay must not count as recovered"); } /// #865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE). [Fact] public async Task ReportTransportFailure_after_stop_never_starts_a_loop() { var reopens = 0; var sup = new ReconnectSupervisor( reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; }, replay: _ => Task.CompletedTask, options: FastBackoff()); await sup.StopAsync(); Should.Throw(() => sup.ReportTransportFailure(new IOException("late"))); sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy); Volatile.Read(ref reopens).ShouldBe(0); } /// Verifies that the supervisor starts in a healthy state with no errors. [Fact] public void InitialState_IsHealthy() { using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff()); sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy); sup.IsDegraded.ShouldBeFalse(); sup.LastError.ShouldBeNull(); } /// Verifies that reporting a transport failure drives the supervisor through reopen and replay cycles back to healthy. [Fact] public async Task ReportTransportFailure_DrivesThroughReopenReplay_BackToHealthy() { var transitions = new List(); var lockObj = new object(); using var sup = new ReconnectSupervisor( reopen: _ => Task.CompletedTask, replay: _ => Task.CompletedTask, options: FastBackoff()); sup.StateChanged += (_, t) => { lock (lockObj) transitions.Add(t); }; sup.ReportTransportFailure(new IOException("transport drop")); await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy"); // Expected sequence: Healthy → TransportLost → Reopening → Replaying → Healthy. IReadOnlyList snapshot; lock (lockObj) snapshot = [.. transitions]; var states = snapshot.Select(t => t.Next).ToArray(); states.ShouldContain(ReconnectSupervisor.State.TransportLost); states.ShouldContain(ReconnectSupervisor.State.Reopening); states.ShouldContain(ReconnectSupervisor.State.Replaying); states[^1].ShouldBe(ReconnectSupervisor.State.Healthy); sup.IsDegraded.ShouldBeFalse(); } /// Verifies that reopen failures trigger retries and the supervisor stays in reopening state between attempts. [Fact] public async Task ReopenFailure_RetriesUntilSuccess_StaysInReopeningBetweenAttempts() { var attempts = 0; using var sup = new ReconnectSupervisor( reopen: _ => { attempts++; return attempts < 3 ? Task.FromException(new IOException("nope")) : Task.CompletedTask; }, replay: _ => Task.CompletedTask, options: FastBackoff()); sup.ReportTransportFailure(new IOException("kick off")); await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy"); attempts.ShouldBe(3); sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy); sup.LastError.ShouldBeNull(); // cleared on Healthy transition } /// Verifies that replay failures trigger a retry of the entire reopen-replay cycle. [Fact] public async Task ReplayFailure_RetriesEntireCycle() { var reopens = 0; var replays = 0; using var sup = new ReconnectSupervisor( reopen: _ => { reopens++; return Task.CompletedTask; }, replay: _ => { replays++; return replays < 2 ? Task.FromException(new IOException("replay nope")) : Task.CompletedTask; }, options: FastBackoff()); sup.ReportTransportFailure(new IOException("kick off")); await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy"); // First cycle: reopen succeeds, replay fails. Second cycle: both succeed. reopens.ShouldBe(2); replays.ShouldBe(2); sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy); } /// Verifies that repeated failure reports during recovery do not spawn parallel recovery loops. [Fact] public async Task RepeatedFailureReports_DuringRecovery_DoNotSpawnParallelLoops() { var attempts = 0; using var sup = new ReconnectSupervisor( reopen: async ct => { attempts++; await Task.Delay(50, ct).ConfigureAwait(false); }, replay: _ => Task.CompletedTask, options: FastBackoff()); sup.ReportTransportFailure(new IOException("first")); // Fire several more reports while reopen is in flight. for (var i = 0; i < 5; i++) { sup.ReportTransportFailure(new IOException($"rapid-{i}")); } await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy"); // One Reopen call regardless of how many failures arrived during recovery. attempts.ShouldBe(1); } /// Verifies that the last error reflects the most recent failure cause from recovery attempts. [Fact] public async Task LastError_ReflectsMostRecentFailureCause() { using var sup = new ReconnectSupervisor( reopen: _ => Task.FromException(new IOException("reopen broke")), replay: _ => Task.CompletedTask, options: new ReconnectOptions( InitialBackoffOverride: TimeSpan.FromMilliseconds(5), MaxBackoffOverride: TimeSpan.FromMilliseconds(10))); sup.ReportTransportFailure(new IOException("initial")); // Wait for the loop's first failed reopen to land rather than sleeping a fixed 100 ms: on a // loaded CI runner the recovery loop had not been scheduled yet, so LastError still held the // reported cause ("initial") — #606. A presence wait returns the instant the condition holds. await TestWait.UntilAsync( () => sup.LastError?.Contains("reopen broke", StringComparison.OrdinalIgnoreCase) == true, "the loop's failed reopen must replace the reported cause", () => $"state={sup.CurrentState} lastError={sup.LastError ?? ""}"); sup.LastError.ShouldNotBeNull(); sup.LastError.ShouldContain("reopen broke"); // updates from the loop's failed reopen attempts } /// Verifies that disposing the supervisor cancels a running recovery loop cleanly. [Fact] public async Task Dispose_CancelsRunningRecoveryLoop_Cleanly() { var cancelled = false; var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var sup = new ReconnectSupervisor( reopen: async ct => { reopenEntered.TrySetResult(); try { await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); } catch (OperationCanceledException) { cancelled = true; throw; } }, replay: _ => Task.CompletedTask, options: FastBackoff()); sup.ReportTransportFailure(new IOException("kick off")); // Wait for the loop to be INSIDE the long reopen (#858: a fixed 50 ms sleep raced the loop's // scheduling under load, and a Dispose before reopen starts never reaches the catch). await TestWait.ForAsync(reopenEntered.Task, "the recovery loop must enter reopen"); Should.NotThrow(() => sup.Dispose()); cancelled.ShouldBeTrue(); } /// Verifies that reporting a transport failure after dispose throws ObjectDisposedException. [Fact] public void ReportTransportFailure_AfterDispose_Throws() { var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff()); sup.Dispose(); Should.Throw(() => sup.ReportTransportFailure(new IOException("x"))); } /// Verifies that waiting for healthy state returns immediately when already healthy. [Fact] public async Task WaitForHealthy_ReturnsImmediately_WhenAlreadyHealthy() { using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff()); // No failure reported — should be Healthy from the start, so the wait completes synchronously // (deterministic, unlike a wall-clock bound that a descheduled thread can overrun — #858). var wait = sup.WaitForHealthyAsync(CancellationToken.None); wait.IsCompletedSuccessfully.ShouldBeTrue("an already-Healthy supervisor must not wait at all"); await wait; } }