lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;

namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;

/// <summary>
///     Tests for <see cref="ReconnectSupervisor"/>'s state machine + backoff. Each
///     scenario drives the supervisor with controllable reopen/replay callbacks and
///     observes the resulting state transitions.
/// </summary>
public sealed class ReconnectSupervisorTests
{
    private static ReconnectOptions FastBackoff() =>
        new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
            MaxBackoffOverride: TimeSpan.FromMilliseconds(20));

    /// <summary>
    ///     #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff
    ///     between attempts — when the state still reads Reopening. Advises wait for a replay only while one is
    ///     actually running, not for a whole backoff.
    /// </summary>
    [Fact]
    public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff()
    {
        var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var reopens = 0;
        using var sup = new ReconnectSupervisor(
            reopen: async _ =>
            {
                if (Interlocked.Increment(ref reopens) == 1)
                {
                    reopenEntered.TrySetResult();
                    await releaseReopen.Task;
                    throw new IOException("gateway still down");
                }
            },
            replay: _ => Task.CompletedTask,
            backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff

        sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt");
        sup.ReportTransportFailure(new IOException("drop"));
        await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start");
        sup.AttemptInProgress.ShouldBeTrue("a reopen is running");

        releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min
        await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag");
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists");
    }

    /// <summary>
    ///     #865 review: a transport failure reported WHILE the replay runs (e.g. the restarted event stream is
    ///     refused a round trip later) must fail that attempt — not go Healthy over a dead transport. The
    ///     supervisor retries (with backoff) and goes Healthy only after a clean replay. Without the fix the
    ///     first replay's success went Healthy and replay ran once.
    /// </summary>
    [Fact]
    public async Task A_failure_reported_during_the_replay_fails_that_attempt()
    {
        ReconnectSupervisor? self = null;
        var replays = 0;
        using var sup = new ReconnectSupervisor(
            reopen: _ => Task.CompletedTask,
            replay: _ =>
            {
                if (Interlocked.Increment(ref replays) == 1)
                {
                    self!.ReportTransportFailure(new IOException("stream refused during replay"));
                }
                return Task.CompletedTask;
            },
            options: FastBackoff());
        self = sup;

        sup.ReportTransportFailure(new IOException("drop"));
        await TestWait.HealthyAsync(sup, "the second, clean replay must recover");
        Volatile.Read(ref replays).ShouldBe(2, "the faulted replay must not count as recovered");
    }

    /// <summary>#865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE).</summary>
    [Fact]
    public async Task ReportTransportFailure_after_stop_never_starts_a_loop()
    {
        var reopens = 0;
        var sup = new ReconnectSupervisor(
            reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());
        await sup.StopAsync();

        Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("late")));
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
        Volatile.Read(ref reopens).ShouldBe(0);
    }

    /// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
    [Fact]
    public void InitialState_IsHealthy()
    {
        using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.IsDegraded.ShouldBeFalse();
        sup.LastError.ShouldBeNull();
    }

    /// <summary>Verifies that reporting a transport failure drives the supervisor through reopen and replay cycles back to healthy.</summary>
    [Fact]
    public async Task ReportTransportFailure_DrivesThroughReopenReplay_BackToHealthy()
    {
        var transitions = new List<StateTransition>();
        var lockObj = new object();

        using var sup = new ReconnectSupervisor(
            reopen: _ => Task.CompletedTask,
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.StateChanged += (_, t) => { lock (lockObj) transitions.Add(t); };

        sup.ReportTransportFailure(new IOException("transport drop"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // Expected sequence: Healthy → TransportLost → Reopening → Replaying → Healthy.
        IReadOnlyList<StateTransition> snapshot;
        lock (lockObj) snapshot = [.. transitions];
        var states = snapshot.Select(t => t.Next).ToArray();

        states.ShouldContain(ReconnectSupervisor.State.TransportLost);
        states.ShouldContain(ReconnectSupervisor.State.Reopening);
        states.ShouldContain(ReconnectSupervisor.State.Replaying);
        states[^1].ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.IsDegraded.ShouldBeFalse();
    }

    /// <summary>Verifies that reopen failures trigger retries and the supervisor stays in reopening state between attempts.</summary>
    [Fact]
    public async Task ReopenFailure_RetriesUntilSuccess_StaysInReopeningBetweenAttempts()
    {
        var attempts = 0;
        using var sup = new ReconnectSupervisor(
            reopen: _ => { attempts++; return attempts < 3 ? Task.FromException(new IOException("nope")) : Task.CompletedTask; },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        attempts.ShouldBe(3);
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.LastError.ShouldBeNull(); // cleared on Healthy transition
    }

    /// <summary>Verifies that replay failures trigger a retry of the entire reopen-replay cycle.</summary>
    [Fact]
    public async Task ReplayFailure_RetriesEntireCycle()
    {
        var reopens = 0;
        var replays = 0;
        using var sup = new ReconnectSupervisor(
            reopen: _ => { reopens++; return Task.CompletedTask; },
            replay: _ => { replays++; return replays < 2 ? Task.FromException(new IOException("replay nope")) : Task.CompletedTask; },
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // First cycle: reopen succeeds, replay fails. Second cycle: both succeed.
        reopens.ShouldBe(2);
        replays.ShouldBe(2);
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
    }

    /// <summary>Verifies that repeated failure reports during recovery do not spawn parallel recovery loops.</summary>
    [Fact]
    public async Task RepeatedFailureReports_DuringRecovery_DoNotSpawnParallelLoops()
    {
        var attempts = 0;
        using var sup = new ReconnectSupervisor(
            reopen: async ct =>
            {
                attempts++;
                await Task.Delay(50, ct).ConfigureAwait(false);
            },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("first"));
        // Fire several more reports while reopen is in flight.
        for (var i = 0; i < 5; i++)
        {
            sup.ReportTransportFailure(new IOException($"rapid-{i}"));
        }

        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // One Reopen call regardless of how many failures arrived during recovery.
        attempts.ShouldBe(1);
    }

    /// <summary>Verifies that the last error reflects the most recent failure cause from recovery attempts.</summary>
    [Fact]
    public async Task LastError_ReflectsMostRecentFailureCause()
    {
        using var sup = new ReconnectSupervisor(
            reopen: _ => Task.FromException(new IOException("reopen broke")),
            replay: _ => Task.CompletedTask,
            options: new ReconnectOptions(
                InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
                MaxBackoffOverride: TimeSpan.FromMilliseconds(10)));

        sup.ReportTransportFailure(new IOException("initial"));

        // Wait for the loop's first failed reopen to land rather than sleeping a fixed 100 ms: on a
        // loaded CI runner the recovery loop had not been scheduled yet, so LastError still held the
        // reported cause ("initial") — #606. A presence wait returns the instant the condition holds.
        await TestWait.UntilAsync(
            () => sup.LastError?.Contains("reopen broke", StringComparison.OrdinalIgnoreCase) == true,
            "the loop's failed reopen must replace the reported cause",
            () => $"state={sup.CurrentState} lastError={sup.LastError ?? "<none>"}");

        sup.LastError.ShouldNotBeNull();
        sup.LastError.ShouldContain("reopen broke"); // updates from the loop's failed reopen attempts
    }

    /// <summary>Verifies that disposing the supervisor cancels a running recovery loop cleanly.</summary>
    [Fact]
    public async Task Dispose_CancelsRunningRecoveryLoop_Cleanly()
    {
        var cancelled = false;
        var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var sup = new ReconnectSupervisor(
            reopen: async ct =>
            {
                reopenEntered.TrySetResult();
                try { await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); }
                catch (OperationCanceledException) { cancelled = true; throw; }
            },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        // Wait for the loop to be INSIDE the long reopen (#858: a fixed 50 ms sleep raced the loop's
        // scheduling under load, and a Dispose before reopen starts never reaches the catch).
        await TestWait.ForAsync(reopenEntered.Task, "the recovery loop must enter reopen");
        Should.NotThrow(() => sup.Dispose());
        cancelled.ShouldBeTrue();
    }

    /// <summary>Verifies that reporting a transport failure after dispose throws ObjectDisposedException.</summary>
    [Fact]
    public void ReportTransportFailure_AfterDispose_Throws()
    {
        var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        sup.Dispose();
        Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("x")));
    }

    /// <summary>Verifies that waiting for healthy state returns immediately when already healthy.</summary>
    [Fact]
    public async Task WaitForHealthy_ReturnsImmediately_WhenAlreadyHealthy()
    {
        using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        // No failure reported — should be Healthy from the start, so the wait completes synchronously
        // (deterministic, unlike a wall-clock bound that a descheduled thread can overrun — #858).
        var wait = sup.WaitForHealthyAsync(CancellationToken.None);
        wait.IsCompletedSuccessfully.ShouldBeTrue("an already-Healthy supervisor must not wait at all");
        await wait;
    }
}