lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/ReconnectSupervisorTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;

namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;

/// <summary>
///     Tests for <see cref="ReconnectSupervisor"/>'s state machine + backoff. Each
///     scenario drives the supervisor with controllable reopen/replay callbacks and
///     observes the resulting state transitions.
/// </summary>
public sealed class ReconnectSupervisorTests
{
    private static ReconnectOptions FastBackoff() =>
        new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
            MaxBackoffOverride: TimeSpan.FromMilliseconds(20));

    /// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
    [Fact]
    public void InitialState_IsHealthy()
    {
        using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.IsDegraded.ShouldBeFalse();
        sup.LastError.ShouldBeNull();
    }

    /// <summary>Verifies that reporting a transport failure drives the supervisor through reopen and replay cycles back to healthy.</summary>
    [Fact]
    public async Task ReportTransportFailure_DrivesThroughReopenReplay_BackToHealthy()
    {
        var transitions = new List<StateTransition>();
        var lockObj = new object();

        using var sup = new ReconnectSupervisor(
            reopen: _ => Task.CompletedTask,
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.StateChanged += (_, t) => { lock (lockObj) transitions.Add(t); };

        sup.ReportTransportFailure(new IOException("transport drop"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // Expected sequence: Healthy → TransportLost → Reopening → Replaying → Healthy.
        IReadOnlyList<StateTransition> snapshot;
        lock (lockObj) snapshot = [.. transitions];
        var states = snapshot.Select(t => t.Next).ToArray();

        states.ShouldContain(ReconnectSupervisor.State.TransportLost);
        states.ShouldContain(ReconnectSupervisor.State.Reopening);
        states.ShouldContain(ReconnectSupervisor.State.Replaying);
        states[^1].ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.IsDegraded.ShouldBeFalse();
    }

    /// <summary>Verifies that reopen failures trigger retries and the supervisor stays in reopening state between attempts.</summary>
    [Fact]
    public async Task ReopenFailure_RetriesUntilSuccess_StaysInReopeningBetweenAttempts()
    {
        var attempts = 0;
        using var sup = new ReconnectSupervisor(
            reopen: _ => { attempts++; return attempts < 3 ? Task.FromException(new IOException("nope")) : Task.CompletedTask; },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        attempts.ShouldBe(3);
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
        sup.LastError.ShouldBeNull(); // cleared on Healthy transition
    }

    /// <summary>Verifies that replay failures trigger a retry of the entire reopen-replay cycle.</summary>
    [Fact]
    public async Task ReplayFailure_RetriesEntireCycle()
    {
        var reopens = 0;
        var replays = 0;
        using var sup = new ReconnectSupervisor(
            reopen: _ => { reopens++; return Task.CompletedTask; },
            replay: _ => { replays++; return replays < 2 ? Task.FromException(new IOException("replay nope")) : Task.CompletedTask; },
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // First cycle: reopen succeeds, replay fails. Second cycle: both succeed.
        reopens.ShouldBe(2);
        replays.ShouldBe(2);
        sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
    }

    /// <summary>Verifies that repeated failure reports during recovery do not spawn parallel recovery loops.</summary>
    [Fact]
    public async Task RepeatedFailureReports_DuringRecovery_DoNotSpawnParallelLoops()
    {
        var attempts = 0;
        using var sup = new ReconnectSupervisor(
            reopen: async ct =>
            {
                attempts++;
                await Task.Delay(50, ct).ConfigureAwait(false);
            },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("first"));
        // Fire several more reports while reopen is in flight.
        for (var i = 0; i < 5; i++)
        {
            sup.ReportTransportFailure(new IOException($"rapid-{i}"));
        }

        await TestWait.HealthyAsync(sup, "the supervisor must recover to Healthy");

        // One Reopen call regardless of how many failures arrived during recovery.
        attempts.ShouldBe(1);
    }

    /// <summary>Verifies that the last error reflects the most recent failure cause from recovery attempts.</summary>
    [Fact]
    public async Task LastError_ReflectsMostRecentFailureCause()
    {
        using var sup = new ReconnectSupervisor(
            reopen: _ => Task.FromException(new IOException("reopen broke")),
            replay: _ => Task.CompletedTask,
            options: new ReconnectOptions(
                InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
                MaxBackoffOverride: TimeSpan.FromMilliseconds(10)));

        sup.ReportTransportFailure(new IOException("initial"));

        // Wait for the loop's first failed reopen to land rather than sleeping a fixed 100 ms: on a
        // loaded CI runner the recovery loop had not been scheduled yet, so LastError still held the
        // reported cause ("initial") — #606. A presence wait returns the instant the condition holds.
        await TestWait.UntilAsync(
            () => sup.LastError?.Contains("reopen broke", StringComparison.OrdinalIgnoreCase) == true,
            "the loop's failed reopen must replace the reported cause",
            () => $"state={sup.CurrentState} lastError={sup.LastError ?? "<none>"}");

        sup.LastError.ShouldNotBeNull();
        sup.LastError.ShouldContain("reopen broke"); // updates from the loop's failed reopen attempts
    }

    /// <summary>Verifies that disposing the supervisor cancels a running recovery loop cleanly.</summary>
    [Fact]
    public async Task Dispose_CancelsRunningRecoveryLoop_Cleanly()
    {
        var cancelled = false;
        var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var sup = new ReconnectSupervisor(
            reopen: async ct =>
            {
                reopenEntered.TrySetResult();
                try { await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); }
                catch (OperationCanceledException) { cancelled = true; throw; }
            },
            replay: _ => Task.CompletedTask,
            options: FastBackoff());

        sup.ReportTransportFailure(new IOException("kick off"));
        // Wait for the loop to be INSIDE the long reopen (#858: a fixed 50 ms sleep raced the loop's
        // scheduling under load, and a Dispose before reopen starts never reaches the catch).
        await TestWait.ForAsync(reopenEntered.Task, "the recovery loop must enter reopen");
        Should.NotThrow(() => sup.Dispose());
        cancelled.ShouldBeTrue();
    }

    /// <summary>Verifies that reporting a transport failure after dispose throws ObjectDisposedException.</summary>
    [Fact]
    public void ReportTransportFailure_AfterDispose_Throws()
    {
        var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        sup.Dispose();
        Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("x")));
    }

    /// <summary>Verifies that waiting for healthy state returns immediately when already healthy.</summary>
    [Fact]
    public async Task WaitForHealthy_ReturnsImmediately_WhenAlreadyHealthy()
    {
        using var sup = new ReconnectSupervisor(_ => Task.CompletedTask, _ => Task.CompletedTask, FastBackoff());
        // No failure reported — should be Healthy from the start, so the wait completes synchronously
        // (deterministic, unlike a wall-clock bound that a descheduled thread can overrun — #858).
        var wait = sup.WaitForHealthyAsync(CancellationToken.None);
        wait.IsCompletedSuccessfully.ShouldBeTrue("an already-Healthy supervisor must not wait at all");
        await wait;
    }
}