lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorReconnectSingleFlightTests.cs

using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     Review 02 H2: the <c>Reconnecting</c> retry timer fired every <c>reconnectInterval</c> whether or not the
///     previous attempt was still running — <c>_initGeneration</c> only discarded the older attempt's RESULT. With an
///     Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
///     invoker) attempt N+1's <c>ShutdownAsync</c> ran while attempt N's <c>InitializeAsync</c> was still inside the
///     driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
///     the device stayed down. The retry is now single-flight.
/// </summary>
public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
{
    [Fact]
    public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
    {
        // A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
        // Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
        var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
        var parent = CreateTestProbe();
        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));

        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        // Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
        // actor connects.
        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);

        driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
        driver.Overlaps.ShouldBe(0,
            $"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
    }

    [Fact]
    public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
    {
        var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
        {
            // Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
            ShutdownBehavior = _ => never.Task,
        };
        var parent = CreateTestProbe();
        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
            shutdownTimeout: TimeSpan.FromMilliseconds(200)));

        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
        never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
        driver.Attempts.ShouldBe(2);
    }

    /// <summary>
    ///     Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
    ///     token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
    ///     attempt is now bounded pessimistically (<c>initializeTimeout</c>); an overrun is abandoned, reported
    ///     as a failure, and the next tick retries.
    /// </summary>
    [Fact]
    public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
    {
        var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
        {
            // Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
            HangOnAttempt = 1,
        };
        var parent = CreateTestProbe();
        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
            initializeTimeout: TimeSpan.FromMilliseconds(300)));

        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
        driver.Attempts.ShouldBe(2);
        driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
    }

    /// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
    private sealed class InterleavingDriver : IDriver
    {
        private readonly TimeSpan _initDuration;
        private readonly int _failUntilAttempt;
        private readonly Lock _gate = new();
        private readonly List<string> _events = new();
        private int _attempts;
        private int _initsInProgress;
        private int _overlaps;

        public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
        {
            _initDuration = initDuration;
            _failUntilAttempt = failUntilAttempt;
        }

        public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }

        /// <summary>The 1-based attempt that never completes and ignores its token (0 = none).</summary>
        public int HangOnAttempt { get; init; }

        private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);

        public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;

        public int Attempts => Volatile.Read(ref _attempts);
        public int Overlaps => Volatile.Read(ref _overlaps);
        public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }

        public string DriverInstanceId => "interleaving-1";
        public string DriverType => "Interleaving";

        public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
        {
            var attempt = Interlocked.Increment(ref _attempts);
            Interlocked.Increment(ref _initsInProgress);
            Record($"init{attempt}>");
            try
            {
                if (attempt == HangOnAttempt) await _never.Task;
                if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
                if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
            }
            finally
            {
                Record($"<init{attempt}");
                Interlocked.Decrement(ref _initsInProgress);
            }
        }

        public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;

        public async Task ShutdownAsync(CancellationToken cancellationToken)
        {
            if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
            Record("shutdown>");
            try
            {
                if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
            }
            finally
            {
                Record("<shutdown");
            }
        }

        public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);

        private void Record(string e) { lock (_gate) _events.Add(e); }
    }
}