lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipParallelReadTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.AbCip;

namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests;

/// <summary>
///     Task 14 — <c>ReadAsync</c> runs its planned groups + per-tag fallbacks inside a bounded
///     concurrency window (<see cref="AbCipDriverOptions.MaxConcurrentReads"/>) instead of strictly
///     one device round-trip at a time, so libplctag can overlap requests. The window is ACROSS
///     tags only: the per-tag runtime lock (05/STAB-4) still serialises Read → GetStatus → Decode
///     against any one cached handle. Health is aggregated once after the batch (worst outcome
///     wins, <c>LastSuccessfulRead</c> advances if any tag succeeded) because per-item writes to
///     the shared health field would race under the window.
/// </summary>
[Trait("Category", "Unit")]
public sealed class AbCipParallelReadTests
{
    private const string DeviceHost = "ab://10.0.0.5/1,0";

    /// <summary>
    ///     Holds every in-flight fake read until either <paramref name="target"/> reads overlap
    ///     (proving the window opened) or a short grace period elapses (so a window of one, which
    ///     can never reach the target, still completes instead of deadlocking).
    /// </summary>
    private sealed class ReadGate(int target)
    {
        private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _inFlight;

        /// <summary>Gets the highest number of fake reads observed in flight simultaneously.</summary>
        public int MaxConcurrent { get; private set; }

        /// <summary>Records one in-flight read and holds it open until the gate releases.</summary>
        /// <param name="cancellationToken">The read's cancellation token.</param>
        public async Task HoldAsync(CancellationToken cancellationToken)
        {
            var n = Interlocked.Increment(ref _inFlight);
            lock (this)
            {
                if (n > MaxConcurrent) MaxConcurrent = n;
            }
            try
            {
                if (n >= target) _release.TrySetResult();
                try
                {
                    await _release.Task.WaitAsync(TimeSpan.FromMilliseconds(150), cancellationToken)
                        .ConfigureAwait(false);
                }
                catch (TimeoutException)
                {
                    // The window never reached the target — the sequential control case.
                }
            }
            finally
            {
                Interlocked.Decrement(ref _inFlight);
            }
        }
    }

    /// <summary>A fake whose read blocks on the shared gate and records its OWN overlap.</summary>
    private sealed class GatedFake(AbCipTagCreateParams p, ReadGate gate) : FakeAbCipTag(p)
    {
        private int _own;

        /// <summary>Gets the highest number of concurrent reads seen on THIS handle.</summary>
        public int MaxOwnConcurrent { get; private set; }

        /// <inheritdoc />
        public override async Task ReadAsync(CancellationToken cancellationToken)
        {
            var mine = Interlocked.Increment(ref _own);
            lock (this)
            {
                if (mine > MaxOwnConcurrent) MaxOwnConcurrent = mine;
            }
            try
            {
                await gate.HoldAsync(cancellationToken).ConfigureAwait(false);
                await base.ReadAsync(cancellationToken).ConfigureAwait(false);
            }
            finally
            {
                Interlocked.Decrement(ref _own);
            }
        }
    }

    private static AbCipDriver NewDriver(
        IAbCipTagFactory factory, int maxConcurrentReads, params AbCipTagDefinition[] tags)
    {
        var opts = new AbCipDriverOptions
        {
            Devices = [new AbCipDeviceOptions(DeviceHost)],
            RawTags = AbCipRawTags.From(tags),
            Probe = new AbCipProbeOptions { Enabled = false },
            MaxConcurrentReads = maxConcurrentReads,
        };
        return new AbCipDriver(opts, "drv-1", factory);
    }

    private static AbCipTagDefinition Tag(string name) =>
        new(name, DeviceHost, $"Motor1.{name}", AbCipDataType.DInt);

    /// <summary>Four distinct tags under a window of eight overlap on the wire.</summary>
    [Fact]
    public async Task Reads_overlap_up_to_the_window()
    {
        var gate = new ReadGate(target: 2);
        var factory = new FakeAbCipTagFactory { Customise = p => new GatedFake(p, gate) { Value = 1 } };
        var drv = NewDriver(factory, maxConcurrentReads: 8,
            Tag("A"), Tag("B"), Tag("C"), Tag("D"));
        await drv.InitializeAsync("{}", CancellationToken.None);

        var snapshots = await drv.ReadAsync(["A", "B", "C", "D"], CancellationToken.None);

        gate.MaxConcurrent.ShouldBeGreaterThanOrEqualTo(2,
            "a window of 8 must let distinct tags overlap on the device");
        snapshots.Count.ShouldBe(4);
        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.Good);
    }

    /// <summary>
    ///     Falsification control for <see cref="Reads_overlap_up_to_the_window"/> — the same
    ///     fixture with a window of one must observe no overlap at all, i.e. today's behaviour.
    /// </summary>
    [Fact]
    public async Task A_window_of_one_is_sequential()
    {
        var gate = new ReadGate(target: 2);
        var factory = new FakeAbCipTagFactory { Customise = p => new GatedFake(p, gate) { Value = 1 } };
        var drv = NewDriver(factory, maxConcurrentReads: 1,
            Tag("A"), Tag("B"), Tag("C"), Tag("D"));
        await drv.InitializeAsync("{}", CancellationToken.None);

        var snapshots = await drv.ReadAsync(["A", "B", "C", "D"], CancellationToken.None);

        gate.MaxConcurrent.ShouldBe(1, "MaxConcurrentReads = 1 must be observably sequential");
        snapshots.Count.ShouldBe(4);
        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.Good);
    }

    /// <summary>
    ///     The window parallelises ACROSS tags, never within one — the same reference twice in a
    ///     batch still serialises on the per-tag runtime lock (05/STAB-4).
    /// </summary>
    [Fact]
    public async Task Same_tag_requested_twice_is_still_serialised()
    {
        var gate = new ReadGate(target: 2);
        GatedFake? shared = null;
        var factory = new FakeAbCipTagFactory
        {
            Customise = p => shared = new GatedFake(p, gate) { Value = 5 },
        };
        var drv = NewDriver(factory, maxConcurrentReads: 8, Tag("A"));
        await drv.InitializeAsync("{}", CancellationToken.None);

        var snapshots = await drv.ReadAsync(["A", "A"], CancellationToken.None);

        shared.ShouldNotBeNull();
        shared!.MaxOwnConcurrent.ShouldBe(1,
            "operations on one shared libplctag Tag handle must never overlap");
        snapshots.Count.ShouldBe(2);
        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.Good);
    }

    /// <summary>
    ///     Counts + overlaps fake <c>InitializeAsync</c> (the Forward-Open stand-in) calls. Holds
    ///     every in-flight connect until either <paramref name="target"/> overlap (proving the
    ///     connects were NOT single-flight) or a short grace period elapses, so a correctly
    ///     serialised run still completes instead of deadlocking.
    /// </summary>
    private sealed class ConnectProbe(int target)
    {
        private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _inFlight;
        private int _attempts;

        /// <summary>Gets the highest number of connect attempts observed in flight simultaneously.</summary>
        public int MaxConcurrent { get; private set; }

        /// <summary>Gets the total number of connect attempts that reached the wire.</summary>
        public int Attempts => Volatile.Read(ref _attempts);

        /// <summary>Records one in-flight connect and holds it open until the probe releases.</summary>
        /// <param name="cancellationToken">The connect's cancellation token.</param>
        public async Task HoldAsync(CancellationToken cancellationToken)
        {
            Interlocked.Increment(ref _attempts);
            var n = Interlocked.Increment(ref _inFlight);
            lock (this)
            {
                if (n > MaxConcurrent) MaxConcurrent = n;
            }
            try
            {
                if (n >= target) _release.TrySetResult();
                try
                {
                    await _release.Task.WaitAsync(TimeSpan.FromMilliseconds(150), cancellationToken)
                        .ConfigureAwait(false);
                }
                catch (TimeoutException)
                {
                    // No second connect overlapped — the single-flight case.
                }
            }
            finally
            {
                Interlocked.Decrement(ref _inFlight);
            }
        }
    }

    /// <summary>A fake standing in for a device that is down — every connect attempt fails.</summary>
    private sealed class DownDeviceFake(AbCipTagCreateParams p, ConnectProbe probe) : FakeAbCipTag(p)
    {
        /// <inheritdoc />
        public override async Task InitializeAsync(CancellationToken cancellationToken)
        {
            await probe.HoldAsync(cancellationToken).ConfigureAwait(false);
            throw new InvalidOperationException("fake Forward-Open failure");
        }
    }

    /// <summary>
    ///     Review fix — cold (never-connected) tags on ONE down device must connect single-flight
    ///     under the parallel read window. The window let up to <c>MaxConcurrentReads</c> work items
    ///     run the unsynchronised <c>ShouldAttempt</c> → connect → <c>RecordFailure</c> sequence on
    ///     the shared per-device <c>ConnectionBackoff</c> concurrently: every cold tag passed
    ///     <c>ShouldAttempt</c> before any failure was recorded (N simultaneous Forward-Opens — a
    ///     thundering herd at the device), and the unsynchronised <c>_consecutiveFailures++</c> lost
    ///     increments, weakening the 05/STAB-8 backoff into a reconnect storm.
    /// </summary>
    [Fact]
    public async Task Cold_tags_on_one_down_device_connect_single_flight()
    {
        var probe = new ConnectProbe(target: 2);
        var factory = new FakeAbCipTagFactory { Customise = p => new DownDeviceFake(p, probe) };
        var drv = NewDriver(factory, maxConcurrentReads: 8,
            Tag("A"), Tag("B"), Tag("C"), Tag("D"), Tag("E"), Tag("F"));
        await drv.InitializeAsync("{}", CancellationToken.None);

        var snapshots = await drv.ReadAsync(["A", "B", "C", "D", "E", "F"], CancellationToken.None);

        probe.MaxConcurrent.ShouldBe(1,
            "cold connects against ONE device must be single-flight — the backoff check + connect + " +
            "record sequence is not internally synchronized");
        probe.Attempts.ShouldBe(1,
            "the one recorded failure must be visible to every sibling still inside the read window, " +
            "so only ONE Forward-Open reaches a down device per batch");
        snapshots.Count.ShouldBe(6);
        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.BadCommunicationError);

        // The failure increment landed and is durable: the 05/STAB-8 window opened by that single
        // recorded failure (base 1 s) is still open, so a second batch makes NO further attempt.
        var again = await drv.ReadAsync(["A", "B", "C", "D", "E", "F"], CancellationToken.None);

        probe.Attempts.ShouldBe(1,
            "no lost increment — the recorded failure must still hold the backoff window shut");
        again.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.BadCommunicationError);
    }

    /// <summary>
    ///     Health is written once after the batch: the worst per-item outcome wins, and
    ///     <c>LastSuccessfulRead</c> still advances because at least one tag read succeeded — the
    ///     contract the comms-loss watchdog reads.
    /// </summary>
    [Fact]
    public async Task Health_after_a_mixed_batch_is_worst_outcome_with_advanced_LastSuccessfulRead()
    {
        var factory = new FakeAbCipTagFactory
        {
            Customise = p => p.TagName.Contains("Bad", StringComparison.Ordinal)
                ? new FakeAbCipTag(p) { ThrowOnRead = true }
                : new FakeAbCipTag(p) { Value = 3 },
        };
        var drv = NewDriver(factory, maxConcurrentReads: 8, Tag("Good"), Tag("Bad"));
        await drv.InitializeAsync("{}", CancellationToken.None);
        var beforeRead = drv.GetHealth().LastSuccessfulRead;
        beforeRead.ShouldNotBeNull();
        await Task.Delay(30, TestContext.Current.CancellationToken);

        var snapshots = await drv.ReadAsync(["Good", "Bad"], CancellationToken.None);

        snapshots[0].StatusCode.ShouldBe(AbCipStatusMapper.Good);
        snapshots[1].StatusCode.ShouldBe(AbCipStatusMapper.BadCommunicationError);
        var health = drv.GetHealth();
        health.State.ShouldBe(DriverState.Degraded, "the worst per-item outcome wins");
        health.LastSuccessfulRead.ShouldNotBeNull();
        health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(beforeRead!.Value,
            "a successful tag in the batch must still advance LastSuccessfulRead");
    }
}