ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionStaleCompletionTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;

/// <summary>
/// An in-flight subscribe that completes AFTER the adapter
/// has been swapped out (failover, or the in-place reconfigure of register row 97).
///
/// <para>
/// <b>The gap these pin.</b> <c>SubscribeCompleted</c>/<c>AlarmSubscribeCompleted</c> carried no
/// adapter generation, and their handlers applied them unconditionally — while an initial
/// subscribe can stay in flight for up to <c>SeedOverallTimeout</c> (120 s), so the window is
/// wide. A stale completion stored a DEAD session's handle under the tag's key, which then made
/// the live completion for the same tag look like a duplicate and get released: the tag reads
/// resolved forever and delivers nothing. It also re-stamped the old endpoint's seed values with
/// the CURRENT generation (bypassing the stale-data guard) and let a connection-level failure row
/// from the dead adapter knock the healthy new connection back into Reconnecting. The alarm twin
/// released the live alarm feed the same way.
/// </para>
///
/// <para>
/// Handles are adapter-scoped — MxGateway's are plain ints that collide across sessions — so a
/// dropped completion's handles are released against the adapter that CREATED them, never the
/// current one.
/// </para>
/// </summary>
[Collection("DataConnectionManagerActor")]
public class DataConnectionStaleCompletionTests : TestKit
{
    private const string OldEndpoint = "opc.tcp://old-plc:4840";
    private const string NewEndpoint = "opc.tcp://new-plc:4840";
    private const string Tag = "sensor/temp";
    private const string Source = "Tank01";

    private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
    private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();

    private readonly DataConnectionOptions _options = new()
    {
        ReconnectInterval = TimeSpan.FromMilliseconds(100),
        TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
        WriteTimeout = TimeSpan.FromSeconds(5),
        SeedReadMaxAttempts = 1,
        SeedReadRetryDelay = TimeSpan.FromMilliseconds(10)
    };

    private static Dictionary<string, string> Config(string endpoint) =>
        new() { ["Endpoint"] = endpoint };

    private IActorRef CreateConnectionActor(string name, IDataConnection adapter) =>
        Sys.ActorOf(Props.Create(() =>
            new DataConnectionActor(
                name, adapter, _options, _health, _factory, "OpcUa",
                Config(OldEndpoint), null, 3)), name);

    private void ExpectNewAdapter(IDataConnection adapter) =>
        _factory.Create("OpcUa", Arg.Is<IDictionary<string, string>>(d => d["Endpoint"] == NewEndpoint))
            .Returns(adapter);

    private static IDataConnection HealthyAdapter(string subscriptionId)
    {
        var adapter = Substitute.For<IDataConnection>();
        adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>())
            .Returns(Task.CompletedTask);
        adapter.Status.Returns(ConnectionHealth.Connected);
        adapter.SubscribeAsync(Arg.Any<string>(), Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(subscriptionId);
        adapter.ReadAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
            .Returns(new ReadResult(false, null, null));
        return adapter;
    }

    private static IDataConnection HealthyAlarmAdapter(string alarmSubscriptionId)
    {
        var adapter = Substitute.For<IDataConnection, IAlarmSubscribableConnection>();
        adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>())
            .Returns(Task.CompletedTask);
        adapter.Status.Returns(ConnectionHealth.Connected);
        ((IAlarmSubscribableConnection)adapter)
            .SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(),
                Arg.Any<AlarmTransitionCallback>(), Arg.Any<CancellationToken>())
            .Returns(Task.FromResult(alarmSubscriptionId));
        return adapter;
    }

    private static NativeAlarmTransition Raise(string sourceRef, string sourceObj) =>
        new(sourceRef, sourceObj, "AnalogLimit.Hi", AlarmTransitionKind.Raise,
            new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 500),
            "Process", "hi", "hi", "", "", null, DateTimeOffset.UtcNow, "92", "90");

    private int HealthTransitions(string connectionName, ConnectionHealth health) =>
        _health.ReceivedCalls().Count(c =>
            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) &&
            c.GetArguments() is [string name, ConnectionHealth reported, ..] &&
            name == connectionName && reported == health);

    private static bool CalledWith(IDataConnection adapter, string method, string firstArg) =>
        adapter.ReceivedCalls().Any(c =>
            c.GetMethodInfo().Name == method && c.GetArguments()[0] as string == firstArg);

    private static bool Called(IDataConnection adapter, string method) =>
        adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == method);

    // ── Tag subscribe ──

    /// <summary>
    /// The headline data-loss case: the stale completion must not park its dead handle under the
    /// tag's key, because the live completion for the same tag then reads as a duplicate and has
    /// its (working) monitored item released — leaving the tag permanently resolved and silent.
    ///
    /// <para>
    /// <b>Fails before the fix:</b> the stale completion stored <c>sub-old</c>, and the live
    /// handle <c>sub-new</c> was released against the new adapter.
    /// </para>
    /// </summary>
    [Fact]
    public void StaleSubscribeCompleted_DoesNotStoreItsHandle_AndLeavesTheLiveMonitoredItemAlone()
    {
        const string name = "stale-sub-handle";
        var gate = new TaskCompletionSource<string>();

        var oldAdapter = HealthyAdapter("sub-old");
        oldAdapter.SubscribeAsync(Tag, Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(_ => gate.Task);
        var newAdapter = HealthyAdapter("sub-new");
        ExpectNewAdapter(newAdapter);

        var actor = CreateConnectionActor(name, oldAdapter);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
            TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        // The Instance Actor's retry — the live subscribe, against the new adapter.
        probe.Send(actor, new SubscribeTagsRequest("c2", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        probe.FishForMessage<SubscribeTagsResponse>(
            m => m.CorrelationId == "c2" && m.Success, TimeSpan.FromSeconds(5));

        // Only now does the generation-N subscribe answer.
        gate.SetResult("sub-old");
        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"),
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync)));

        // The registry still holds the LIVE handle: the teardown releases sub-new, not sub-old.
        probe.Send(actor, new UnsubscribeTagsRequest("c3", "inst1", name, DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-new"),
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        Assert.False(CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"));
    }

    /// <summary>
    /// The handles a dropped completion did create are released — against the adapter that
    /// created them. Releasing them against the CURRENT adapter is not a harmless mismatch:
    /// MxGateway subscription handles are plain ints reused across sessions, so the release
    /// would land on whatever the new session happens to have under that number.
    /// </summary>
    [Fact]
    public void StaleSubscribeCompleted_ReleasesItsHandles_AgainstTheAdapterThatCreatedThem()
    {
        const string name = "stale-sub-release";
        var gate = new TaskCompletionSource<string>();

        var oldAdapter = HealthyAdapter("sub-old");
        oldAdapter.SubscribeAsync(Tag, Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(_ => gate.Task);
        var newAdapter = HealthyAdapter("sub-new");
        ExpectNewAdapter(newAdapter);

        var actor = CreateConnectionActor(name, oldAdapter);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
            TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        gate.SetResult("sub-old");

        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"),
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync)));

        // The requester is told the subscribe did not take, so its retry timer re-sends: the
        // dropped completion registered nothing, so the reconnect re-subscribe cannot cover it.
        probe.FishForMessage<SubscribeTagsResponse>(
            m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5));
    }

    /// <summary>
    /// The seed values a dropped completion carries are readings from the endpoint the operator
    /// has just corrected AWAY from. Applying them re-stamped each one with the CURRENT adapter
    /// generation, which is precisely what the generation guard on <c>TagValueReceived</c>
    /// exists to stop — the old endpoint's process values presented as fresh.
    /// </summary>
    [Fact]
    public void StaleSubscribeCompleted_DoesNotDeliverItsSeedValues()
    {
        const string name = "stale-sub-seed";
        var gate = new TaskCompletionSource<string>();

        var oldAdapter = HealthyAdapter("sub-old");
        oldAdapter.SubscribeAsync(Tag, Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(_ => gate.Task);
        oldAdapter.ReadAsync(Tag, Arg.Any<CancellationToken>())
            .Returns(new ReadResult(true, new TagValue(999, QualityCode.Good, DateTimeOffset.UtcNow), null));
        var newAdapter = HealthyAdapter("sub-new");
        ExpectNewAdapter(newAdapter);

        var actor = CreateConnectionActor(name, oldAdapter);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
            TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        gate.SetResult("sub-old");

        var received = probe.ReceiveWhile(TimeSpan.FromSeconds(2), m => m);

        // Positive control: the completion really did arrive and was dropped, rather than the
        // window simply having been too short to observe anything.
        Assert.Contains(received, m => m is SubscribeTagsResponse { Success: false });
        Assert.DoesNotContain(received, m => m is TagValueUpdate u && Equals(u.Value, 999));
    }

    /// <summary>
    /// A connection-level failure row belongs to the adapter that produced it. Acting on a stale
    /// one drives the healthy replacement connection into Reconnecting — dropping every tag to
    /// bad quality and re-subscribing the whole registry for nothing.
    /// </summary>
    [Fact]
    public void StaleConnectionLevelFailure_DoesNotDriveTheNewConnectionBackToReconnecting()
    {
        const string name = "stale-sub-connfail";
        var gate = new TaskCompletionSource<string>();

        var oldAdapter = HealthyAdapter("sub-old");
        oldAdapter.SubscribeAsync(Tag, Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(_ => gate.Task);
        var newAdapter = HealthyAdapter("sub-new");
        ExpectNewAdapter(newAdapter);

        var actor = CreateConnectionActor(name, oldAdapter);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
            TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var disconnectsBefore = HealthTransitions(name, ConnectionHealth.Disconnected);

        // InvalidOperationException is what IsConnectionLevelFailure classifies as "transport
        // down", i.e. the row that asks the state machine to reconnect.
        gate.SetException(new InvalidOperationException("adapter not connected"));

        probe.FishForMessage<SubscribeTagsResponse>(
            m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5));

        // Entering Reconnecting would push bad quality to every subscriber — the probe is one.
        probe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
        Assert.Equal(disconnectsBefore, HealthTransitions(name, ConnectionHealth.Disconnected));
        Assert.Equal(2, HealthTransitions(name, ConnectionHealth.Connected));
    }

    // ── Alarm subscribe ──

    /// <summary>
    /// The alarm twin. The stale completion lands first and (before the fix) stored its dead feed
    /// id, so the LIVE feed's completion took the duplicate branch and released the feed the new
    /// adapter had just established — a source that stays mirrored but never updates again.
    /// </summary>
    [Fact]
    public void StaleAlarmSubscribeCompleted_DoesNotReleaseTheLiveAlarmFeed()
    {
        const string name = "stale-alarm-feed";
        var oldGate = new TaskCompletionSource<string>();
        var newGate = new TaskCompletionSource<string>();
        AlarmTransitionCallback? newCallback = null;

        var oldAdapter = HealthyAlarmAdapter("alarm-old");
        ((IAlarmSubscribableConnection)oldAdapter)
            .SubscribeAlarmsAsync(Source, Arg.Any<string?>(),
                Arg.Any<AlarmTransitionCallback>(), Arg.Any<CancellationToken>())
            .Returns(_ => oldGate.Task);

        var newAdapter = HealthyAlarmAdapter("alarm-new");
        ((IAlarmSubscribableConnection)newAdapter)
            .SubscribeAlarmsAsync(Source, Arg.Any<string?>(),
                Arg.Do<AlarmTransitionCallback>(c => newCallback = c), Arg.Any<CancellationToken>())
            .Returns(_ => newGate.Task);
        ExpectNewAdapter(newAdapter);

        var actor = CreateConnectionActor(name, oldAdapter);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeAlarmsRequest("c1", "inst1", name, Source, null, DateTimeOffset.UtcNow));
        AwaitCondition(() => CalledWith(oldAdapter, "SubscribeAlarmsAsync", Source), TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));

        // The subscriber registry survives the swap, so the reconnect re-subscribe re-opens the
        // feed against the new adapter without the NativeAlarmActor doing anything.
        AwaitCondition(() => CalledWith(newAdapter, "SubscribeAlarmsAsync", Source),
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        // The dead feed answers first, the live one second — the ordering that made the live
        // completion look redundant.
        oldGate.SetResult("alarm-old");
        AwaitCondition(() => CalledWith(oldAdapter, "UnsubscribeAlarmsAsync", "alarm-old"),
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        newGate.SetResult("alarm-new");

        AwaitCondition(() => newCallback != null, TimeSpan.FromSeconds(5));
        // Settle (and drain the swap's expected NativeAlarmSourceUnavailable) so the live
        // completion has certainly been applied before asserting nothing released its feed.
        probe.ReceiveWhile(TimeSpan.FromMilliseconds(300), m => m);
        Assert.False(Called(newAdapter, "UnsubscribeAlarmsAsync"));

        // And the feed is genuinely live: a transition on it still routes to the subscriber.
        newCallback!(Raise("Tank01.Hi", Source));
        probe.FishForMessage<NativeAlarmTransitionUpdate>(
            u => u.Transition.SourceObjectReference == Source, TimeSpan.FromSeconds(5));
    }
}