ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionStaleConnectResultTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;

/// <summary>
/// The connect leg of the stale-completion family that
/// <c>DataConnectionStaleCompletionTests</c> pins for the subscribe leg.
///
/// <para>
/// <b>The gap these pin.</b> <c>ConnectResult</c> was piped untagged, so a connect launched
/// against adapter generation N and answered after the adapter had been swapped (failover, or the
/// in-place reconfigure) was applied to its REPLACEMENT. A stale success drove
/// <c>ReSubscribeAll</c> and <c>BecomeConnected</c> against an adapter that had never connected;
/// the replacement's own result then landed in the Connected state, where nothing handles it. With
/// a bad new endpoint the batch subscribes failed at connection level with a log line only, no
/// <c>AdapterDisconnected</c> ever fired, and the idempotent-reconfigure guard refused a re-send
/// of the same configuration — only a restart recovered the connection. A stale FAILURE was just
/// as wrong: it counted toward failover and re-armed the reconnect timer underneath the attempt
/// that was genuinely in flight.
/// </para>
///
/// <para>
/// The state machine must also stay able to finish the CURRENT attempt after a stale result was
/// dropped — Connecting/Reconnecting is held until the matching-generation result arrives.
/// </para>
/// </summary>
[Collection("DataConnectionManagerActor")]
public class DataConnectionStaleConnectResultTests : TestKit
{
    private const string FirstEndpoint = "opc.tcp://first-plc:4840";
    private const string MidEndpoint = "opc.tcp://mid-plc:4840";
    private const string LastEndpoint = "opc.tcp://last-plc:4840";
    private const string Tag = "sensor/temp";

    private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
    private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();

    private readonly DataConnectionOptions _options = new()
    {
        ReconnectInterval = TimeSpan.FromMilliseconds(100),
        TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
        WriteTimeout = TimeSpan.FromSeconds(5),
        SeedReadMaxAttempts = 1,
        SeedReadRetryDelay = TimeSpan.FromMilliseconds(10)
    };

    private static Dictionary<string, string> Config(string endpoint) =>
        new() { ["Endpoint"] = endpoint };

    private IActorRef CreateConnectionActor(string name, IDataConnection adapter) =>
        Sys.ActorOf(Props.Create(() =>
            new DataConnectionActor(
                name, adapter, _options, _health, _factory, "OpcUa",
                Config(FirstEndpoint), null, 3)), name);

    private void ExpectAdapterFor(string endpoint, IDataConnection adapter) =>
        _factory.Create("OpcUa", Arg.Is<IDictionary<string, string>>(d => d["Endpoint"] == endpoint))
            .Returns(adapter);

    /// <summary>An adapter whose connect completes when <paramref name="connect"/> does.</summary>
    private static IDataConnection Adapter(Task connect, string subscriptionId)
    {
        var adapter = Substitute.For<IDataConnection>();
        adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>())
            .Returns(connect);
        adapter.Status.Returns(ConnectionHealth.Connected);
        adapter.SubscribeAsync(Arg.Any<string>(), Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns(subscriptionId);
        adapter.ReadAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
            .Returns(new ReadResult(false, null, null));
        return adapter;
    }

    private int HealthTransitions(string connectionName, ConnectionHealth health) =>
        _health.ReceivedCalls().Count(c =>
            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) &&
            c.GetArguments() is [string name, ConnectionHealth reported, ..] &&
            name == connectionName && reported == health);

    private static int CallCount(IDataConnection adapter, string method) =>
        adapter.ReceivedCalls().Count(c => c.GetMethodInfo().Name == method);

    /// <summary>
    /// Drives the actor to: connected on the first adapter with one live tag subscription, then
    /// reconfigured onto <paramref name="mid"/> (whose connect is still in flight), then
    /// reconfigured again onto <paramref name="last"/> (whose connect is also still in flight).
    /// The mid adapter's pending connect is now two generations stale.
    /// </summary>
    private IActorRef ArrangeTwoGenerationsBehind(
        string name, IDataConnection first, IDataConnection mid, IDataConnection last)
    {
        ExpectAdapterFor(MidEndpoint, mid);
        ExpectAdapterFor(LastEndpoint, last);

        var actor = CreateConnectionActor(name, first);
        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        var probe = CreateTestProbe();
        probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        probe.FishForMessage<SubscribeTagsResponse>(m => m.Success, TimeSpan.FromSeconds(5));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(MidEndpoint), null, 3));
        AwaitCondition(() => CallCount(mid, nameof(IDataConnection.ConnectAsync)) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(LastEndpoint), null, 3));
        AwaitCondition(() => CallCount(last, nameof(IDataConnection.ConnectAsync)) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));

        return actor;
    }

    /// <summary>
    /// The headline case: a success from the adapter the operator has just corrected AWAY from
    /// must not declare the connection up, and must not re-issue the subscription registry
    /// against the replacement adapter that has not connected yet.
    ///
    /// <para><b>Fails before the fix:</b> the actor entered Connected and re-subscribed
    /// <c>Tag</c> on the never-connected last adapter.</para>
    /// </summary>
    [Fact]
    public void StaleConnectSuccess_DoesNotConnect_AndDoesNotResubscribeAgainstTheNewAdapter()
    {
        const string name = "stale-connect-success";
        var midGate = new TaskCompletionSource();
        var lastGate = new TaskCompletionSource();

        var first = Adapter(Task.CompletedTask, "sub-first");
        var mid = Adapter(midGate.Task, "sub-mid");
        var last = Adapter(lastGate.Task, "sub-last");

        ArrangeTwoGenerationsBehind(name, first, mid, last);

        // The two-generations-old connect finally answers — successfully.
        midGate.SetResult();
        // Settle: nothing here subscribes the default test actor, so this is purely a wait long
        // enough for the piped result to have been received and dropped.
        ExpectNoMsg(TimeSpan.FromMilliseconds(500));

        Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected));
        Assert.Equal(0, CallCount(last, nameof(IDataConnection.SubscribeAsync)));
    }

    /// <summary>
    /// A stale FAILURE must not touch the attempt that is genuinely in flight. Acting on it
    /// counted toward the failover budget and re-armed the reconnect timer, so a second
    /// <c>ConnectAsync</c> was launched against an adapter already connecting.
    ///
    /// <para><b>Fails before the fix:</b> the last adapter was asked to connect a second time
    /// one <c>ReconnectInterval</c> after the stale failure landed.</para>
    /// </summary>
    [Fact]
    public void StaleConnectFailure_DoesNotDisturbTheCurrentAttempt()
    {
        const string name = "stale-connect-failure";
        var midGate = new TaskCompletionSource();
        var lastGate = new TaskCompletionSource();

        var first = Adapter(Task.CompletedTask, "sub-first");
        var mid = Adapter(midGate.Task, "sub-mid");
        var last = Adapter(lastGate.Task, "sub-last");

        ArrangeTwoGenerationsBehind(name, first, mid, last);

        midGate.SetException(new Exception("connection refused"));
        // Comfortably longer than ReconnectInterval, so a re-armed timer would have fired.
        ExpectNoMsg(TimeSpan.FromMilliseconds(600));

        Assert.Equal(1, CallCount(last, nameof(IDataConnection.ConnectAsync)));
        Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected));
    }

    /// <summary>
    /// Dropping a stale result must not wedge the state machine: the actor stays in Reconnecting
    /// and the CURRENT generation's own result is still processed normally — connection up, and
    /// the whole subscription registry transparently re-subscribed against the live adapter.
    /// </summary>
    [Fact]
    public void CurrentGenerationConnectResult_IsStillProcessed_AfterAStaleOneWasDropped()
    {
        const string name = "stale-connect-then-live";
        var midGate = new TaskCompletionSource();
        var lastGate = new TaskCompletionSource();

        var first = Adapter(Task.CompletedTask, "sub-first");
        var mid = Adapter(midGate.Task, "sub-mid");
        var last = Adapter(lastGate.Task, "sub-last");

        ArrangeTwoGenerationsBehind(name, first, mid, last);

        midGate.SetResult();
        ExpectNoMsg(TimeSpan.FromMilliseconds(300));
        Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected));

        // Now the live adapter answers.
        lastGate.SetResult();

        AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        AwaitCondition(() => CallCount(last, nameof(IDataConnection.SubscribeAsync)) >= 1,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
    }
}