using Akka.Actor; using Akka.TestKit.Xunit2; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors; /// /// Arch-review 2026-08-19 P2-3 — the connect leg of the stale-completion family that /// DataConnectionStaleCompletionTests pins for the subscribe leg. /// /// /// The gap these pin. ConnectResult was piped untagged, so a connect launched /// against adapter generation N and answered after the adapter had been swapped (failover, or the /// row-97 in-place reconfigure) was applied to its REPLACEMENT. A stale success drove /// ReSubscribeAll and BecomeConnected against an adapter that had never connected; /// the replacement's own result then landed in the Connected state, where nothing handles it. With /// a bad new endpoint the batch subscribes failed at connection level with a log line only, no /// AdapterDisconnected ever fired, and the idempotent-reconfigure guard refused a re-send /// of the same configuration — only a restart recovered the connection. A stale FAILURE was just /// as wrong: it counted toward failover and re-armed the reconnect timer underneath the attempt /// that was genuinely in flight. /// /// /// /// The state machine must also stay able to finish the CURRENT attempt after a stale result was /// dropped — Connecting/Reconnecting is held until the matching-generation result arrives. /// /// [Collection("DataConnectionManagerActor")] public class DataConnectionStaleConnectResultTests : TestKit { private const string FirstEndpoint = "opc.tcp://first-plc:4840"; private const string MidEndpoint = "opc.tcp://mid-plc:4840"; private const string LastEndpoint = "opc.tcp://last-plc:4840"; private const string Tag = "sensor/temp"; private readonly ISiteHealthCollector _health = Substitute.For(); private readonly IDataConnectionFactory _factory = Substitute.For(); private readonly DataConnectionOptions _options = new() { ReconnectInterval = TimeSpan.FromMilliseconds(100), TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200), WriteTimeout = TimeSpan.FromSeconds(5), SeedReadMaxAttempts = 1, SeedReadRetryDelay = TimeSpan.FromMilliseconds(10) }; private static Dictionary Config(string endpoint) => new() { ["Endpoint"] = endpoint }; private IActorRef CreateConnectionActor(string name, IDataConnection adapter) => Sys.ActorOf(Props.Create(() => new DataConnectionActor( name, adapter, _options, _health, _factory, "OpcUa", Config(FirstEndpoint), null, 3)), name); private void ExpectAdapterFor(string endpoint, IDataConnection adapter) => _factory.Create("OpcUa", Arg.Is>(d => d["Endpoint"] == endpoint)) .Returns(adapter); /// An adapter whose connect completes when does. private static IDataConnection Adapter(Task connect, string subscriptionId) { var adapter = Substitute.For(); adapter.ConnectAsync(Arg.Any>(), Arg.Any()) .Returns(connect); adapter.Status.Returns(ConnectionHealth.Connected); adapter.SubscribeAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(subscriptionId); adapter.ReadAsync(Arg.Any(), Arg.Any()) .Returns(new ReadResult(false, null, null)); return adapter; } private int HealthTransitions(string connectionName, ConnectionHealth health) => _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) && c.GetArguments() is [string name, ConnectionHealth reported, ..] && name == connectionName && reported == health); private static int CallCount(IDataConnection adapter, string method) => adapter.ReceivedCalls().Count(c => c.GetMethodInfo().Name == method); /// /// Drives the actor to: connected on the first adapter with one live tag subscription, then /// reconfigured onto (whose connect is still in flight), then /// reconfigured again onto (whose connect is also still in flight). /// The mid adapter's pending connect is now two generations stale. /// private IActorRef ArrangeTwoGenerationsBehind( string name, IDataConnection first, IDataConnection mid, IDataConnection last) { ExpectAdapterFor(MidEndpoint, mid); ExpectAdapterFor(LastEndpoint, last); var actor = CreateConnectionActor(name, first); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); probe.FishForMessage(m => m.Success, TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(MidEndpoint), null, 3)); AwaitCondition(() => CallCount(mid, nameof(IDataConnection.ConnectAsync)) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(LastEndpoint), null, 3)); AwaitCondition(() => CallCount(last, nameof(IDataConnection.ConnectAsync)) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); return actor; } /// /// The headline case: a success from the adapter the operator has just corrected AWAY from /// must not declare the connection up, and must not re-issue the subscription registry /// against the replacement adapter that has not connected yet. /// /// Fails before the fix: the actor entered Connected and re-subscribed /// Tag on the never-connected last adapter. /// [Fact] public void StaleConnectSuccess_DoesNotConnect_AndDoesNotResubscribeAgainstTheNewAdapter() { const string name = "stale-connect-success"; var midGate = new TaskCompletionSource(); var lastGate = new TaskCompletionSource(); var first = Adapter(Task.CompletedTask, "sub-first"); var mid = Adapter(midGate.Task, "sub-mid"); var last = Adapter(lastGate.Task, "sub-last"); ArrangeTwoGenerationsBehind(name, first, mid, last); // The two-generations-old connect finally answers — successfully. midGate.SetResult(); // Settle: nothing here subscribes the default test actor, so this is purely a wait long // enough for the piped result to have been received and dropped. ExpectNoMsg(TimeSpan.FromMilliseconds(500)); Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected)); Assert.Equal(0, CallCount(last, nameof(IDataConnection.SubscribeAsync))); } /// /// A stale FAILURE must not touch the attempt that is genuinely in flight. Acting on it /// counted toward the failover budget and re-armed the reconnect timer, so a second /// ConnectAsync was launched against an adapter already connecting. /// /// Fails before the fix: the last adapter was asked to connect a second time /// one ReconnectInterval after the stale failure landed. /// [Fact] public void StaleConnectFailure_DoesNotDisturbTheCurrentAttempt() { const string name = "stale-connect-failure"; var midGate = new TaskCompletionSource(); var lastGate = new TaskCompletionSource(); var first = Adapter(Task.CompletedTask, "sub-first"); var mid = Adapter(midGate.Task, "sub-mid"); var last = Adapter(lastGate.Task, "sub-last"); ArrangeTwoGenerationsBehind(name, first, mid, last); midGate.SetException(new Exception("connection refused")); // Comfortably longer than ReconnectInterval, so a re-armed timer would have fired. ExpectNoMsg(TimeSpan.FromMilliseconds(600)); Assert.Equal(1, CallCount(last, nameof(IDataConnection.ConnectAsync))); Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected)); } /// /// Dropping a stale result must not wedge the state machine: the actor stays in Reconnecting /// and the CURRENT generation's own result is still processed normally — connection up, and /// the whole subscription registry transparently re-subscribed against the live adapter. /// [Fact] public void CurrentGenerationConnectResult_IsStillProcessed_AfterAStaleOneWasDropped() { const string name = "stale-connect-then-live"; var midGate = new TaskCompletionSource(); var lastGate = new TaskCompletionSource(); var first = Adapter(Task.CompletedTask, "sub-first"); var mid = Adapter(midGate.Task, "sub-mid"); var last = Adapter(lastGate.Task, "sub-last"); ArrangeTwoGenerationsBehind(name, first, mid, last); midGate.SetResult(); ExpectNoMsg(TimeSpan.FromMilliseconds(300)); Assert.Equal(1, HealthTransitions(name, ConnectionHealth.Connected)); // Now the live adapter answers. lastGate.SetResult(); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); AwaitCondition(() => CallCount(last, nameof(IDataConnection.SubscribeAsync)) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); } }