using Akka.Actor; using Akka.TestKit.Xunit2; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection; using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors; /// /// An in-flight subscribe that completes AFTER the adapter /// has been swapped out (failover, or the in-place reconfigure of register row 97). /// /// /// The gap these pin. SubscribeCompleted/AlarmSubscribeCompleted carried no /// adapter generation, and their handlers applied them unconditionally — while an initial /// subscribe can stay in flight for up to SeedOverallTimeout (120 s), so the window is /// wide. A stale completion stored a DEAD session's handle under the tag's key, which then made /// the live completion for the same tag look like a duplicate and get released: the tag reads /// resolved forever and delivers nothing. It also re-stamped the old endpoint's seed values with /// the CURRENT generation (bypassing the stale-data guard) and let a connection-level failure row /// from the dead adapter knock the healthy new connection back into Reconnecting. The alarm twin /// released the live alarm feed the same way. /// /// /// /// Handles are adapter-scoped — MxGateway's are plain ints that collide across sessions — so a /// dropped completion's handles are released against the adapter that CREATED them, never the /// current one. /// /// [Collection("DataConnectionManagerActor")] public class DataConnectionStaleCompletionTests : TestKit { private const string OldEndpoint = "opc.tcp://old-plc:4840"; private const string NewEndpoint = "opc.tcp://new-plc:4840"; private const string Tag = "sensor/temp"; private const string Source = "Tank01"; private readonly ISiteHealthCollector _health = Substitute.For(); private readonly IDataConnectionFactory _factory = Substitute.For(); private readonly DataConnectionOptions _options = new() { ReconnectInterval = TimeSpan.FromMilliseconds(100), TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200), WriteTimeout = TimeSpan.FromSeconds(5), SeedReadMaxAttempts = 1, SeedReadRetryDelay = TimeSpan.FromMilliseconds(10) }; private static Dictionary Config(string endpoint) => new() { ["Endpoint"] = endpoint }; private IActorRef CreateConnectionActor(string name, IDataConnection adapter) => Sys.ActorOf(Props.Create(() => new DataConnectionActor( name, adapter, _options, _health, _factory, "OpcUa", Config(OldEndpoint), null, 3)), name); private void ExpectNewAdapter(IDataConnection adapter) => _factory.Create("OpcUa", Arg.Is>(d => d["Endpoint"] == NewEndpoint)) .Returns(adapter); private static IDataConnection HealthyAdapter(string subscriptionId) { var adapter = Substitute.For(); adapter.ConnectAsync(Arg.Any>(), Arg.Any()) .Returns(Task.CompletedTask); adapter.Status.Returns(ConnectionHealth.Connected); adapter.SubscribeAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(subscriptionId); adapter.ReadAsync(Arg.Any(), Arg.Any()) .Returns(new ReadResult(false, null, null)); return adapter; } private static IDataConnection HealthyAlarmAdapter(string alarmSubscriptionId) { var adapter = Substitute.For(); adapter.ConnectAsync(Arg.Any>(), Arg.Any()) .Returns(Task.CompletedTask); adapter.Status.Returns(ConnectionHealth.Connected); ((IAlarmSubscribableConnection)adapter) .SubscribeAlarmsAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(alarmSubscriptionId)); return adapter; } private static NativeAlarmTransition Raise(string sourceRef, string sourceObj) => new(sourceRef, sourceObj, "AnalogLimit.Hi", AlarmTransitionKind.Raise, new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 500), "Process", "hi", "hi", "", "", null, DateTimeOffset.UtcNow, "92", "90"); private int HealthTransitions(string connectionName, ConnectionHealth health) => _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) && c.GetArguments() is [string name, ConnectionHealth reported, ..] && name == connectionName && reported == health); private static bool CalledWith(IDataConnection adapter, string method, string firstArg) => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == method && c.GetArguments()[0] as string == firstArg); private static bool Called(IDataConnection adapter, string method) => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == method); // ── Tag subscribe ── /// /// The headline data-loss case: the stale completion must not park its dead handle under the /// tag's key, because the live completion for the same tag then reads as a duplicate and has /// its (working) monitored item released — leaving the tag permanently resolved and silent. /// /// /// Fails before the fix: the stale completion stored sub-old, and the live /// handle sub-new was released against the new adapter. /// /// [Fact] public void StaleSubscribeCompleted_DoesNotStoreItsHandle_AndLeavesTheLiveMonitoredItemAlone() { const string name = "stale-sub-handle"; var gate = new TaskCompletionSource(); var oldAdapter = HealthyAdapter("sub-old"); oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any()) .Returns(_ => gate.Task); var newAdapter = HealthyAdapter("sub-new"); ExpectNewAdapter(newAdapter); var actor = CreateConnectionActor(name, oldAdapter); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag), TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3)); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); // The Instance Actor's retry — the live subscribe, against the new adapter. probe.Send(actor, new SubscribeTagsRequest("c2", "inst1", name, [Tag], DateTimeOffset.UtcNow)); probe.FishForMessage( m => m.CorrelationId == "c2" && m.Success, TimeSpan.FromSeconds(5)); // Only now does the generation-N subscribe answer. gate.SetResult("sub-old"); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"), TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync))); // The registry still holds the LIVE handle: the teardown releases sub-new, not sub-old. probe.Send(actor, new UnsubscribeTagsRequest("c3", "inst1", name, DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-new"), TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); Assert.False(CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old")); } /// /// The handles a dropped completion did create are released — against the adapter that /// created them. Releasing them against the CURRENT adapter is not a harmless mismatch: /// MxGateway subscription handles are plain ints reused across sessions, so the release /// would land on whatever the new session happens to have under that number. /// [Fact] public void StaleSubscribeCompleted_ReleasesItsHandles_AgainstTheAdapterThatCreatedThem() { const string name = "stale-sub-release"; var gate = new TaskCompletionSource(); var oldAdapter = HealthyAdapter("sub-old"); oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any()) .Returns(_ => gate.Task); var newAdapter = HealthyAdapter("sub-new"); ExpectNewAdapter(newAdapter); var actor = CreateConnectionActor(name, oldAdapter); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag), TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3)); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); gate.SetResult("sub-old"); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"), TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync))); // The requester is told the subscribe did not take, so its retry timer re-sends: the // dropped completion registered nothing, so the reconnect re-subscribe cannot cover it. probe.FishForMessage( m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5)); } /// /// The seed values a dropped completion carries are readings from the endpoint the operator /// has just corrected AWAY from. Applying them re-stamped each one with the CURRENT adapter /// generation, which is precisely what the generation guard on TagValueReceived /// exists to stop — the old endpoint's process values presented as fresh. /// [Fact] public void StaleSubscribeCompleted_DoesNotDeliverItsSeedValues() { const string name = "stale-sub-seed"; var gate = new TaskCompletionSource(); var oldAdapter = HealthyAdapter("sub-old"); oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any()) .Returns(_ => gate.Task); oldAdapter.ReadAsync(Tag, Arg.Any()) .Returns(new ReadResult(true, new TagValue(999, QualityCode.Good, DateTimeOffset.UtcNow), null)); var newAdapter = HealthyAdapter("sub-new"); ExpectNewAdapter(newAdapter); var actor = CreateConnectionActor(name, oldAdapter); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag), TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3)); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); gate.SetResult("sub-old"); var received = probe.ReceiveWhile(TimeSpan.FromSeconds(2), m => m); // Positive control: the completion really did arrive and was dropped, rather than the // window simply having been too short to observe anything. Assert.Contains(received, m => m is SubscribeTagsResponse { Success: false }); Assert.DoesNotContain(received, m => m is TagValueUpdate u && Equals(u.Value, 999)); } /// /// A connection-level failure row belongs to the adapter that produced it. Acting on a stale /// one drives the healthy replacement connection into Reconnecting — dropping every tag to /// bad quality and re-subscribing the whole registry for nothing. /// [Fact] public void StaleConnectionLevelFailure_DoesNotDriveTheNewConnectionBackToReconnecting() { const string name = "stale-sub-connfail"; var gate = new TaskCompletionSource(); var oldAdapter = HealthyAdapter("sub-old"); oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any()) .Returns(_ => gate.Task); var newAdapter = HealthyAdapter("sub-new"); ExpectNewAdapter(newAdapter); var actor = CreateConnectionActor(name, oldAdapter); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag), TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3)); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var disconnectsBefore = HealthTransitions(name, ConnectionHealth.Disconnected); // InvalidOperationException is what IsConnectionLevelFailure classifies as "transport // down", i.e. the row that asks the state machine to reconnect. gate.SetException(new InvalidOperationException("adapter not connected")); probe.FishForMessage( m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5)); // Entering Reconnecting would push bad quality to every subscriber — the probe is one. probe.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); Assert.Equal(disconnectsBefore, HealthTransitions(name, ConnectionHealth.Disconnected)); Assert.Equal(2, HealthTransitions(name, ConnectionHealth.Connected)); } // ── Alarm subscribe ── /// /// The alarm twin. The stale completion lands first and (before the fix) stored its dead feed /// id, so the LIVE feed's completion took the duplicate branch and released the feed the new /// adapter had just established — a source that stays mirrored but never updates again. /// [Fact] public void StaleAlarmSubscribeCompleted_DoesNotReleaseTheLiveAlarmFeed() { const string name = "stale-alarm-feed"; var oldGate = new TaskCompletionSource(); var newGate = new TaskCompletionSource(); AlarmTransitionCallback? newCallback = null; var oldAdapter = HealthyAlarmAdapter("alarm-old"); ((IAlarmSubscribableConnection)oldAdapter) .SubscribeAlarmsAsync(Source, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => oldGate.Task); var newAdapter = HealthyAlarmAdapter("alarm-new"); ((IAlarmSubscribableConnection)newAdapter) .SubscribeAlarmsAsync(Source, Arg.Any(), Arg.Do(c => newCallback = c), Arg.Any()) .Returns(_ => newGate.Task); ExpectNewAdapter(newAdapter); var actor = CreateConnectionActor(name, oldAdapter); AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); var probe = CreateTestProbe(); probe.Send(actor, new SubscribeAlarmsRequest("c1", "inst1", name, Source, null, DateTimeOffset.UtcNow)); AwaitCondition(() => CalledWith(oldAdapter, "SubscribeAlarmsAsync", Source), TimeSpan.FromSeconds(5)); actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3)); // The subscriber registry survives the swap, so the reconnect re-subscribe re-opens the // feed against the new adapter without the NativeAlarmActor doing anything. AwaitCondition(() => CalledWith(newAdapter, "SubscribeAlarmsAsync", Source), TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); // The dead feed answers first, the live one second — the ordering that made the live // completion look redundant. oldGate.SetResult("alarm-old"); AwaitCondition(() => CalledWith(oldAdapter, "UnsubscribeAlarmsAsync", "alarm-old"), TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); newGate.SetResult("alarm-new"); AwaitCondition(() => newCallback != null, TimeSpan.FromSeconds(5)); // Settle (and drain the swap's expected NativeAlarmSourceUnavailable) so the live // completion has certainly been applied before asserting nothing released its feed. probe.ReceiveWhile(TimeSpan.FromMilliseconds(300), m => m); Assert.False(Called(newAdapter, "UnsubscribeAlarmsAsync")); // And the feed is genuinely live: a transition on it still routes to the subscriber. newCallback!(Raise("Tank01.Hi", Source)); probe.FishForMessage( u => u.Transition.SourceObjectReference == Source, TimeSpan.FromSeconds(5)); } }