using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
///
/// Arch-review 2026-08-19 P2-1 — an in-flight subscribe that completes AFTER the adapter
/// has been swapped out (failover, or the in-place reconfigure of register row 97).
///
///
/// The gap these pin. SubscribeCompleted/AlarmSubscribeCompleted carried no
/// adapter generation, and their handlers applied them unconditionally — while an initial
/// subscribe can stay in flight for up to SeedOverallTimeout (120 s), so the window is
/// wide. A stale completion stored a DEAD session's handle under the tag's key, which then made
/// the live completion for the same tag look like a duplicate and get released: the tag reads
/// resolved forever and delivers nothing. It also re-stamped the old endpoint's seed values with
/// the CURRENT generation (bypassing the stale-data guard) and let a connection-level failure row
/// from the dead adapter knock the healthy new connection back into Reconnecting. The alarm twin
/// released the live alarm feed the same way.
///
///
///
/// Handles are adapter-scoped — MxGateway's are plain ints that collide across sessions — so a
/// dropped completion's handles are released against the adapter that CREATED them, never the
/// current one.
///
///
[Collection("DataConnectionManagerActor")]
public class DataConnectionStaleCompletionTests : TestKit
{
private const string OldEndpoint = "opc.tcp://old-plc:4840";
private const string NewEndpoint = "opc.tcp://new-plc:4840";
private const string Tag = "sensor/temp";
private const string Source = "Tank01";
private readonly ISiteHealthCollector _health = Substitute.For();
private readonly IDataConnectionFactory _factory = Substitute.For();
private readonly DataConnectionOptions _options = new()
{
ReconnectInterval = TimeSpan.FromMilliseconds(100),
TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
WriteTimeout = TimeSpan.FromSeconds(5),
SeedReadMaxAttempts = 1,
SeedReadRetryDelay = TimeSpan.FromMilliseconds(10)
};
private static Dictionary Config(string endpoint) =>
new() { ["Endpoint"] = endpoint };
private IActorRef CreateConnectionActor(string name, IDataConnection adapter) =>
Sys.ActorOf(Props.Create(() =>
new DataConnectionActor(
name, adapter, _options, _health, _factory, "OpcUa",
Config(OldEndpoint), null, 3)), name);
private void ExpectNewAdapter(IDataConnection adapter) =>
_factory.Create("OpcUa", Arg.Is>(d => d["Endpoint"] == NewEndpoint))
.Returns(adapter);
private static IDataConnection HealthyAdapter(string subscriptionId)
{
var adapter = Substitute.For();
adapter.ConnectAsync(Arg.Any>(), Arg.Any())
.Returns(Task.CompletedTask);
adapter.Status.Returns(ConnectionHealth.Connected);
adapter.SubscribeAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(subscriptionId);
adapter.ReadAsync(Arg.Any(), Arg.Any())
.Returns(new ReadResult(false, null, null));
return adapter;
}
private static IDataConnection HealthyAlarmAdapter(string alarmSubscriptionId)
{
var adapter = Substitute.For();
adapter.ConnectAsync(Arg.Any>(), Arg.Any())
.Returns(Task.CompletedTask);
adapter.Status.Returns(ConnectionHealth.Connected);
((IAlarmSubscribableConnection)adapter)
.SubscribeAlarmsAsync(Arg.Any(), Arg.Any(),
Arg.Any(), Arg.Any())
.Returns(Task.FromResult(alarmSubscriptionId));
return adapter;
}
private static NativeAlarmTransition Raise(string sourceRef, string sourceObj) =>
new(sourceRef, sourceObj, "AnalogLimit.Hi", AlarmTransitionKind.Raise,
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 500),
"Process", "hi", "hi", "", "", null, DateTimeOffset.UtcNow, "92", "90");
private int HealthTransitions(string connectionName, ConnectionHealth health) =>
_health.ReceivedCalls().Count(c =>
c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) &&
c.GetArguments() is [string name, ConnectionHealth reported, ..] &&
name == connectionName && reported == health);
private static bool CalledWith(IDataConnection adapter, string method, string firstArg) =>
adapter.ReceivedCalls().Any(c =>
c.GetMethodInfo().Name == method && c.GetArguments()[0] as string == firstArg);
private static bool Called(IDataConnection adapter, string method) =>
adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == method);
// ── Tag subscribe ──
///
/// The headline data-loss case: the stale completion must not park its dead handle under the
/// tag's key, because the live completion for the same tag then reads as a duplicate and has
/// its (working) monitored item released — leaving the tag permanently resolved and silent.
///
///
/// Fails before the fix: the stale completion stored sub-old, and the live
/// handle sub-new was released against the new adapter.
///
///
[Fact]
public void StaleSubscribeCompleted_DoesNotStoreItsHandle_AndLeavesTheLiveMonitoredItemAlone()
{
const string name = "stale-sub-handle";
var gate = new TaskCompletionSource();
var oldAdapter = HealthyAdapter("sub-old");
oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any())
.Returns(_ => gate.Task);
var newAdapter = HealthyAdapter("sub-new");
ExpectNewAdapter(newAdapter);
var actor = CreateConnectionActor(name, oldAdapter);
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
TimeSpan.FromSeconds(5));
actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
// The Instance Actor's retry — the live subscribe, against the new adapter.
probe.Send(actor, new SubscribeTagsRequest("c2", "inst1", name, [Tag], DateTimeOffset.UtcNow));
probe.FishForMessage(
m => m.CorrelationId == "c2" && m.Success, TimeSpan.FromSeconds(5));
// Only now does the generation-N subscribe answer.
gate.SetResult("sub-old");
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"),
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync)));
// The registry still holds the LIVE handle: the teardown releases sub-new, not sub-old.
probe.Send(actor, new UnsubscribeTagsRequest("c3", "inst1", name, DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-new"),
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
Assert.False(CalledWith(newAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"));
}
///
/// The handles a dropped completion did create are released — against the adapter that
/// created them. Releasing them against the CURRENT adapter is not a harmless mismatch:
/// MxGateway subscription handles are plain ints reused across sessions, so the release
/// would land on whatever the new session happens to have under that number.
///
[Fact]
public void StaleSubscribeCompleted_ReleasesItsHandles_AgainstTheAdapterThatCreatedThem()
{
const string name = "stale-sub-release";
var gate = new TaskCompletionSource();
var oldAdapter = HealthyAdapter("sub-old");
oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any())
.Returns(_ => gate.Task);
var newAdapter = HealthyAdapter("sub-new");
ExpectNewAdapter(newAdapter);
var actor = CreateConnectionActor(name, oldAdapter);
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
TimeSpan.FromSeconds(5));
actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
gate.SetResult("sub-old");
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.UnsubscribeAsync), "sub-old"),
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
Assert.False(Called(newAdapter, nameof(IDataConnection.UnsubscribeAsync)));
// The requester is told the subscribe did not take, so its retry timer re-sends: the
// dropped completion registered nothing, so the reconnect re-subscribe cannot cover it.
probe.FishForMessage(
m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5));
}
///
/// The seed values a dropped completion carries are readings from the endpoint the operator
/// has just corrected AWAY from. Applying them re-stamped each one with the CURRENT adapter
/// generation, which is precisely what the generation guard on TagValueReceived
/// exists to stop — the old endpoint's process values presented as fresh.
///
[Fact]
public void StaleSubscribeCompleted_DoesNotDeliverItsSeedValues()
{
const string name = "stale-sub-seed";
var gate = new TaskCompletionSource();
var oldAdapter = HealthyAdapter("sub-old");
oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any())
.Returns(_ => gate.Task);
oldAdapter.ReadAsync(Tag, Arg.Any())
.Returns(new ReadResult(true, new TagValue(999, QualityCode.Good, DateTimeOffset.UtcNow), null));
var newAdapter = HealthyAdapter("sub-new");
ExpectNewAdapter(newAdapter);
var actor = CreateConnectionActor(name, oldAdapter);
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
TimeSpan.FromSeconds(5));
actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
gate.SetResult("sub-old");
var received = probe.ReceiveWhile(TimeSpan.FromSeconds(2), m => m);
// Positive control: the completion really did arrive and was dropped, rather than the
// window simply having been too short to observe anything.
Assert.Contains(received, m => m is SubscribeTagsResponse { Success: false });
Assert.DoesNotContain(received, m => m is TagValueUpdate u && Equals(u.Value, 999));
}
///
/// A connection-level failure row belongs to the adapter that produced it. Acting on a stale
/// one drives the healthy replacement connection into Reconnecting — dropping every tag to
/// bad quality and re-subscribing the whole registry for nothing.
///
[Fact]
public void StaleConnectionLevelFailure_DoesNotDriveTheNewConnectionBackToReconnecting()
{
const string name = "stale-sub-connfail";
var gate = new TaskCompletionSource();
var oldAdapter = HealthyAdapter("sub-old");
oldAdapter.SubscribeAsync(Tag, Arg.Any(), Arg.Any())
.Returns(_ => gate.Task);
var newAdapter = HealthyAdapter("sub-new");
ExpectNewAdapter(newAdapter);
var actor = CreateConnectionActor(name, oldAdapter);
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(oldAdapter, nameof(IDataConnection.SubscribeAsync), Tag),
TimeSpan.FromSeconds(5));
actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 2,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var disconnectsBefore = HealthTransitions(name, ConnectionHealth.Disconnected);
// InvalidOperationException is what IsConnectionLevelFailure classifies as "transport
// down", i.e. the row that asks the state machine to reconnect.
gate.SetException(new InvalidOperationException("adapter not connected"));
probe.FishForMessage(
m => m.CorrelationId == "c1" && !m.Success, TimeSpan.FromSeconds(5));
// Entering Reconnecting would push bad quality to every subscriber — the probe is one.
probe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
Assert.Equal(disconnectsBefore, HealthTransitions(name, ConnectionHealth.Disconnected));
Assert.Equal(2, HealthTransitions(name, ConnectionHealth.Connected));
}
// ── Alarm subscribe ──
///
/// The alarm twin. The stale completion lands first and (before the fix) stored its dead feed
/// id, so the LIVE feed's completion took the duplicate branch and released the feed the new
/// adapter had just established — a source that stays mirrored but never updates again.
///
[Fact]
public void StaleAlarmSubscribeCompleted_DoesNotReleaseTheLiveAlarmFeed()
{
const string name = "stale-alarm-feed";
var oldGate = new TaskCompletionSource();
var newGate = new TaskCompletionSource();
AlarmTransitionCallback? newCallback = null;
var oldAdapter = HealthyAlarmAdapter("alarm-old");
((IAlarmSubscribableConnection)oldAdapter)
.SubscribeAlarmsAsync(Source, Arg.Any(),
Arg.Any(), Arg.Any())
.Returns(_ => oldGate.Task);
var newAdapter = HealthyAlarmAdapter("alarm-new");
((IAlarmSubscribableConnection)newAdapter)
.SubscribeAlarmsAsync(Source, Arg.Any(),
Arg.Do(c => newCallback = c), Arg.Any())
.Returns(_ => newGate.Task);
ExpectNewAdapter(newAdapter);
var actor = CreateConnectionActor(name, oldAdapter);
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= 1,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeAlarmsRequest("c1", "inst1", name, Source, null, DateTimeOffset.UtcNow));
AwaitCondition(() => CalledWith(oldAdapter, "SubscribeAlarmsAsync", Source), TimeSpan.FromSeconds(5));
actor.Tell(new ReconfigureConnectionCommand(name, "OpcUa", Config(NewEndpoint), null, 3));
// The subscriber registry survives the swap, so the reconnect re-subscribe re-opens the
// feed against the new adapter without the NativeAlarmActor doing anything.
AwaitCondition(() => CalledWith(newAdapter, "SubscribeAlarmsAsync", Source),
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
// The dead feed answers first, the live one second — the ordering that made the live
// completion look redundant.
oldGate.SetResult("alarm-old");
AwaitCondition(() => CalledWith(oldAdapter, "UnsubscribeAlarmsAsync", "alarm-old"),
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
newGate.SetResult("alarm-new");
AwaitCondition(() => newCallback != null, TimeSpan.FromSeconds(5));
// Settle (and drain the swap's expected NativeAlarmSourceUnavailable) so the live
// completion has certainly been applied before asserting nothing released its feed.
probe.ReceiveWhile(TimeSpan.FromMilliseconds(300), m => m);
Assert.False(Called(newAdapter, "UnsubscribeAlarmsAsync"));
// And the feed is genuinely live: a transition on it still routes to the subscriber.
newCallback!(Raise("Tank01.Hi", Source));
probe.FishForMessage(
u => u.Transition.SourceObjectReference == Source, TimeSpan.FromSeconds(5));
}
}