fix(dcl): advance the session generation on same-adapter reconnect and session-scope MxGateway subscription ids (review 2026-10-21 02/C1, 02/L1)

This commit is contained in:
Joseph Doherty
2026-10-22 06:23:06 -04:00
parent b811a2451b
commit 8c1bd2c3fc
7 changed files with 584 additions and 15 deletions
@@ -0,0 +1,296 @@
using System.Collections.Concurrent;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
/// <summary>
/// The SAME-adapter twin of <see cref="DataConnectionStaleCompletionTests"/> (review 2026-10-21
/// 02/C1). A plain drop + reconnect keeps the adapter object — only the session underneath it
/// changes — so before the fix the adapter generation did not advance and an initial-subscribe
/// completion issued against the DEAD session passed every stale guard and was applied against the
/// new one:
/// <list type="bullet">
/// <item>(a) landing before the reconnect re-subscribe row for its tag, it parked the dead handle
/// under the tag's key, so the LIVE row then looked redundant and its handle was released — the
/// tag reads resolved and never delivers again;</item>
/// <item>(b) landing after it, its dead handle was released against the same adapter — and an
/// MxGateway handle is a session-scoped int the new session may already have handed to a
/// different tag;</item>
/// <item>a connection-level failure row in it knocked the freshly reconnected link back into
/// Reconnecting.</item>
/// </list>
/// </summary>
[Collection("DataConnectionManagerActor")]
public class DataConnectionSameAdapterReconnectTests : TestKit
{
private const string Tag = "sensor/temp";
private const string OtherTag = "sensor/pressure";
private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();
private readonly DataConnectionOptions _options = new()
{
ReconnectInterval = TimeSpan.FromMilliseconds(100),
TagResolutionRetryInterval = TimeSpan.FromSeconds(30),
WriteTimeout = TimeSpan.FromSeconds(5),
SeedReadMaxAttempts = 1,
SeedReadRetryDelay = TimeSpan.FromMilliseconds(10),
SubscribeBatchDelay = TimeSpan.Zero
};
private IActorRef CreateConnectionActor(string name, IDataConnection adapter)
{
var config = new Dictionary<string, string> { ["Endpoint"] = "opc.tcp://plc:4840" };
return Sys.ActorOf(Props.Create(() =>
new DataConnectionActor(
name, adapter, _options, _health, _factory, "OpcUa", config, null, 3)), name);
}
private int HealthTransitions(string connectionName, ConnectionHealth health) =>
_health.ReceivedCalls().Count(c =>
c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) &&
c.GetArguments() is [string name, ConnectionHealth reported, ..] &&
name == connectionName && reported == health);
private void AwaitConnectedCount(string name, int count) =>
AwaitCondition(() => HealthTransitions(name, ConnectionHealth.Connected) >= count,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
private void AwaitResolved(IActorRef actor, int resolved) =>
AwaitCondition(() => actor.Ask<DataConnectionHealthReport>(
new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(1)).Result.ResolvedTags == resolved,
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(20));
/// <summary>Drops the session and waits for the same adapter to reconnect.</summary>
private void DropAndReconnect(string name, GatedAdapter adapter, int connectedCountAfter)
{
adapter.RaiseDisconnected();
AwaitConnectedCount(name, connectedCountAfter);
}
/// <summary>
/// (a) The stale completion lands BEFORE the re-subscribe row for its tag. The live handle
/// must never be released, and the registry must still hold it afterwards.
/// </summary>
[Fact]
public void StaleCompletion_BeforeResubscribeRow_DoesNotReleaseTheLiveHandle()
{
const string name = "same-adapter-a";
var adapter = new GatedAdapter();
var actor = CreateConnectionActor(name, adapter);
AwaitConnectedCount(name, 1);
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
var initial = adapter.AwaitCall(0);
DropAndReconnect(name, adapter, 2);
var resubscribe = adapter.AwaitCall(1);
var disconnectsAfterReconnect = HealthTransitions(name, ConnectionHealth.Disconnected);
// Dead-session completion first …
initial.Complete(Tag, "dead-1");
probe.FishForMessage<SubscribeTagsResponse>(m => m.CorrelationId == "c1", TimeSpan.FromSeconds(5));
// … then the live re-subscribe row.
resubscribe.Complete(Tag, "live-2");
// Let the live completion (and any release it would trigger) be applied.
Thread.Sleep(500);
Assert.DoesNotContain("live-2", adapter.ReleasedIds);
Assert.Equal(disconnectsAfterReconnect, HealthTransitions(name, ConnectionHealth.Disconnected));
// The registry holds the LIVE handle: tearing the instance down releases live-2.
probe.Send(actor, new UnsubscribeTagsRequest("c2", "inst1", name, DateTimeOffset.UtcNow));
AwaitCondition(() => adapter.ReleasedIds.Contains("live-2"), TimeSpan.FromSeconds(5));
}
/// <summary>
/// (b) The stale completion lands AFTER the re-subscribe row. Its handle belongs to the dead
/// session and must not be released against the same adapter: the new session may have reused
/// that handle for a different, live tag (MxGateway handles are session-scoped ints).
/// </summary>
[Fact]
public void StaleCompletion_AfterResubscribeRow_NeverReleasesAHandleTheNewSessionReused()
{
const string name = "same-adapter-b";
var adapter = new GatedAdapter();
var actor = CreateConnectionActor(name, adapter);
AwaitConnectedCount(name, 1);
var probe = CreateTestProbe();
// inst2 owns OtherTag, fully established in the first session.
probe.Send(actor, new SubscribeTagsRequest("o1", "inst2", name, [OtherTag], DateTimeOffset.UtcNow));
adapter.AwaitCall(0).Complete(OtherTag, "h0");
probe.FishForMessage<SubscribeTagsResponse>(m => m.CorrelationId == "o1" && m.Success, TimeSpan.FromSeconds(5));
// inst1's subscribe is still in flight when the session drops.
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
var initial = adapter.AwaitCall(1);
DropAndReconnect(name, adapter, 2);
var resubscribe = adapter.AwaitCall(2);
var disconnectsAfterReconnect = HealthTransitions(name, ConnectionHealth.Disconnected);
// The new session hands OtherTag the handle number "h1" …
resubscribe.CompleteMany((OtherTag, "h1"), (Tag, "h2"));
AwaitResolved(actor, 2);
// … which is the very number the dead session gave inst1's in-flight subscribe.
initial.Complete(Tag, "h1");
probe.FishForMessage<SubscribeTagsResponse>(m => m.CorrelationId == "c1", TimeSpan.FromSeconds(5));
actor.Tell(new DataConnectionActor.GetHealthReport(), probe.Ref);
probe.FishForMessage<DataConnectionHealthReport>(_ => true, TimeSpan.FromSeconds(5));
Assert.DoesNotContain("h1", adapter.ReleasedIds);
Assert.DoesNotContain("h2", adapter.ReleasedIds);
Assert.Equal(disconnectsAfterReconnect, HealthTransitions(name, ConnectionHealth.Disconnected));
}
/// <summary>
/// A connection-level failure row in a dead-session completion must not knock the freshly
/// reconnected link back into Reconnecting.
/// </summary>
[Fact]
public void StaleConnectionLevelFailure_DoesNotDriveTheReconnectedLinkBackToReconnecting()
{
const string name = "same-adapter-connfail";
var adapter = new GatedAdapter();
var actor = CreateConnectionActor(name, adapter);
AwaitConnectedCount(name, 1);
var probe = CreateTestProbe();
probe.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
var initial = adapter.AwaitCall(0);
DropAndReconnect(name, adapter, 2);
adapter.AwaitCall(1).Complete(Tag, "live-2");
var disconnectsAfterReconnect = HealthTransitions(name, ConnectionHealth.Disconnected);
// InvalidOperationException is classified as connection-level (transport down).
initial.Fail(new InvalidOperationException("adapter not connected"));
probe.FishForMessage<SubscribeTagsResponse>(m => m.CorrelationId == "c1", TimeSpan.FromSeconds(5));
actor.Tell(new DataConnectionActor.GetHealthReport(), probe.Ref);
probe.FishForMessage<DataConnectionHealthReport>(_ => true, TimeSpan.FromSeconds(5));
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
Assert.Equal(disconnectsAfterReconnect, HealthTransitions(name, ConnectionHealth.Disconnected));
Assert.Equal(2, HealthTransitions(name, ConnectionHealth.Connected));
Assert.DoesNotContain("live-2", adapter.ReleasedIds);
}
/// <summary>One gated <c>SubscribeBatchAsync</c> call.</summary>
internal sealed class PendingBatch
{
private readonly TaskCompletionSource<IReadOnlyList<TagSubscribeResult>> _tcs =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public PendingBatch(IReadOnlyList<string> tags) => Tags = tags;
public IReadOnlyList<string> Tags { get; }
public Task<IReadOnlyList<TagSubscribeResult>> Task => _tcs.Task;
public void Complete(string tag, string id) => CompleteMany((tag, id));
public void CompleteMany(params (string Tag, string Id)[] rows) =>
_tcs.SetResult(rows.Select(r => new TagSubscribeResult(r.Tag, true, r.Id, null)).ToList());
public void Fail(Exception ex) => _tcs.SetException(ex);
}
/// <summary>
/// Batch-capable adapter whose every subscribe batch is held until the test completes it, so
/// the arrival order of dead-session and live-session completions is fully controlled. The
/// same instance is reused across reconnects, exactly like the shipped adapters.
/// </summary>
internal sealed class GatedAdapter : IDataConnection, IBatchSubscribableConnection
{
private readonly ConcurrentDictionary<int, PendingBatch> _calls = new();
private int _callCount;
public readonly ConcurrentQueue<string> ReleasedIds = new();
public ConnectionHealth Status { get; private set; } = ConnectionHealth.Disconnected;
public event Action? Disconnected;
public void RaiseDisconnected()
{
Status = ConnectionHealth.Disconnected;
Disconnected?.Invoke();
}
public PendingBatch AwaitCall(int index)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (DateTime.UtcNow < deadline)
{
if (_calls.TryGetValue(index, out var call))
return call;
Thread.Sleep(10);
}
throw new TimeoutException($"Subscribe batch #{index} was never issued.");
}
public Task ConnectAsync(IDictionary<string, string> connectionDetails, CancellationToken cancellationToken = default)
{
Status = ConnectionHealth.Connected;
return System.Threading.Tasks.Task.CompletedTask;
}
public Task DisconnectAsync(CancellationToken cancellationToken = default) =>
System.Threading.Tasks.Task.CompletedTask;
public Task<IReadOnlyList<TagSubscribeResult>> SubscribeBatchAsync(
IReadOnlyList<string> tagPaths, SubscriptionCallback callback, CancellationToken cancellationToken = default)
{
var call = new PendingBatch(tagPaths.ToList());
_calls[Interlocked.Increment(ref _callCount) - 1] = call;
return call.Task;
}
public Task UnsubscribeBatchAsync(IReadOnlyList<string> subscriptionIds, CancellationToken cancellationToken = default)
{
foreach (var id in subscriptionIds)
ReleasedIds.Enqueue(id);
return System.Threading.Tasks.Task.CompletedTask;
}
public Task<string> SubscribeAsync(string tagPath, SubscriptionCallback callback, CancellationToken cancellationToken = default) =>
throw new NotSupportedException("batch-capable adapter");
public Task UnsubscribeAsync(string subscriptionId, CancellationToken cancellationToken = default)
{
ReleasedIds.Enqueue(subscriptionId);
return System.Threading.Tasks.Task.CompletedTask;
}
public Task<ReadResult> ReadAsync(string tagPath, CancellationToken cancellationToken = default) =>
System.Threading.Tasks.Task.FromResult(new ReadResult(false, null, "no value"));
public Task<IReadOnlyDictionary<string, ReadResult>> ReadBatchAsync(
IEnumerable<string> tagPaths, CancellationToken cancellationToken = default) =>
System.Threading.Tasks.Task.FromResult<IReadOnlyDictionary<string, ReadResult>>(
new Dictionary<string, ReadResult>());
public Task<WriteResult> WriteAsync(string tagPath, object? value, CancellationToken cancellationToken = default) =>
System.Threading.Tasks.Task.FromResult(new WriteResult(true, null));
public Task<IReadOnlyDictionary<string, WriteResult>> WriteBatchAsync(
IDictionary<string, object?> values, CancellationToken cancellationToken = default) =>
System.Threading.Tasks.Task.FromResult<IReadOnlyDictionary<string, WriteResult>>(
values.ToDictionary(kv => kv.Key, _ => new WriteResult(true, null)));
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}
@@ -24,6 +24,13 @@ public sealed class FakeMxGatewayClient : IMxGatewayClient, IMxGatewayClientFact
public Func<IReadOnlyList<string>, IReadOnlyList<MxReadOutcome>>? ReadHandler;
public Func<IReadOnlyList<(string TagPath, object? Value)>, IReadOnlyList<MxWriteOutcome>>? WriteHandler;
public Func<string?, (IReadOnlyList<MxBrowseChild>, bool)>? BrowseHandler;
/// <summary>
/// When true, every <see cref="ConnectAsync"/> restarts handle numbering at 1 — the
/// gateway's per-session item handles, which collide across sessions.
/// </summary>
public bool RestartHandlesOnConnect;
/// <summary>The transition callback of the most recently opened alarm stream.</summary>
public Action<ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms.NativeAlarmTransition>? AlarmCallback;
private int _nextHandle;
/// <inheritdoc />
@@ -33,6 +40,8 @@ public sealed class FakeMxGatewayClient : IMxGatewayClient, IMxGatewayClientFact
public Task ConnectAsync(MxGatewayConnectionOptions o, CancellationToken ct = default)
{
ConnectedWith = o;
if (RestartHandlesOnConnect)
_nextHandle = 0;
return Task.CompletedTask;
}
@@ -115,6 +124,7 @@ public sealed class FakeMxGatewayClient : IMxGatewayClient, IMxGatewayClientFact
// recorded so the union-filter (longest-common-prefix) behaviour is testable.
lock (AlarmStreamPrefixes)
AlarmStreamPrefixes.Add(alarmFilterPrefix);
AlarmCallback = onTransition;
return Task.CompletedTask;
}
@@ -122,7 +122,8 @@ public class MxGatewayDataConnectionTests
fake.OnUpdate!(new MxValueUpdate("T", 1, QualityCode.Good, DateTimeOffset.UtcNow));
Assert.Equal(0, hits);
Assert.Contains(subId, fake.Unsubscribed);
// Ids are session-scoped ("{session}:{handle}"); the gateway receives the raw handle.
Assert.Equal(subId[(subId.IndexOf(':') + 1)..], Assert.Single(fake.Unsubscribed));
}
// ── read / write + error classification ──
@@ -0,0 +1,136 @@
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
/// <summary>
/// The adapter half of review 2026-10-21 02/C1 + 02/L1. The adapter instance is reused across
/// reconnects, but every reconnect opens a NEW gateway session whose item handles are plain ints
/// that restart (collide) per session. A subscription id handed out by a dead session must
/// therefore never reach the gateway again — it would unadvise whatever the live session gave that
/// number — and the routing maps must not keep dead-session entries alive.
/// </summary>
[Collection("DataConnectionManagerActor")]
public class MxGatewaySessionScopedHandleTests
{
private static Dictionary<string, string> Details() => new()
{
["Endpoint"] = "http://gw:5000",
["ApiKey"] = "key",
["ReadTimeoutMs"] = "2000",
};
private static MxGatewayDataConnection NewAdapter(FakeMxGatewayClient fake) =>
new(fake, NullLogger<MxGatewayDataConnection>.Instance);
/// <summary>
/// A batch release of a dead-session id never reaches the gateway, even when the new session
/// has handed the same handle number to a live tag — and that live tag keeps routing.
/// </summary>
[Fact]
public async Task DeadSessionId_IsNeverReleasedAtTheGateway_EvenWhenTheHandleWasReused()
{
var fake = new FakeMxGatewayClient { RestartHandlesOnConnect = true };
var adapter = NewAdapter(fake);
await adapter.ConnectAsync(Details());
var dead = (await adapter.SubscribeBatchAsync(["Old.Tag"], (_, _) => { }))[0].SubscriptionId!;
await adapter.ConnectAsync(Details()); // reconnect: new session, handles restart at 1
var liveHits = 0;
var live = (await adapter.SubscribeBatchAsync(["Live.Tag"], (_, _) => liveHits++))[0].SubscriptionId!;
await adapter.UnsubscribeBatchAsync([dead]);
await adapter.UnsubscribeAsync(dead);
Assert.Empty(fake.Unsubscribed);
Assert.NotEqual(dead, live);
await WaitUntil(() => fake.OnUpdate is not null);
fake.OnUpdate!(new MxValueUpdate("Live.Tag", 2, QualityCode.Good, DateTimeOffset.UtcNow));
Assert.True(liveHits >= 1);
// A current-session id still releases normally.
await adapter.UnsubscribeBatchAsync([live]);
Assert.Single(fake.Unsubscribed);
}
/// <summary>
/// A reconnect drops the previous session's routing entries (L1): an update for a tag the
/// new session has not (re-)subscribed is not routed to the dead session's callback.
/// </summary>
[Fact]
public async Task Reconnect_ClearsTheDeadSessionsRoutingEntries()
{
var fake = new FakeMxGatewayClient();
var adapter = NewAdapter(fake);
await adapter.ConnectAsync(Details());
var hits = 0;
await adapter.SubscribeBatchAsync(["A.Tag"], (_, _) => hits++);
await adapter.ConnectAsync(Details());
await WaitUntil(() => fake.OnUpdate is not null);
fake.OnUpdate!(new MxValueUpdate("A.Tag", 1, QualityCode.Good, DateTimeOffset.UtcNow));
Assert.Equal(0, hits);
}
/// <summary>
/// A disconnect drops the routing entries too, so a later update is not delivered to a
/// subscription that no longer exists at the gateway.
/// </summary>
[Fact]
public async Task Disconnect_ClearsRoutingEntries()
{
var fake = new FakeMxGatewayClient();
var adapter = NewAdapter(fake);
await adapter.ConnectAsync(Details());
var hits = 0;
await adapter.SubscribeBatchAsync(["A.Tag"], (_, _) => hits++);
await WaitUntil(() => fake.OnUpdate is not null);
await adapter.DisconnectAsync();
fake.OnUpdate!(new MxValueUpdate("A.Tag", 1, QualityCode.Good, DateTimeOffset.UtcNow));
Assert.Equal(0, hits);
}
/// <summary>
/// Releasing an alarm subscription id the adapter does not know (a dead session's) must not
/// decrement the live feed's reference count — at zero the shared stream is torn down while
/// live subscribers still depend on it.
/// </summary>
[Fact]
public async Task UnknownAlarmSubscriptionId_DoesNotTearDownTheLiveStream()
{
var fake = new FakeMxGatewayClient();
var adapter = NewAdapter(fake);
await adapter.ConnectAsync(Details());
var delivered = 0;
await adapter.SubscribeAlarmsAsync("Tank01", null, t =>
{
if (t.Kind == AlarmTransitionKind.SnapshotComplete)
Interlocked.Increment(ref delivered);
});
await WaitUntil(() => fake.AlarmCallback is not null);
await adapter.UnsubscribeAlarmsAsync(Guid.NewGuid().ToString());
fake.AlarmCallback!(MxGatewayAlarmMapper.SnapshotComplete());
Assert.Equal(1, Volatile.Read(ref delivered));
}
private static async Task WaitUntil(Func<bool> condition, int timeoutMs = 30_000)
{
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
while (!condition())
{
if (DateTime.UtcNow > deadline)
throw new TimeoutException("Condition not met in time.");
await Task.Delay(10);
}
}
}