fix(dcl): P2-5 seed probe-resolved tags; P2-6 close the Connecting/Reconnecting AdapterDisconnected gap
P2-5 — the tag-resolution PROBE was the only subscribe path that did not seed. HandleSubscribe carries seeds back on SubscribeCompleted and ReSubscribeAll re-seeds after a reconnect, because a fresh advise only yields a value on the source's next CHANGE. HandleRetryTagResolution did neither, so a tag that resolved LATE (device still booting, item created after deployment) and then never changed had no value path at all. OPC UA masks it with the initial data-change notification on subscription creation; an MxGateway advise can legitimately return VT_EMPTY, so a static tag stayed Uncertain for the life of the connection. New SeedNewlyResolvedTags reads the round's newly resolved tags and delivers them through the normal generation-guarded TagValueReceived path — directly rather than on the completion message (the probe only covers tags already registered in _instancesByTag, so the initial subscribe's ordering hazard does not apply), and as its own fire-and-forget task so the seed reads never delay the BatchSubscribeCompleted that re-arms the retry timer. P2-6 — only the Connected state handled AdapterDisconnected; in Connecting and Reconnecting it fell through to Unhandled. A connect that succeeds and drops again before its piped ConnectResult is applied therefore delivered the disconnect FIRST, where it was discarded, and ConnectResult(true) then declared the connection up on a dead adapter. An adapter raises Disconnected at most once per successful connect (OpcUaDataConnection's once-only _disconnectFired, reset only by the next successful ConnectAsync), so a connection with no subscriptions — nothing to fail at connection level and drive the reconnect machine — sat Connected on a dead adapter until the process restarted. Both states now record the disconnect in _disconnectDuringConnectAttempt (cleared at the top of HandleAttemptConnect, so it only ever refers to the attempt in flight) and both result handlers demote such a "success" to the ordinary connect-FAILURE tail — counting toward the failover budget and re-arming the retry timer. A flag rather than Stash: the stash has no selective drop, so a stashed disconnect would be replayed after the next successful connect and cost a spurious reconnect cycle. AdapterDisconnected is now stamped with the adapter generation (hooked via a per-adapter closure instead of a method group; SwapAdapter hooks AFTER the bump), and IsStaleDisconnect drops a notification from a swapped-out adapter in all three states — the disconnect twin of IsStaleConnectResult. SwapAdapter's unhook cannot stop an event already executing on a background thread, so without the fence an endpoint the operator has just reconfigured AWAY from could knock the healthy replacement into a reconnect. Tests: DataConnectionProbeSeedTests + DataConnectionDisconnectRaceTests (4 new). The race is reproduced by ORDERING, not timing — the connect is held on a gate, Disconnected is raised while the actor sits idle, and a GetHealthReport round trip is the mailbox barrier; no sleeps. All 4 were proved to fail against the pre-fix behaviour via a targeted revert of the four behaviour sites (the stale-generation test cannot be proved by a plain revert — the record carried no generation before this change, so the test would not compile; noted in its doc comment). Full DCL suite 346/346 green.
This commit is contained in:
Joseph Doherty
+106
@@ -0,0 +1,106 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Arch-review 2026-08-19 <b>P2-5</b> — the tag-resolution PROBE path did not seed the tags it
|
||||
/// resolved.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>The gap this pins.</b> Both other subscribe paths read each newly advised tag's current
|
||||
/// value and deliver it (<c>HandleSubscribe</c> carries the seeds back on <c>SubscribeCompleted</c>;
|
||||
/// <c>ReSubscribeAll</c> re-seeds after a reconnect), because a fresh advise only produces a value
|
||||
/// on the source's next CHANGE. <c>HandleRetryTagResolution</c> did not. OPC UA masks that with an
|
||||
/// initial data-change notification on subscription creation, but an MxGateway advise can
|
||||
/// legitimately return <c>VT_EMPTY</c> — so a STATIC tag that resolved LATE (device still booting,
|
||||
/// item created after deployment) had no value path at all and its attribute stayed Uncertain for
|
||||
/// the life of the connection.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Collection("DataConnectionManagerActor")]
|
||||
public class DataConnectionProbeSeedTests : TestKit
|
||||
{
|
||||
private const string Tag = "plant/static/setpoint";
|
||||
|
||||
private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
|
||||
private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();
|
||||
|
||||
private readonly DataConnectionOptions _options = new()
|
||||
{
|
||||
ReconnectInterval = TimeSpan.FromMilliseconds(100),
|
||||
TagResolutionRetryInterval = TimeSpan.FromMilliseconds(150),
|
||||
WriteTimeout = TimeSpan.FromSeconds(5),
|
||||
SeedReadMaxAttempts = 1,
|
||||
SeedReadRetryDelay = TimeSpan.FromMilliseconds(10)
|
||||
};
|
||||
|
||||
private IActorRef CreateConnectionActor(string name, IDataConnection adapter) =>
|
||||
Sys.ActorOf(Props.Create(() =>
|
||||
new DataConnectionActor(name, adapter, _options, _health, _factory, "OpcUa")), name);
|
||||
|
||||
private bool HasEnteredConnectedState(string connectionName) =>
|
||||
_health.ReceivedCalls().Any(c =>
|
||||
c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionHealth) &&
|
||||
c.GetArguments() is [string name, ConnectionHealth.Connected, ..] &&
|
||||
name == connectionName);
|
||||
|
||||
/// <summary>
|
||||
/// A tag that fails to resolve on the initial subscribe, resolves on the first probe round,
|
||||
/// and then never changes again must still reach its subscriber with a Good value.
|
||||
///
|
||||
/// <para>
|
||||
/// The adapter here models exactly the MxGateway shape the finding calls out: the advise
|
||||
/// succeeds on the retry but the subscription callback is NEVER invoked (a <c>VT_EMPTY</c>
|
||||
/// advise on a static item), so the seed read is the tag's only possible value source.
|
||||
/// </para>
|
||||
///
|
||||
/// <para><b>Fails before the fix:</b> the probe resolved the tag and stopped there — the
|
||||
/// subscriber received only the Bad-quality resolution-failure push from the initial
|
||||
/// subscribe, and this fish for a Good value timed out.</para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ProbeResolvedTag_IsSeeded_SoAStaticTagLeavesUncertain()
|
||||
{
|
||||
const string name = "probe-seed";
|
||||
var subscribeCalls = 0;
|
||||
|
||||
var adapter = Substitute.For<IDataConnection>();
|
||||
adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask);
|
||||
adapter.Status.Returns(ConnectionHealth.Connected);
|
||||
adapter.SubscribeAsync(Tag, Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ => Interlocked.Increment(ref subscribeCalls) == 1
|
||||
// Initial subscribe: the device is still booting — a genuine (non
|
||||
// connection-level) resolution failure, so the tag lands in _unresolvedTags.
|
||||
? Task.FromException<string>(new KeyNotFoundException("node not found yet"))
|
||||
// Probe round: the advise succeeds. The callback is deliberately never invoked.
|
||||
: Task.FromResult("sub-1"));
|
||||
adapter.ReadAsync(Tag, Arg.Any<CancellationToken>())
|
||||
.Returns(new ReadResult(true, new TagValue(42, QualityCode.Good, DateTimeOffset.UtcNow), null));
|
||||
|
||||
var actor = CreateConnectionActor(name, adapter);
|
||||
AwaitCondition(() => HasEnteredConnectedState(name),
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
|
||||
|
||||
var subscriber = CreateTestProbe();
|
||||
subscriber.Send(actor, new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
|
||||
|
||||
// The initial subscribe fails: Bad-quality push, then the ack.
|
||||
subscriber.FishForMessage<SubscribeTagsResponse>(_ => true, TimeSpan.FromSeconds(5));
|
||||
|
||||
// The probe round resolves the tag; the seed read is the only thing that can now
|
||||
// deliver a value, because the advise callback never fires for this static item.
|
||||
var seeded = subscriber.FishForMessage<TagValueUpdate>(
|
||||
m => m.TagPath == Tag && m.Quality == QualityCode.Good, TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.Equal(42, seeded.Value);
|
||||
Assert.True(Volatile.Read(ref subscribeCalls) >= 2, "the probe round must have re-subscribed the tag");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user