feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)

Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1), so a connection that lands there sat on Bad quality forever. - OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises Disconnected). A level < 200 sustained for the degraded window (default 10 s), or the first sample after any (re)connect, triggers a probe of the other endpoint (short-lived session: connect, read, dispose). Switch only when it reports a Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of the previous ServiceLevel switch. Both degraded, other unreachable or level unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak. - The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) - zero delay, no unstable accounting, ReSubscribeAll on the new endpoint. - Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds, minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults (enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or migration. Central UI: fields in the Backup card next to Failover Retry Count. Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and ServiceLevelProbeTimeout (15 s). - Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests, and a real in-process OPC UA server pair end to end. All 15 mutants killed. - Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 03:06:45 -05:00
parent 16209831ca
commit 712c5117ee
23 changed files with 1760 additions and 36 deletions
@@ -0,0 +1,110 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
+
+/// <summary>
+/// ScadaBridge#138 / OtOpcUa Warm design §5 "Alarms are compatible", at the actor: the promotion
+/// refresh OtOpcUa sends on its own (an unsolicited RefreshStart / retained conditions / RefreshEnd
+/// on an EXISTING alarm feed, already framed by <c>RealOpcUaClient</c> as SnapshotStart / Snapshot /
+/// SnapshotComplete under a new epoch — see <c>RealOpcUaPromotionRefreshTests</c>) goes through the
+/// existing snapshot handling: it is forwarded in order to the subscriber, and it RECONCILES the
+/// connection's alarm cache — a condition the promoted node no longer retains is gone from what a
+/// later subscriber is replayed.
+/// </summary>
+public class DataConnectionPromotionRefreshTests : TestKit
+{
+    private const string Source = "Tank01";
+
+    private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
+    private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();
+    private readonly DataConnectionOptions _options = new()
+    {
+        ReconnectInterval = TimeSpan.FromMilliseconds(100),
+        TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
+        WriteTimeout = TimeSpan.FromSeconds(5),
+    };
+
+    private static NativeAlarmTransition Boundary(AlarmTransitionKind kind, Guid epoch) => new(
+        "", Source, "", kind,
+        new AlarmConditionState(false, true, null, AlarmShelveState.Unshelved, false, 0),
+        "", "", "", "", "", null, DateTimeOffset.UtcNow, "", "",
+        SnapshotSourceReference: Source, SnapshotId: epoch);
+
+    private static NativeAlarmTransition Condition(
+        string condition, AlarmTransitionKind kind, Guid? epoch, bool active = true, bool acked = false) => new(
+        $"{Source}.{condition}", Source, "AlarmConditionType", kind,
+        new AlarmConditionState(active, acked, null, AlarmShelveState.Unshelved, false, 500),
+        "", "", "msg", "", "", null, DateTimeOffset.UtcNow, "", "",
+        SnapshotSourceReference: epoch is null ? null : Source, SnapshotId: epoch);
+
+    /// <summary>
+    /// Subscribe-time snapshot {Hi, Lo}; then the unsolicited promotion snapshot {Hi} (Lo cleared
+    /// while the node was Standby). The subscriber sees the promotion frame in order, and a second
+    /// subscriber that joins afterwards is replayed Hi only.
+    /// </summary>
+    [Fact]
+    public void UnsolicitedPromotionSnapshot_IsForwardedInOrder_AndReconcilesTheCache()
+    {
+        AlarmTransitionCallback? cb = null;
+        var adapter = Substitute.For<IDataConnection, IAlarmSubscribableConnection>();
+        adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
+        ((IAlarmSubscribableConnection)adapter)
+            .SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(),
+                Arg.Do<AlarmTransitionCallback>(c => cb = c), Arg.Any<CancellationToken>())
+            .Returns(Task.FromResult("alarm-sub-1"));
+        var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
+            "conn", adapter, _options, _health, _factory, "OpcUa")));
+
+        var first = CreateTestProbe();
+        actor.Tell(new SubscribeAlarmsRequest("c1", "inst1", "conn", Source, null, DateTimeOffset.UtcNow), first);
+        first.ExpectMsg<SubscribeAlarmsResponse>(m => m.Success);
+        Assert.NotNull(cb);
+
+        // Subscribe-time refresh on the then-Active node.
+        var subscribeEpoch = Guid.NewGuid();
+        cb!(Boundary(AlarmTransitionKind.SnapshotStart, subscribeEpoch));
+        cb(Condition("Hi", AlarmTransitionKind.Snapshot, subscribeEpoch));
+        cb(Condition("Lo", AlarmTransitionKind.Snapshot, subscribeEpoch));
+        cb(Boundary(AlarmTransitionKind.SnapshotComplete, subscribeEpoch));
+        first.ReceiveN(4, TimeSpan.FromSeconds(3));
+
+        // Promotion on the other node: an unsolicited refresh on this existing feed.
+        var promotionEpoch = Guid.NewGuid();
+        cb(Boundary(AlarmTransitionKind.SnapshotStart, promotionEpoch));
+        cb(Condition("Hi", AlarmTransitionKind.Snapshot, promotionEpoch));
+        cb(Boundary(AlarmTransitionKind.SnapshotComplete, promotionEpoch));
+
+        var frame = first.ReceiveN(3, TimeSpan.FromSeconds(3)).Cast<NativeAlarmTransitionUpdate>().ToList();
+        Assert.Equal(
+            [AlarmTransitionKind.SnapshotStart, AlarmTransitionKind.Snapshot, AlarmTransitionKind.SnapshotComplete],
+            frame.Select(u => u.Transition.Kind));
+        Assert.All(frame, u => Assert.Equal(promotionEpoch, u.Transition.SnapshotId));
+        Assert.Equal($"{Source}.Hi", frame[1].Transition.SourceReference);
+
+        // A live event after the promotion is still routed live.
+        cb(Condition("Hi", AlarmTransitionKind.Acknowledge, epoch: null, acked: true));
+        first.ExpectMsg<NativeAlarmTransitionUpdate>(u => u.Transition.Kind == AlarmTransitionKind.Acknowledge);
+
+        // A subscriber joining now is replayed the reconciled state: Hi only, Lo is gone.
+        var second = CreateTestProbe();
+        actor.Tell(new SubscribeAlarmsRequest("c2", "inst2", "conn", Source, null, DateTimeOffset.UtcNow), second);
+        var replay = second.ReceiveN(4, TimeSpan.FromSeconds(3));
+        var transitions = replay.OfType<NativeAlarmTransitionUpdate>().Select(u => u.Transition).ToList();
+        Assert.Single(replay.OfType<SubscribeAlarmsResponse>(), r => r.Success);
+        Assert.Equal(
+            [AlarmTransitionKind.SnapshotStart, AlarmTransitionKind.Snapshot, AlarmTransitionKind.SnapshotComplete],
+            transitions.Select(t => t.Kind));
+        Assert.Equal([$"{Source}.Hi"], transitions.Where(t => t.Kind == AlarmTransitionKind.Snapshot).Select(t => t.SourceReference));
+        second.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+    }
+}