feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)

Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a
healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1),
so a connection that lands there sat on Bad quality forever.

- OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected
  endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises
  Disconnected). A level < 200 sustained for the degraded window (default 10 s), or
  the first sample after any (re)connect, triggers a probe of the other endpoint
  (short-lived session: connect, read, dispose). Switch only when it reports a
  Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of
  the previous ServiceLevel switch. Both degraded, other unreachable or level
  unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak.
- The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all
  three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) -
  zero delay, no unstable accounting, ReSubscribeAll on the new endpoint.
- Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds,
  minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults
  (enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or
  migration. Central UI: fields in the Backup card next to Failover Retry Count.
  Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and
  ServiceLevelProbeTimeout (15 s).
- Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited
  promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests,
  and a real in-process OPC UA server pair end to end. All 15 mutants killed.
- Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi

tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/OpcUaServiceLevelTests.cs

@@ -0,0 +1,211 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
+using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
+
+/// <summary>
+/// ScadaBridge#138 at the OPC UA adapter: <see cref="OpcUaDataConnection.ReadServiceLevelAsync"/>
+/// against a substitute client (status / failure mapping) and against real in-process OPC UA
+/// servers (the actual <c>i=2267</c> node), plus the whole failover through the real
+/// <see cref="DataConnectionFactory"/> against a real server pair.
+/// </summary>
+public sealed class OpcUaServiceLevelTests : TestKit
+{
+    private readonly string _clientPki =
+        Path.Combine(Path.GetTempPath(), "sb138-client", Guid.NewGuid().ToString("N"));
+
+    /// <summary>Initializes the test kit.</summary>
+    public OpcUaServiceLevelTests() : base(@"akka.loglevel = DEBUG") { }
+
+    /// <inheritdoc />
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+        try { Directory.Delete(_clientPki, recursive: true); } catch { /* best-effort */ }
+    }
+
+    private OpcUaGlobalOptions ClientOptions() => new()
+    {
+        ApplicationName = "SB138Client",
+        ApplicationUri = "urn:localhost:SB138Client",
+        OwnCertificateStorePath = Path.Combine(_clientPki, "own"),
+        TrustedPeerStorePath = Path.Combine(_clientPki, "trusted"),
+        TrustedIssuerStorePath = Path.Combine(_clientPki, "issuers"),
+        RejectedCertificateStorePath = Path.Combine(_clientPki, "rejected"),
+        AutoCreateApplicationCertificate = true,
+    };
+
+    private static IDictionary<string, string> Flat(string url, OpcUaServiceLevelFailoverConfig? sl = null) =>
+        OpcUaEndpointConfigSerializer.ToFlatDict(new OpcUaEndpointConfig
+        {
+            EndpointUrl = url,
+            SecurityMode = OpcUaSecurityMode.None,
+            ServiceLevelFailover = sl,
+        });
+
+    private static (OpcUaDataConnection Adapter, IOpcUaClient Client) WithSubstituteClient()
+    {
+        var client = Substitute.For<IOpcUaClient>();
+        client.IsConnected.Returns(true);
+        var factory = Substitute.For<IOpcUaClientFactory>();
+        factory.Create().Returns(client);
+        return (new OpcUaDataConnection(factory, NullLogger<OpcUaDataConnection>.Instance), client);
+    }
+
+    // ── Status / failure mapping (substitute client) ──
+
+    /// <summary>The value comes from <c>i=2267</c>; a Good read of a byte is that byte.</summary>
+    [Fact]
+    public async Task Read_GoodByte_IsTheLevel_FromTheServiceLevelNode()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)(byte)90, DateTime.UtcNow, 0u));
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        Assert.Equal((byte)90, await adapter.ReadServiceLevelAsync());
+    }
+
+    /// <summary>A Bad status, or a value that is not a byte-ranged integer, is unknown (null).</summary>
+    [Fact]
+    public async Task Read_BadStatusOrNonByteValue_IsUnknown()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)(byte)250, DateTime.UtcNow, 0x80340000u)); // BadNodeIdUnknown
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)"250", DateTime.UtcNow, 0u));
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)300, DateTime.UtcNow, 0u));
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)240, DateTime.UtcNow, 0u)); // Int32 in range → accepted
+        Assert.Equal((byte)240, await adapter.ReadServiceLevelAsync());
+    }
+
+    /// <summary>
+    /// A failed read is unknown and — unlike <see cref="OpcUaDataConnection.ReadAsync"/> — never
+    /// raises <c>Disconnected</c>: the poll must not drive the reconnect machine.
+    /// </summary>
+    [Fact]
+    public async Task Read_Failure_IsUnknown_AndNeverRaisesDisconnected()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .ThrowsAsync(new InvalidOperationException("session dropped"));
+        var disconnected = 0;
+        adapter.Disconnected += () => disconnected++;
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+        Assert.Equal(0, disconnected);
+
+        // Positive control: the ordinary read path DOES raise it for the same failure.
+        client.ReadValueAsync("ns=2;s=x", Arg.Any<CancellationToken>())
+            .ThrowsAsync(new InvalidOperationException("session dropped"));
+        await Assert.ThrowsAsync<InvalidOperationException>(() => adapter.ReadAsync("ns=2;s=x"));
+        Assert.Equal(1, disconnected);
+    }
+
+    /// <summary>Before connect (no client) the level is unknown, and no read is attempted.</summary>
+    [Fact]
+    public async Task Read_NotConnected_IsUnknown()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+        client.IsConnected.Returns(false);
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+        await client.DidNotReceive().ReadValueAsync(Arg.Any<string>(), Arg.Any<CancellationToken>());
+    }
+
+    // ── Real OPC UA servers ──
+
+    /// <summary>The real adapter reads the real server's <c>Server_ServiceLevel</c> as it changes.</summary>
+    [Fact]
+    public async Task RealServer_ReadServiceLevel_TracksTheServersValue()
+    {
+        await using var server = await SecureOpcUaTestServer.StartAsync();
+        await using var adapter = new OpcUaDataConnection(
+            new RealOpcUaClientFactory(ClientOptions()), NullLogger<OpcUaDataConnection>.Instance);
+        await adapter.ConnectAsync(Flat(server.Url));
+
+        foreach (var level in new byte[] { 250, 240, 90, 50, 1 })
+        {
+            server.SetServiceLevel(level);
+            Assert.Equal(level, await adapter.ReadServiceLevelAsync());
+        }
+    }
+
+    /// <summary>
+    /// End to end on two real servers through the production factory: the connection starts on a
+    /// primary that is the Warm standby (90) while the backup is Active (250), and ends up on the
+    /// backup. Then the mirror case on a fresh connection — today's 250/240 pair — stays on the
+    /// primary.
+    /// </summary>
+    [Fact]
+    public async Task RealServerPair_StandbyPrimary_FailsOverToTheActiveBackup_ActivePair_StaysPut()
+    {
+        await using var a = await SecureOpcUaTestServer.StartAsync();
+        await using var b = await SecureOpcUaTestServer.StartAsync();
+        var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(ClientOptions()));
+        var options = new DataConnectionOptions
+        {
+            ReconnectInterval = TimeSpan.FromMilliseconds(200),
+            ServiceLevelPollInterval = TimeSpan.FromMilliseconds(100),
+            ServiceLevelProbeTimeout = TimeSpan.FromSeconds(20),
+        };
+        var sl = new OpcUaServiceLevelFailoverConfig { DegradedWindowSeconds = 1, MinDwellSeconds = 0 };
+
+        async Task<string> Endpoint(IActorRef actor) =>
+            (await actor.Ask<DataConnectionHealthReport>(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(5)))
+            .ActiveEndpoint;
+
+        // Warm: A is the standby, B the Active.
+        a.SetServiceLevel(90);
+        b.SetServiceLevel(250);
+        var health = Substitute.For<ISiteHealthCollector>();
+        var primary = Flat(a.Url, sl);
+        var warm = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
+            "warm", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
+            primary, Flat(b.Url), 3, null)));
+        await AwaitConditionAsync(async () => await Endpoint(warm) == "Backup", TimeSpan.FromSeconds(30));
+        AwaitCondition(() => health.ReceivedCalls().Any(c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["warm", "Connected to backup"]), TimeSpan.FromSeconds(30));
+        Sys.Stop(warm);
+
+        // Today: A is the Secondary (240), B the Primary (250) — both Active band.
+        a.SetServiceLevel(240);
+        var today = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
+            "today", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
+            primary, Flat(b.Url), 3, null)));
+        AwaitCondition(() => health.ReceivedCalls().Any(c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["today", "Connected to primary"]), TimeSpan.FromSeconds(30));
+        await Task.Delay(TimeSpan.FromSeconds(2.5)); // > window, ~25 polls
+        Assert.Equal("Primary", await Endpoint(today));
+        Assert.DoesNotContain(health.ReceivedCalls(), c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["today", "Connected to backup"]);
+    }
+}

tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/RealOpcUaPromotionRefreshTests.cs

@@ -0,0 +1,112 @@
+using System.Collections.Concurrent;
+using System.Reflection;
+using Opc.Ua;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
+
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
+
+/// <summary>
+/// ScadaBridge#138 / OtOpcUa Warm design §4.5 + §5 "Alarms are compatible": when a Warm standby
+/// is promoted, OtOpcUa runs a ConditionRefresh on every live event item ON ITS OWN — the client
+/// never asked. The existing items therefore receive an unsolicited RefreshStart, the retained
+/// conditions, RefreshEnd, then new events. ScadaBridge must treat that exactly like the refresh it
+/// triggers itself on subscribe: one framed snapshot with its own epoch, and live derivation
+/// before and after it.
+/// </summary>
+public class RealOpcUaPromotionRefreshTests
+{
+    private const string Handle = "h1";
+    private const string Source = "ns=2;s=Tank01";
+
+    private static readonly MethodInfo Handler =
+        typeof(RealOpcUaClient).GetMethod("HandleAlarmEvent", BindingFlags.Instance | BindingFlags.NonPublic)!;
+
+    /// <summary>What <c>CreateAlarmSubscriptionAsync</c> sets up for a live handle: its per-condition state map.</summary>
+    private static void SeedSubscribedHandle(RealOpcUaClient client)
+    {
+        var field = typeof(RealOpcUaClient).GetField("_alarmLastState", BindingFlags.Instance | BindingFlags.NonPublic)!;
+        var map = (ConcurrentDictionary<string, Dictionary<string, (bool Active, bool Acked)>>)field.GetValue(client)!;
+        map[Handle] = new Dictionary<string, (bool Active, bool Acked)>(StringComparer.Ordinal);
+    }
+
+    private static void Boundary(RealOpcUaClient client, NodeId type, List<NativeAlarmTransition> sink) =>
+        Handler.Invoke(client, [Handle, Source,
+            new EventFieldList { EventFields = new Variant[] { Variant.From(type), Variant.Null, Variant.Null, Variant.Null, Variant.Null, Variant.Null } },
+            (Action<NativeAlarmTransition>)sink.Add]);
+
+    private static void Condition(RealOpcUaClient client, string condition, bool active, bool acked, List<NativeAlarmTransition> sink)
+    {
+        var fields = new Variant[]
+        {
+            Variant.From(ObjectTypeIds.AlarmConditionType), // 0 EventType
+            Variant.From(new NodeId("Tank01", 2)),          // 1 SourceNode
+            Variant.From("Tank01"),                         // 2 SourceName
+            Variant.From(DateTime.UtcNow),                  // 3 Time
+            Variant.From(new LocalizedText("msg")),         // 4 Message
+            Variant.From((ushort)500),                      // 5 Severity
+            Variant.From(active),                           // 6 ActiveState/Id
+            Variant.From(acked),                            // 7 AckedState/Id
+            Variant.Null,                                   // 8 ConfirmedState/Id
+            Variant.From(false),                            // 9 SuppressedOrShelved
+            Variant.Null,                                   // 10 ShelvingState
+            Variant.From(condition),                        // 11 ConditionName
+        };
+        Handler.Invoke(client, [Handle, Source, new EventFieldList { EventFields = fields },
+            (Action<NativeAlarmTransition>)sink.Add]);
+    }
+
+    /// <summary>
+    /// Subscribe-time refresh, live traffic, then the UNSOLICITED promotion refresh on the same
+    /// item, then live traffic again. The promotion replay is framed SnapshotStart → Snapshot… →
+    /// SnapshotComplete under a fresh epoch (distinct from the subscribe-time one), and live events
+    /// after it are derived live again — the replay does not leave the item stuck in Snapshot mode.
+    /// </summary>
+    [Fact]
+    public void UnsolicitedPromotionRefresh_IsFramedAsItsOwnSnapshot_AndLiveDerivationResumes()
+    {
+        var client = new RealOpcUaClient();
+        SeedSubscribedHandle(client);
+        var sink = new List<NativeAlarmTransition>();
+
+        // Subscribe-time refresh (the one ScadaBridge triggers): Hi is active.
+        Boundary(client, ObjectTypeIds.RefreshStartEventType, sink);
+        Condition(client, "Hi", active: true, acked: false, sink);
+        Boundary(client, ObjectTypeIds.RefreshEndEventType, sink);
+
+        // Live while the old Active serves: Lo raises.
+        Condition(client, "Lo", active: true, acked: false, sink);
+        var liveBefore = sink[^1];
+
+        // Promotion on the other node: an unsolicited refresh on the existing item.
+        Boundary(client, ObjectTypeIds.RefreshStartEventType, sink);
+        Condition(client, "Hi", active: true, acked: false, sink);
+        Condition(client, "Lo", active: true, acked: false, sink);
+        Boundary(client, ObjectTypeIds.RefreshEndEventType, sink);
+
+        // Live after promotion: Lo is acknowledged.
+        Condition(client, "Lo", active: true, acked: true, sink);
+        var liveAfter = sink[^1];
+
+        Assert.Equal(
+            ["SnapshotStart", "Snapshot", "SnapshotComplete", liveBefore.Kind.ToString(),
+             "SnapshotStart", "Snapshot", "Snapshot", "SnapshotComplete", liveAfter.Kind.ToString()],
+            sink.Select(t => t.Kind.ToString()));
+
+        // Live kinds really are live (not Snapshot) on both sides of the promotion refresh.
+        Assert.Equal(AlarmTransitionKind.Raise, liveBefore.Kind);
+        Assert.Equal(AlarmTransitionKind.Acknowledge, liveAfter.Kind);
+        Assert.Null(liveBefore.SnapshotId);
+        Assert.Null(liveAfter.SnapshotId);
+
+        // The promotion replay has its own epoch, shared by its boundaries and its conditions.
+        var subscribeEpoch = sink[0].SnapshotId;
+        var promotionEpoch = sink[4].SnapshotId;
+        Assert.NotNull(subscribeEpoch);
+        Assert.NotNull(promotionEpoch);
+        Assert.NotEqual(subscribeEpoch, promotionEpoch);
+        Assert.All(sink.Skip(4).Take(4), t => Assert.Equal(promotionEpoch, t.SnapshotId));
+        Assert.All(sink.Skip(4).Take(4), t => Assert.Equal(Source, t.SnapshotSourceReference));
+    }
+}

tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/SecureOpcUaTestServer.cs

@@ -18,11 +18,14 @@ namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
 internal sealed class SecureOpcUaTestServer : IAsyncDisposable
 {
     private readonly ApplicationInstance _app;
+    private readonly StandardServer _server;
     private readonly string _pkiRoot;
 
-    private SecureOpcUaTestServer(ApplicationInstance app, string pkiRoot, string url, X509Certificate2 certificate)
+    private SecureOpcUaTestServer(
+        ApplicationInstance app, StandardServer server, string pkiRoot, string url, X509Certificate2 certificate)
     {
         _app = app;
+        _server = server;
         _pkiRoot = pkiRoot;
         Url = url;
         Certificate = certificate;
@@ -110,16 +113,25 @@ internal sealed class SecureOpcUaTestServer : IAsyncDisposable
             ApplicationType = ApplicationType.Server,
         };
         await app.CheckApplicationInstanceCertificatesAsync(silent: true, lifeTimeInMonths: 12);
-        await app.StartAsync(new TestServer(telemetry, apiKey));
+        var server = new TestServer(telemetry, apiKey);
+        await app.StartAsync(server);
 
         // SDK 2: the loaded application certificate lives in the certificate manager, not on the identifier.
         using var entry = config.CertificateManager.AcquireApplicationCertificateByType(
             ObjectTypeIds.RsaSha256ApplicationCertificateType)
             ?? throw new InvalidOperationException("The test server has no application certificate.");
         var cert = X509CertificateLoader.LoadCertificate(entry.Certificate.RawData);
-        return new SecureOpcUaTestServer(app, pkiRoot, url, cert);
+        return new SecureOpcUaTestServer(app, server, pkiRoot, url, cert);
     }
 
+    /// <summary>
+    /// Sets the server's <c>Server_ServiceLevel</c> (<c>i=2267</c>) — what a redundant server
+    /// publishes for its role (ScadaBridge#138: 250/240 Active, 90/50 Warm standby, 1 booting).
+    /// </summary>
+    /// <param name="level">The ServiceLevel to publish.</param>
+    public void SetServiceLevel(byte level) =>
+        _server.CurrentInstance.ServerObject.ServiceLevel!.Value = level;
+
     /// <inheritdoc />
     public async ValueTask DisposeAsync()
     {