feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)

Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1), so a connection that lands there sat on Bad quality forever. - OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises Disconnected). A level < 200 sustained for the degraded window (default 10 s), or the first sample after any (re)connect, triggers a probe of the other endpoint (short-lived session: connect, read, dispose). Switch only when it reports a Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of the previous ServiceLevel switch. Both degraded, other unreachable or level unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak. - The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) - zero delay, no unstable accounting, ReSubscribeAll on the new endpoint. - Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds, minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults (enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or migration. Central UI: fields in the Backup card next to Failover Retry Count. Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and ServiceLevelProbeTimeout (15 s). - Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests, and a real in-process OPC UA server pair end to end. All 15 mutants killed. - Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 03:06:45 -05:00
parent 16209831ca
commit 712c5117ee
23 changed files with 1760 additions and 36 deletions
@@ -0,0 +1,211 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
+using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
+using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
+
+/// <summary>
+/// ScadaBridge#138 at the OPC UA adapter: <see cref="OpcUaDataConnection.ReadServiceLevelAsync"/>
+/// against a substitute client (status / failure mapping) and against real in-process OPC UA
+/// servers (the actual <c>i=2267</c> node), plus the whole failover through the real
+/// <see cref="DataConnectionFactory"/> against a real server pair.
+/// </summary>
+public sealed class OpcUaServiceLevelTests : TestKit
+{
+    private readonly string _clientPki =
+        Path.Combine(Path.GetTempPath(), "sb138-client", Guid.NewGuid().ToString("N"));
+
+    /// <summary>Initializes the test kit.</summary>
+    public OpcUaServiceLevelTests() : base(@"akka.loglevel = DEBUG") { }
+
+    /// <inheritdoc />
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+        try { Directory.Delete(_clientPki, recursive: true); } catch { /* best-effort */ }
+    }
+
+    private OpcUaGlobalOptions ClientOptions() => new()
+    {
+        ApplicationName = "SB138Client",
+        ApplicationUri = "urn:localhost:SB138Client",
+        OwnCertificateStorePath = Path.Combine(_clientPki, "own"),
+        TrustedPeerStorePath = Path.Combine(_clientPki, "trusted"),
+        TrustedIssuerStorePath = Path.Combine(_clientPki, "issuers"),
+        RejectedCertificateStorePath = Path.Combine(_clientPki, "rejected"),
+        AutoCreateApplicationCertificate = true,
+    };
+
+    private static IDictionary<string, string> Flat(string url, OpcUaServiceLevelFailoverConfig? sl = null) =>
+        OpcUaEndpointConfigSerializer.ToFlatDict(new OpcUaEndpointConfig
+        {
+            EndpointUrl = url,
+            SecurityMode = OpcUaSecurityMode.None,
+            ServiceLevelFailover = sl,
+        });
+
+    private static (OpcUaDataConnection Adapter, IOpcUaClient Client) WithSubstituteClient()
+    {
+        var client = Substitute.For<IOpcUaClient>();
+        client.IsConnected.Returns(true);
+        var factory = Substitute.For<IOpcUaClientFactory>();
+        factory.Create().Returns(client);
+        return (new OpcUaDataConnection(factory, NullLogger<OpcUaDataConnection>.Instance), client);
+    }
+
+    // ── Status / failure mapping (substitute client) ──
+
+    /// <summary>The value comes from <c>i=2267</c>; a Good read of a byte is that byte.</summary>
+    [Fact]
+    public async Task Read_GoodByte_IsTheLevel_FromTheServiceLevelNode()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)(byte)90, DateTime.UtcNow, 0u));
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        Assert.Equal((byte)90, await adapter.ReadServiceLevelAsync());
+    }
+
+    /// <summary>A Bad status, or a value that is not a byte-ranged integer, is unknown (null).</summary>
+    [Fact]
+    public async Task Read_BadStatusOrNonByteValue_IsUnknown()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)(byte)250, DateTime.UtcNow, 0x80340000u)); // BadNodeIdUnknown
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)"250", DateTime.UtcNow, 0u));
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)300, DateTime.UtcNow, 0u));
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .Returns(((object?)240, DateTime.UtcNow, 0u)); // Int32 in range → accepted
+        Assert.Equal((byte)240, await adapter.ReadServiceLevelAsync());
+    }
+
+    /// <summary>
+    /// A failed read is unknown and — unlike <see cref="OpcUaDataConnection.ReadAsync"/> — never
+    /// raises <c>Disconnected</c>: the poll must not drive the reconnect machine.
+    /// </summary>
+    [Fact]
+    public async Task Read_Failure_IsUnknown_AndNeverRaisesDisconnected()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
+            .ThrowsAsync(new InvalidOperationException("session dropped"));
+        var disconnected = 0;
+        adapter.Disconnected += () => disconnected++;
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+        Assert.Equal(0, disconnected);
+
+        // Positive control: the ordinary read path DOES raise it for the same failure.
+        client.ReadValueAsync("ns=2;s=x", Arg.Any<CancellationToken>())
+            .ThrowsAsync(new InvalidOperationException("session dropped"));
+        await Assert.ThrowsAsync<InvalidOperationException>(() => adapter.ReadAsync("ns=2;s=x"));
+        Assert.Equal(1, disconnected);
+    }
+
+    /// <summary>Before connect (no client) the level is unknown, and no read is attempted.</summary>
+    [Fact]
+    public async Task Read_NotConnected_IsUnknown()
+    {
+        var (adapter, client) = WithSubstituteClient();
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+
+        await adapter.ConnectAsync(new Dictionary<string, string>());
+        client.IsConnected.Returns(false);
+        Assert.Null(await adapter.ReadServiceLevelAsync());
+        await client.DidNotReceive().ReadValueAsync(Arg.Any<string>(), Arg.Any<CancellationToken>());
+    }
+
+    // ── Real OPC UA servers ──
+
+    /// <summary>The real adapter reads the real server's <c>Server_ServiceLevel</c> as it changes.</summary>
+    [Fact]
+    public async Task RealServer_ReadServiceLevel_TracksTheServersValue()
+    {
+        await using var server = await SecureOpcUaTestServer.StartAsync();
+        await using var adapter = new OpcUaDataConnection(
+            new RealOpcUaClientFactory(ClientOptions()), NullLogger<OpcUaDataConnection>.Instance);
+        await adapter.ConnectAsync(Flat(server.Url));
+
+        foreach (var level in new byte[] { 250, 240, 90, 50, 1 })
+        {
+            server.SetServiceLevel(level);
+            Assert.Equal(level, await adapter.ReadServiceLevelAsync());
+        }
+    }
+
+    /// <summary>
+    /// End to end on two real servers through the production factory: the connection starts on a
+    /// primary that is the Warm standby (90) while the backup is Active (250), and ends up on the
+    /// backup. Then the mirror case on a fresh connection — today's 250/240 pair — stays on the
+    /// primary.
+    /// </summary>
+    [Fact]
+    public async Task RealServerPair_StandbyPrimary_FailsOverToTheActiveBackup_ActivePair_StaysPut()
+    {
+        await using var a = await SecureOpcUaTestServer.StartAsync();
+        await using var b = await SecureOpcUaTestServer.StartAsync();
+        var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(ClientOptions()));
+        var options = new DataConnectionOptions
+        {
+            ReconnectInterval = TimeSpan.FromMilliseconds(200),
+            ServiceLevelPollInterval = TimeSpan.FromMilliseconds(100),
+            ServiceLevelProbeTimeout = TimeSpan.FromSeconds(20),
+        };
+        var sl = new OpcUaServiceLevelFailoverConfig { DegradedWindowSeconds = 1, MinDwellSeconds = 0 };
+
+        async Task<string> Endpoint(IActorRef actor) =>
+            (await actor.Ask<DataConnectionHealthReport>(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(5)))
+            .ActiveEndpoint;
+
+        // Warm: A is the standby, B the Active.
+        a.SetServiceLevel(90);
+        b.SetServiceLevel(250);
+        var health = Substitute.For<ISiteHealthCollector>();
+        var primary = Flat(a.Url, sl);
+        var warm = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
+            "warm", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
+            primary, Flat(b.Url), 3, null)));
+        await AwaitConditionAsync(async () => await Endpoint(warm) == "Backup", TimeSpan.FromSeconds(30));
+        AwaitCondition(() => health.ReceivedCalls().Any(c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["warm", "Connected to backup"]), TimeSpan.FromSeconds(30));
+        Sys.Stop(warm);
+
+        // Today: A is the Secondary (240), B the Primary (250) — both Active band.
+        a.SetServiceLevel(240);
+        var today = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
+            "today", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
+            primary, Flat(b.Url), 3, null)));
+        AwaitCondition(() => health.ReceivedCalls().Any(c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["today", "Connected to primary"]), TimeSpan.FromSeconds(30));
+        await Task.Delay(TimeSpan.FromSeconds(2.5)); // > window, ~25 polls
+        Assert.Equal("Primary", await Endpoint(today));
+        Assert.DoesNotContain(health.ReceivedCalls(), c =>
+            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
+            && c.GetArguments() is ["today", "Connected to backup"]);
+    }
+}