feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)

Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1), so a connection that lands there sat on Bad quality forever. - OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises Disconnected). A level < 200 sustained for the degraded window (default 10 s), or the first sample after any (re)connect, triggers a probe of the other endpoint (short-lived session: connect, read, dispose). Switch only when it reports a Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of the previous ServiceLevel switch. Both degraded, other unreachable or level unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak. - The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) - zero delay, no unstable accounting, ReSubscribeAll on the new endpoint. - Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds, minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults (enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or migration. Central UI: fields in the Backup card next to Failover Retry Count. Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and ServiceLevelProbeTimeout (15 s). - Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests, and a real in-process OPC UA server pair end to end. All 15 mutants killed. - Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 03:06:45 -05:00
parent 16209831ca
commit 712c5117ee
23 changed files with 1760 additions and 36 deletions
@@ -0,0 +1,119 @@
+using System.Collections.Concurrent;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
+
+/// <summary>
+/// The two OPC UA servers of a redundant pair as the ScadaBridge#138 tests see them: a
+/// ServiceLevel per endpoint (changeable at any time, null = unreadable), whether each endpoint
+/// accepts a session, and a record of what the actor did to them.
+/// </summary>
+internal sealed class ServiceLevelRig
+{
+    private readonly ConcurrentDictionary<string, Func<byte?>> _levels = new();
+    private readonly ConcurrentDictionary<string, bool> _unreachable = new();
+
+    /// <summary>Every endpoint a session was opened on, in order — live sessions and probes alike.</summary>
+    public ConcurrentQueue<string> Connects { get; } = new();
+
+    /// <summary>Every tag subscribed, as "endpoint|tag" — only a live session receives these.</summary>
+    public ConcurrentQueue<string> Subscribes { get; } = new();
+
+    /// <summary>Number of ServiceLevel reads, per endpoint.</summary>
+    public ConcurrentDictionary<string, int> Reads { get; } = new();
+
+    /// <summary>Sets a fixed ServiceLevel for <paramref name="endpoint"/>.</summary>
+    public void SetLevel(string endpoint, byte? level) => _levels[endpoint] = () => level;
+
+    /// <summary>Sets a ServiceLevel source evaluated on every read of <paramref name="endpoint"/>.</summary>
+    public void SetLevel(string endpoint, Func<byte?> level) => _levels[endpoint] = level;
+
+    /// <summary>Makes <paramref name="endpoint"/> refuse (or accept again) new sessions.</summary>
+    public void SetReachable(string endpoint, bool reachable) => _unreachable[endpoint] = !reachable;
+
+    public bool IsReachable(string endpoint) => !_unreachable.GetValueOrDefault(endpoint);
+
+    public byte? Read(string endpoint)
+    {
+        Reads.AddOrUpdate(endpoint, 1, (_, n) => n + 1);
+        return _levels.TryGetValue(endpoint, out var level) ? level() : null;
+    }
+
+    public int ConnectCount(string endpoint) => Connects.Count(e => e == endpoint);
+
+    public int ReadCount(string endpoint) => Reads.GetValueOrDefault(endpoint);
+}
+
+/// <summary>
+/// An <see cref="IDataConnection"/> with the <see cref="IServiceLevelReadable"/> capability, bound to
+/// one endpoint of a <see cref="ServiceLevelRig"/>. Subscriptions and reads always succeed.
+/// </summary>
+internal sealed class FakeServiceLevelConnection : IDataConnection, IServiceLevelReadable
+{
+    private readonly ServiceLevelRig _rig;
+    private volatile bool _connected;
+
+    public FakeServiceLevelConnection(ServiceLevelRig rig, string endpoint)
+    {
+        _rig = rig;
+        Endpoint = endpoint;
+    }
+
+    public string Endpoint { get; }
+
+    public ConnectionHealth Status => _connected ? ConnectionHealth.Connected : ConnectionHealth.Disconnected;
+
+    public event Action? Disconnected;
+
+    public Task ConnectAsync(IDictionary<string, string> connectionDetails, CancellationToken cancellationToken = default)
+    {
+        _rig.Connects.Enqueue(Endpoint);
+        if (!_rig.IsReachable(Endpoint))
+            return Task.FromException(new InvalidOperationException($"{Endpoint} refused the session"));
+        _connected = true;
+        return Task.CompletedTask;
+    }
+
+    public Task DisconnectAsync(CancellationToken cancellationToken = default)
+    {
+        _connected = false;
+        return Task.CompletedTask;
+    }
+
+    public Task<byte?> ReadServiceLevelAsync(CancellationToken cancellationToken = default) =>
+        Task.FromResult(_connected ? _rig.Read(Endpoint) : null);
+
+    public Task<string> SubscribeAsync(string tagPath, SubscriptionCallback callback, CancellationToken cancellationToken = default)
+    {
+        _rig.Subscribes.Enqueue($"{Endpoint}|{tagPath}");
+        return Task.FromResult($"{Endpoint}:{tagPath}");
+    }
+
+    public Task UnsubscribeAsync(string subscriptionId, CancellationToken cancellationToken = default) => Task.CompletedTask;
+
+    public Task<ReadResult> ReadAsync(string tagPath, CancellationToken cancellationToken = default) =>
+        Task.FromResult(new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null));
+
+    public Task<IReadOnlyDictionary<string, ReadResult>> ReadBatchAsync(IEnumerable<string> tagPaths, CancellationToken cancellationToken = default) =>
+        Task.FromResult<IReadOnlyDictionary<string, ReadResult>>(tagPaths.ToDictionary(
+            t => t, _ => new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null)));
+
+    public Task<WriteResult> WriteAsync(string tagPath, object? value, CancellationToken cancellationToken = default) =>
+        Task.FromResult(new WriteResult(true, null));
+
+    public Task<IReadOnlyDictionary<string, WriteResult>> WriteBatchAsync(IDictionary<string, object?> values, CancellationToken cancellationToken = default) =>
+        Task.FromResult<IReadOnlyDictionary<string, WriteResult>>(values.ToDictionary(v => v.Key, _ => new WriteResult(true, null)));
+
+    public void RaiseDisconnected()
+    {
+        _connected = false;
+        Disconnected?.Invoke();
+    }
+
+    public ValueTask DisposeAsync()
+    {
+        _connected = false;
+        return ValueTask.CompletedTask;
+    }
+}