Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/FakeServiceLevelConnection.cs
T
Joseph Doherty c4ec51e89a fix(dcl): fail over within seconds when the connected OPC UA server is going away (#141)
QA Warm gate (lmxopcua#907, 2026-11-13): a graceful stop of the Warm Primary
reported ServiceLevel 0 for ~6 s - shorter than the 10 s degraded window - the
next unreadable sample cleared the streak, and the connection retried the stopped
node FailoverRetryCount times: Good on the peer after 66 s, peer Active in 0.2 s.

Two going-away triggers on top of #138, both still requiring a Healthy (>= 200),
higher level on the other endpoint and honouring the minimum dwell:
- ServiceLevel 0 (Part 4 Maintenance) probes the other endpoint at once instead
  of waiting out the window - once per degraded streak, so a non-Healthy peer is
  re-probed once per window, not every poll. Other Degraded levels (standby 90/50,
  booting 1) still wait the window.
- A lost link whose session's last KNOWN ServiceLevel was < 200 (unreadable
  samples don't overwrite it) probes the other endpoint before the first
  reconnect attempt, holding the reconnect timer. Healthy -> switch and reconnect
  at zero delay, the stopped node is never retried; otherwise the ordinary retry
  cycle runs (less the probe's time). After only Healthy levels (250/240, Unset),
  inside the dwell, or with the feature off: unchanged retry-then-failover.

Tests: DataConnectionGoingAwayFailoverTests (10 cases, each trigger and
non-trigger with positive controls, connect-order artifact assertions) and a real
in-process OPC UA server pair graceful-stop e2e on production defaults: 1.1 s
(lost-link trigger alone 12.1 s, neither 33.1 s -> fails the 10 s bound).
13 mutants, all killed. DCL tests 608/608 (3 full runs); solution 0 warnings.
Docs: Component-DataConnectionLayer, components/DataConnectionLayer.

Not deployed; QA live re-check pending.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 09:36:59 -05:00

140 lines
6.1 KiB
C#

using System.Collections.Concurrent;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
/// <summary>
/// The two OPC UA servers of a redundant pair as the ScadaBridge#138 tests see them: a
/// ServiceLevel per endpoint (changeable at any time, null = unreadable), whether each endpoint
/// accepts a session, and a record of what the actor did to them.
/// </summary>
internal sealed class ServiceLevelRig
{
private readonly ConcurrentDictionary<string, Func<byte?>> _levels = new();
private readonly ConcurrentDictionary<string, bool> _unreachable = new();
/// <summary>Every endpoint a session was opened on, in order — live sessions and probes alike.</summary>
public ConcurrentQueue<string> Connects { get; } = new();
/// <summary>Every tag subscribed, as "endpoint|tag" — only a live session receives these.</summary>
public ConcurrentQueue<string> Subscribes { get; } = new();
/// <summary>Number of ServiceLevel reads, per endpoint.</summary>
public ConcurrentDictionary<string, int> Reads { get; } = new();
/// <summary>Sets a fixed ServiceLevel for <paramref name="endpoint"/>.</summary>
public void SetLevel(string endpoint, byte? level) => _levels[endpoint] = () => level;
/// <summary>Sets a ServiceLevel source evaluated on every read of <paramref name="endpoint"/>.</summary>
public void SetLevel(string endpoint, Func<byte?> level) => _levels[endpoint] = level;
/// <summary>Makes <paramref name="endpoint"/> refuse (or accept again) new sessions.</summary>
public void SetReachable(string endpoint, bool reachable) => _unreachable[endpoint] = !reachable;
public bool IsReachable(string endpoint) => !_unreachable.GetValueOrDefault(endpoint);
public byte? Read(string endpoint)
{
Reads.AddOrUpdate(endpoint, 1, (_, n) => n + 1);
return _levels.TryGetValue(endpoint, out var level) ? level() : null;
}
public int ConnectCount(string endpoint) => Connects.Count(e => e == endpoint);
/// <summary>Every adapter that opened a session, so a test can kill the live one (ScadaBridge#141).</summary>
public ConcurrentQueue<FakeServiceLevelConnection> Sessions { get; } = new();
/// <summary>
/// The server at <paramref name="endpoint"/> goes away: it refuses new sessions and every live
/// session on it raises <c>Disconnected</c> — what a stopped OtOpcUa node looks like to the client.
/// </summary>
public void Stop(string endpoint)
{
SetReachable(endpoint, false);
foreach (var session in Sessions.Where(s => s.Endpoint == endpoint && s.IsConnected))
session.RaiseDisconnected();
}
/// <summary>The endpoints sessions were opened on after the first <paramref name="skip"/> connects.</summary>
public string[] ConnectsAfter(int skip) => Connects.Skip(skip).ToArray();
public int ReadCount(string endpoint) => Reads.GetValueOrDefault(endpoint);
}
/// <summary>
/// An <see cref="IDataConnection"/> with the <see cref="IServiceLevelReadable"/> capability, bound to
/// one endpoint of a <see cref="ServiceLevelRig"/>. Subscriptions and reads always succeed.
/// </summary>
internal sealed class FakeServiceLevelConnection : IDataConnection, IServiceLevelReadable
{
private readonly ServiceLevelRig _rig;
private volatile bool _connected;
public FakeServiceLevelConnection(ServiceLevelRig rig, string endpoint)
{
_rig = rig;
Endpoint = endpoint;
}
public string Endpoint { get; }
public ConnectionHealth Status => _connected ? ConnectionHealth.Connected : ConnectionHealth.Disconnected;
public event Action? Disconnected;
public Task ConnectAsync(IDictionary<string, string> connectionDetails, CancellationToken cancellationToken = default)
{
_rig.Connects.Enqueue(Endpoint);
if (!_rig.IsReachable(Endpoint))
return Task.FromException(new InvalidOperationException($"{Endpoint} refused the session"));
_connected = true;
_rig.Sessions.Enqueue(this);
return Task.CompletedTask;
}
public bool IsConnected => _connected;
public Task DisconnectAsync(CancellationToken cancellationToken = default)
{
_connected = false;
return Task.CompletedTask;
}
public Task<byte?> ReadServiceLevelAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(_connected ? _rig.Read(Endpoint) : null);
public Task<string> SubscribeAsync(string tagPath, SubscriptionCallback callback, CancellationToken cancellationToken = default)
{
_rig.Subscribes.Enqueue($"{Endpoint}|{tagPath}");
return Task.FromResult($"{Endpoint}:{tagPath}");
}
public Task UnsubscribeAsync(string subscriptionId, CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task<ReadResult> ReadAsync(string tagPath, CancellationToken cancellationToken = default) =>
Task.FromResult(new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null));
public Task<IReadOnlyDictionary<string, ReadResult>> ReadBatchAsync(IEnumerable<string> tagPaths, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyDictionary<string, ReadResult>>(tagPaths.ToDictionary(
t => t, _ => new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null)));
public Task<WriteResult> WriteAsync(string tagPath, object? value, CancellationToken cancellationToken = default) =>
Task.FromResult(new WriteResult(true, null));
public Task<IReadOnlyDictionary<string, WriteResult>> WriteBatchAsync(IDictionary<string, object?> values, CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlyDictionary<string, WriteResult>>(values.ToDictionary(v => v.Key, _ => new WriteResult(true, null)));
public void RaiseDisconnected()
{
_connected = false;
Disconnected?.Invoke();
}
public ValueTask DisposeAsync()
{
_connected = false;
return ValueTask.CompletedTask;
}
}