using System.Diagnostics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
///
/// ScadaBridge#138 — ServiceLevel-aware failover for a primary/backup OPC UA connection, the
/// ScadaBridge prerequisite for OtOpcUa Warm redundancy (lmxopcua#717). Under Warm the standby keeps
/// a healthy session but serves BadOutOfService data at a standby ServiceLevel (90 / 50 /
/// boot 1); the Active node is 200 or more (250 Primary / 240 Secondary today).
///
///
/// Each trigger has a matching non-trigger case, and every "no switch" assertion is paired with a
/// positive control proving the evaluation it relies on actually ran (the ServiceLevel was read, or
/// the other endpoint was probed) — an inert feature would otherwise pass them all.
///
///
public class DataConnectionServiceLevelFailoverTests : TestKit
{
private const string Primary = "opc.tcp://otopcua-a:4840";
private const string Backup = "opc.tcp://otopcua-b:4840";
private const string Tag = "nsu=https://zb.com/otopcua/uns;s=Line1.Pump.Speed";
private readonly ServiceLevelRig _rig = new();
private readonly ISiteHealthCollector _health = Substitute.For();
private readonly IDataConnectionFactory _factory = Substitute.For();
private readonly DataConnectionOptions _options = new()
{
ReconnectInterval = TimeSpan.FromMilliseconds(100),
TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
WriteTimeout = TimeSpan.FromSeconds(5),
SeedReadMaxAttempts = 1,
SeedReadRetryDelay = TimeSpan.FromMilliseconds(10),
ServiceLevelPollInterval = TimeSpan.FromMilliseconds(50),
ServiceLevelProbeTimeout = TimeSpan.FromSeconds(2),
};
/// Wires the factory to build a rig-bound fake for whichever endpoint it is asked for.
public DataConnectionServiceLevelFailoverTests()
: base(@"akka.loglevel = DEBUG")
{
_factory.Create(Arg.Any(), Arg.Any>())
.Returns(ci => new FakeServiceLevelConnection(_rig, ((IDictionary)ci[1])["Endpoint"]));
}
private static Dictionary Config(
string endpoint, bool? enabled = null, int? windowSeconds = null, int? dwellSeconds = null)
{
var config = new Dictionary { ["Endpoint"] = endpoint };
if (enabled is { } e) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey] = e.ToString();
if (windowSeconds is { } w) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey] = w.ToString();
if (dwellSeconds is { } d) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey] = d.ToString();
return config;
}
private IActorRef Start(
string name, IDictionary primaryConfig, IDictionary? backupConfig,
IDataConnection? initialAdapter = null)
{
var adapter = initialAdapter ?? new FakeServiceLevelConnection(_rig, Primary);
var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
name, adapter, _options, _health, _factory, "OpcUa", primaryConfig, backupConfig, 3, null)), name);
actor.Tell(new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
ExpectMsg(TimeSpan.FromSeconds(5));
return actor;
}
private string ActiveEndpoint(IActorRef actor) =>
actor.Ask(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(3))
.GetAwaiter().GetResult().ActiveEndpoint;
private bool LiveOn(string endpoint) => _rig.Subscribes.Contains($"{endpoint}|{Tag}");
///
/// Warm promotion seen from a connection on the old Active: it was Healthy (250), then the
/// node becomes the Standby (90) while the other node is promoted (250). The switch waits for
/// the degraded window, then lands on the backup, where every subscription is re-issued.
///
[Fact]
public void Sustained_Degraded_FailsOver_AfterTheWindow_AndResubscribesOnTheOtherEndpoint()
{
_rig.SetLevel(Primary, 250);
_rig.SetLevel(Backup, 240);
var actor = Start("sl-window", Config(Primary, windowSeconds: 2, dwellSeconds: 0), Config(Backup));
AwaitCondition(() => _rig.ReadCount(Primary) >= 3, TimeSpan.FromSeconds(5));
var degradedAt = Stopwatch.StartNew();
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
// Not before the window: half-way through, still on the primary and nothing probed.
Thread.Sleep(1000);
Assert.Equal("Primary", ActiveEndpoint(actor));
Assert.Equal(0, _rig.ConnectCount(Backup));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
Assert.True(degradedAt.Elapsed >= TimeSpan.FromSeconds(1.9),
$"switched after {degradedAt.Elapsed.TotalSeconds:F2}s, before the 2s window");
Assert.Equal("Backup", ActiveEndpoint(actor));
}
///
/// Connect-time preference: the connection starts on a primary that is the Warm standby (90)
/// while the backup is Active (250). It leaves at once — no 30 s window — because the first
/// sample of a session is evaluated immediately.
///
[Fact]
public void Connect_OnDegradedEndpoint_PrefersTheHealthierOther_WithoutWaitingTheWindow()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
var sw = Stopwatch.StartNew();
var actor = Start("sl-connect", Config(Primary, windowSeconds: 30), Config(Backup));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
Assert.True(sw.Elapsed < TimeSpan.FromSeconds(10), $"took {sw.Elapsed.TotalSeconds:F1}s");
Assert.Equal("Backup", ActiveEndpoint(actor));
}
/// Booting node (ServiceLevel 1) on connect, Active other → leave it too.
[Fact]
public void Connect_OnBootingEndpoint_ServiceLevel1_PrefersTheActiveOther()
{
_rig.SetLevel(Primary, 1);
_rig.SetLevel(Backup, 240);
var actor = Start("sl-boot", Config(Primary, windowSeconds: 30), Config(Backup));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
Assert.Equal("Backup", ActiveEndpoint(actor));
}
///
/// Today's OtOpcUa pair: both nodes Active-band, 250/240 in either order. Nothing is ever
/// probed and nothing switches — a 240 Secondary never triggers. Positive control: the
/// connected endpoint's ServiceLevel WAS read many times across more than a full window.
///
/// The connected (primary) endpoint's ServiceLevel.
/// The backup endpoint's ServiceLevel.
[Theory]
[InlineData(240, 250)]
[InlineData(250, 240)]
[InlineData(200, 250)]
public void ActiveBandPair_NeverSwitches_NorProbes(byte primaryLevel, byte backupLevel)
{
_rig.SetLevel(Primary, primaryLevel);
_rig.SetLevel(Backup, backupLevel);
var actor = Start($"sl-today-{primaryLevel}-{backupLevel}", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
Thread.Sleep(2000);
Assert.True(_rig.ReadCount(Primary) >= 10, $"only {_rig.ReadCount(Primary)} reads — the watch did not run");
Assert.Equal(0, _rig.ConnectCount(Backup));
Assert.Equal("Primary", ActiveEndpoint(actor));
}
///
/// Both endpoints degraded (Warm standby 90 vs standby-degraded 50, either way round): stay
/// put. Positive control: the other endpoint WAS probed, so the verdict, not a dead watch,
/// kept the connection where it is.
///
/// The connected (primary) endpoint's ServiceLevel.
/// The backup endpoint's ServiceLevel.
[Theory]
[InlineData(50, 90)]
[InlineData(90, 50)]
[InlineData(90, 199)]
public void BothDegraded_StaysPut_AfterProbingTheOther(byte primaryLevel, byte backupLevel)
{
_rig.SetLevel(Primary, primaryLevel);
_rig.SetLevel(Backup, backupLevel);
var actor = Start($"sl-both-{primaryLevel}-{backupLevel}", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
AwaitCondition(() => _rig.ReadCount(Backup) >= 2, TimeSpan.FromSeconds(5));
Assert.Equal("Primary", ActiveEndpoint(actor));
Assert.False(LiveOn(Backup));
}
///
/// The other endpoint is down: stay on the degraded one (it still answers). Positive control:
/// sessions to the backup were attempted.
///
[Fact]
public void OtherEndpointUnreachable_StaysPut()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
_rig.SetReachable(Backup, false);
var actor = Start("sl-unreachable", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
AwaitCondition(() => _rig.ConnectCount(Backup) >= 2, TimeSpan.FromSeconds(5));
Assert.Equal("Primary", ActiveEndpoint(actor));
Assert.False(LiveOn(Backup));
// ...and once it comes back, the next window moves the connection.
_rig.SetReachable(Backup, true);
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
}
///
/// An unreadable ServiceLevel (Bad status, server without the node) is unknown, never
/// Degraded: no probe, no switch. Positive control: the reads happened.
///
[Fact]
public void UnknownServiceLevel_IsNotDegraded()
{
_rig.SetLevel(Primary, (byte?)null);
_rig.SetLevel(Backup, 250);
var actor = Start("sl-unknown", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
Thread.Sleep(2000);
Assert.True(_rig.ReadCount(Primary) >= 10);
Assert.Equal(0, _rig.ConnectCount(Backup));
Assert.Equal("Primary", ActiveEndpoint(actor));
}
///
/// Hysteresis: a level that keeps dipping below 200 but recovers between samples never forms
/// an unbroken window, so it never switches. Positive control: Degraded samples were seen.
///
[Fact]
public void FlickeringLevel_NeverCompletesTheWindow()
{
var reads = 0;
// Healthy for the first samples, so the connect-time check is not what is exercised; then
// three Degraded samples out of every four.
_rig.SetLevel(Primary, () =>
{
var n = Interlocked.Increment(ref reads);
return n <= 3 || n % 4 == 0 ? (byte)250 : (byte)90;
});
_rig.SetLevel(Backup, 250);
var actor = Start("sl-flicker", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
Thread.Sleep(2500);
Assert.True(Volatile.Read(ref reads) >= 20);
Assert.Equal("Primary", ActiveEndpoint(actor));
Assert.False(LiveOn(Backup));
}
///
/// Anti-flap: right after a ServiceLevel switch the roles flip again. The connection does not
/// bounce straight back — not even through the connect-time check — until the dwell has passed;
/// then it does.
///
[Fact]
public void Dwell_BlocksAnImmediateSwitchBack_ThenAllowsIt()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
var actor = Start("sl-dwell", Config(Primary, windowSeconds: 1, dwellSeconds: 8), Config(Backup));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
var switchedAt = Stopwatch.StartNew();
// Roles flip back at once. The dwell runs from the switch decision, which comes before the
// observed re-subscribe on the backup by a reconnect + subscribe round trip, so the
// "blocked" window checked here (3 s) leaves wide slack inside the 8 s dwell under load.
_rig.SetLevel(Primary, 250);
_rig.SetLevel(Backup, 90);
var primaryConnectsAfterSwitch = _rig.ConnectCount(Primary);
Thread.Sleep(3000);
Assert.Equal("Backup", ActiveEndpoint(actor));
Assert.Equal(primaryConnectsAfterSwitch, _rig.ConnectCount(Primary)); // not even probed
// Positive control: once the dwell is over, it does switch back.
AwaitCondition(() => ActiveEndpoint(actor) == "Primary", TimeSpan.FromSeconds(20));
Assert.True(switchedAt.Elapsed >= TimeSpan.FromSeconds(5),
$"switched back after {switchedAt.Elapsed.TotalSeconds:F2}s, well inside the 8s dwell");
}
///
/// ServiceLevelFailover.Enabled=False on the primary: the ServiceLevel is never even
/// read, whatever it is.
///
[Fact]
public void Disabled_NeverReadsTheServiceLevel()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
var actor = Start("sl-disabled", Config(Primary, enabled: false, windowSeconds: 1), Config(Backup));
Thread.Sleep(1500);
Assert.Equal(0, _rig.ReadCount(Primary));
Assert.Equal(0, _rig.ConnectCount(Backup));
Assert.Equal("Primary", ActiveEndpoint(actor));
}
/// Only the PRIMARY's copy of the setting counts: disabling it on the backup changes nothing.
[Fact]
public void BackupCopyOfTheSetting_IsIgnored()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
Start("sl-backup-copy", Config(Primary, windowSeconds: 1), Config(Backup, enabled: false));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
}
/// A connection without a backup has nowhere to go: the ServiceLevel is never read.
[Fact]
public void NoBackup_NeverReadsTheServiceLevel()
{
_rig.SetLevel(Primary, 90);
var actor = Start("sl-nobackup", Config(Primary, windowSeconds: 1), backupConfig: null);
Thread.Sleep(1500);
Assert.True(LiveOn(Primary)); // positive control: the session is up and subscribed
Assert.Equal(0, _rig.ReadCount(Primary));
Assert.Equal("Primary (no backup)", ActiveEndpoint(actor));
}
///
/// Symmetry: sitting on the backup (after a connectivity failover) when the backup becomes the
/// Warm standby moves the connection back to the primary.
///
[Fact]
public void OnBackup_DegradedBackup_FailsBackToAHealthyPrimary()
{
_rig.SetLevel(Primary, 90);
_rig.SetLevel(Backup, 250);
var actor = Start("sl-symmetric", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
var primarySubscribes = _rig.Subscribes.Count(s => s.StartsWith(Primary + "|", StringComparison.Ordinal));
_rig.SetLevel(Primary, 250);
_rig.SetLevel(Backup, 90);
AwaitCondition(() => _rig.Subscribes.Count(s => s.StartsWith(Primary + "|", StringComparison.Ordinal)) > primarySubscribes,
TimeSpan.FromSeconds(6));
Assert.Equal("Primary", ActiveEndpoint(actor));
}
///
/// An adapter without (MxGateway, or any plain adapter) is
/// never watched and never probed, even with a backup and the feature enabled.
///
[Fact]
public void AdapterWithoutTheCapability_IsNeverWatched()
{
var plain = Substitute.For();
plain.ConnectAsync(Arg.Any>(), Arg.Any()).Returns(Task.CompletedTask);
plain.SubscribeAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns("sub-1");
plain.ReadAsync(Arg.Any(), Arg.Any())
.Returns(new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null));
_rig.SetLevel(Backup, 250);
var actor = Start("sl-plain", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup), plain);
Thread.Sleep(1500);
Assert.Equal(0, _rig.ConnectCount(Backup));
Assert.Equal("Primary", ActiveEndpoint(actor));
}
/// The policy parser: defaults when absent, and clamps nonsense to safe values.
[Fact]
public void Policy_DefaultsAndClamps()
{
var defaults = ServiceLevelFailoverPolicy.From(new Dictionary { ["Endpoint"] = Primary });
Assert.True(defaults.Enabled);
Assert.Equal(TimeSpan.FromSeconds(10), defaults.DegradedWindow);
Assert.Equal(TimeSpan.FromSeconds(60), defaults.MinDwell);
var clamped = ServiceLevelFailoverPolicy.From(Config(Primary, enabled: false, windowSeconds: 0, dwellSeconds: -5));
Assert.False(clamped.Enabled);
Assert.Equal(TimeSpan.FromSeconds(1), clamped.DegradedWindow);
Assert.Equal(TimeSpan.Zero, clamped.MinDwell);
Assert.True(ServiceLevelFailoverPolicy.From(null).Enabled);
}
}