using System.Diagnostics; using Akka.Actor; using Akka.TestKit.Xunit2; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection; using ZB.MOM.WW.ScadaBridge.Commons.Serialization; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors; /// /// ScadaBridge#141 — a redundant OPC UA server that is going away. On the QA Warm gate a graceful /// stop of the Primary reported ServiceLevel 0 for ~6 s (shorter than the 10 s degraded window), the /// next sample was unreadable and cleared the streak, and the connection then retried the stopped /// node FailoverRetryCount times before failing over: 66 s, while the peer was Active in 0.2 s. /// /// /// Two triggers, each with its non-trigger: (1) ServiceLevel 0 probes the other endpoint without the /// window (other Degraded levels still wait it out); (2) a link lost while the session's last KNOWN /// level was Degraded probes the other endpoint before the first reconnect and switches if it is /// Healthy (a link lost after Healthy levels — today's 250/240 pair — retries exactly as before). /// Anti-flap is kept: Healthy-and-higher and the dwell. The "no shortcut" assertions are made on the /// order of sessions opened (the artifact), each paired with a positive control. /// /// public class DataConnectionGoingAwayFailoverTests : TestKit { private const string Primary = "opc.tcp://otopcua-a:4840"; private const string Backup = "opc.tcp://otopcua-b:4840"; private const string Tag = "nsu=https://zb.com/otopcua/uns;s=Line1.Pump.Speed"; private const int RetryCount = 3; private readonly ServiceLevelRig _rig = new(); private readonly ISiteHealthCollector _health = Substitute.For(); private readonly IDataConnectionFactory _factory = Substitute.For(); // ReconnectInterval 1 s: the old path to the backup after a lost link needs RetryCount failed // reconnects of the stopped node, i.e. at least 3 s and three Primary sessions attempted. private readonly DataConnectionOptions _options = new() { ReconnectInterval = TimeSpan.FromSeconds(1), TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200), WriteTimeout = TimeSpan.FromSeconds(5), SeedReadMaxAttempts = 1, SeedReadRetryDelay = TimeSpan.FromMilliseconds(10), ServiceLevelPollInterval = TimeSpan.FromMilliseconds(50), ServiceLevelProbeTimeout = TimeSpan.FromSeconds(2), }; /// Wires the factory to build a rig-bound fake for whichever endpoint it is asked for. public DataConnectionGoingAwayFailoverTests() : base(@"akka.loglevel = DEBUG") { _factory.Create(Arg.Any(), Arg.Any>()) .Returns(ci => new FakeServiceLevelConnection(_rig, ((IDictionary)ci[1])["Endpoint"])); } private static Dictionary Config( string endpoint, bool? enabled = null, int? windowSeconds = null, int? dwellSeconds = null) { var config = new Dictionary { ["Endpoint"] = endpoint }; if (enabled is { } e) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey] = e.ToString(); if (windowSeconds is { } w) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey] = w.ToString(); if (dwellSeconds is { } d) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey] = d.ToString(); return config; } private IActorRef Start(string name, IDictionary primaryConfig, IDictionary? backupConfig) { var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor( name, new FakeServiceLevelConnection(_rig, Primary), _options, _health, _factory, "OpcUa", primaryConfig, backupConfig, RetryCount, null)), name); actor.Tell(new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow)); ExpectMsg(TimeSpan.FromSeconds(5)); return actor; } private string ActiveEndpoint(IActorRef actor) => actor.Ask(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(3)) .GetAwaiter().GetResult().ActiveEndpoint; private bool LiveOn(string endpoint) => _rig.Subscribes.Contains($"{endpoint}|{Tag}"); /// Waits until the connected endpoint has been sampled more times. private void AwaitSamples(string endpoint, int more) { var target = _rig.ReadCount(endpoint) + more; // Generous: under a full parallel test run the 50 ms poll can be starved for seconds. AwaitCondition(() => _rig.ReadCount(endpoint) >= target, TimeSpan.FromSeconds(15)); } // ── Trigger 1: ServiceLevel 0 skips the degraded window ── /// /// The graceful stop as seen by the poll: the Primary drops from 250 to 0 while the peer is /// promoted to 250. With a 30 s window the connection still moves within a couple of seconds. /// [Fact] public void ServiceLevel0_SkipsTheWindow_AndFailsOverToTheHealthyPeer() { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 90); var actor = Start("ga-zero", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); var sw = Stopwatch.StartNew(); _rig.SetLevel(Primary, 0); _rig.SetLevel(Backup, 250); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10)); Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"took {sw.Elapsed.TotalSeconds:F1}s against a 30s window"); Assert.Equal("Backup", ActiveEndpoint(actor)); } /// /// Non-trigger: any other Degraded level (Warm standby 90, booting 1) still waits out the window — /// the shortcut is for 0 only. Positive control in the same connection: the level was sampled /// across the wait, and the moment it reads 0 the connection does move. /// /// A Degraded level other than 0. [Theory] [InlineData(90)] [InlineData(1)] public void OtherDegradedLevels_StillWaitTheWindow_Until0IsReported(byte degraded) { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 250); var actor = Start($"ga-window-{degraded}", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); _rig.SetLevel(Primary, degraded); AwaitSamples(Primary, 10); // well inside the 30 s window Assert.Equal(0, _rig.ConnectCount(Backup)); // not even probed Assert.Equal("Primary", ActiveEndpoint(actor)); _rig.SetLevel(Primary, 0); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10)); } /// /// Anti-flap kept: ServiceLevel 0 with the peer Degraded too stays put (positive control: the /// peer WAS probed). The shortcut is taken once per streak — the peer is then re-probed once per /// window, not on every poll — and once the peer turns Healthy the next window moves the connection. /// [Fact] public void ServiceLevel0_PeerNotHealthy_StaysPut_AndReprobesOncePerWindow() { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 90); var actor = Start("ga-zero-both", Config(Primary, windowSeconds: 3, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); _rig.SetLevel(Primary, 0); AwaitCondition(() => _rig.ReadCount(Backup) >= 1, TimeSpan.FromSeconds(5)); AwaitSamples(Primary, 20); // ~1 s of polls at 50 ms, inside the 3 s window Assert.Equal(1, _rig.ReadCount(Backup)); Assert.Equal("Primary", ActiveEndpoint(actor)); Assert.False(LiveOn(Backup)); _rig.SetLevel(Backup, 250); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10)); } /// /// Anti-flap kept: ServiceLevel 0 inside the dwell of a previous ServiceLevel switch does not move /// the connection back; after the dwell it does. /// [Fact] public void ServiceLevel0_InsideTheDwell_Waits_ThenSwitches() { _rig.SetLevel(Primary, 90); _rig.SetLevel(Backup, 250); var actor = Start("ga-zero-dwell", Config(Primary, windowSeconds: 30, dwellSeconds: 6), Config(Backup)); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5)); // connect-time switch starts the dwell var switchedAt = Stopwatch.StartNew(); _rig.SetLevel(Backup, 0); _rig.SetLevel(Primary, 250); var primaryConnects = _rig.ConnectCount(Primary); Thread.Sleep(2500); Assert.Equal(primaryConnects, _rig.ConnectCount(Primary)); Assert.Equal("Backup", ActiveEndpoint(actor)); AwaitCondition(() => ActiveEndpoint(actor) == "Primary", TimeSpan.FromSeconds(15)); Assert.True(switchedAt.Elapsed >= TimeSpan.FromSeconds(4), $"switched back after {switchedAt.Elapsed.TotalSeconds:F1}s, inside the 6s dwell"); } // ── Trigger 2: a link lost after a Degraded level ── /// /// The QA sequence: Degraded, then unreadable (the server is closing — the streak is cleared), /// then the link drops. The last KNOWN level was Degraded, so the peer is probed first and the /// stopped node is never retried. /// [Fact] public void LinkLost_AfterDegradedThenUnknown_ProbesThePeer_AndNeverRetriesTheStoppedNode() { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 240); var actor = Start("ga-lost", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); _rig.SetLevel(Primary, 90); AwaitSamples(Primary, 3); _rig.SetLevel(Primary, (byte?)null); AwaitSamples(Primary, 3); Assert.Equal(0, _rig.ConnectCount(Backup)); var connectsBefore = _rig.Connects.Count; var sw = Stopwatch.StartNew(); _rig.Stop(Primary); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10)); Assert.True(sw.Elapsed < TimeSpan.FromSeconds(RetryCount * _options.ReconnectInterval.TotalSeconds), $"took {sw.Elapsed.TotalSeconds:F1}s — as long as the retry cycle"); Assert.DoesNotContain(Primary, _rig.ConnectsAfter(connectsBefore)); Assert.Equal("Backup", ActiveEndpoint(actor)); } /// /// Non-trigger: today's 250/240 pair. A link lost after Healthy levels retries the same node /// FailoverRetryCount times before the connectivity failover — exactly as before #141. /// Positive control: the connectivity failover itself still happens afterwards. /// [Fact] public void LinkLost_AfterHealthyLevels_RetriesAsBefore_ThenFailsOver() { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 240); var actor = Start("ga-lost-healthy", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); var connectsBefore = _rig.Connects.Count; _rig.Stop(Primary); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(15)); var after = _rig.ConnectsAfter(connectsBefore); Assert.Equal(Enumerable.Repeat(Primary, RetryCount), after.Take(RetryCount)); Assert.Equal(Backup, after[RetryCount]); Assert.Equal("Backup", ActiveEndpoint(actor)); } /// /// Non-trigger (anti-flap): link lost after a Degraded level, but the peer is Degraded too. The /// peer is probed (positive control), then the ordinary retry cycle runs on the same node. /// [Fact] public void LinkLost_AfterDegraded_PeerNotHealthy_FallsBackToTheRetryCycle() { _rig.SetLevel(Primary, 250); _rig.SetLevel(Backup, 50); var actor = Start("ga-lost-both", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup)); AwaitSamples(Primary, 3); _rig.SetLevel(Primary, 90); AwaitSamples(Primary, 3); var connectsBefore = _rig.Connects.Count; _rig.Stop(Primary); AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Count(e => e == Primary) >= 2, TimeSpan.FromSeconds(10)); var after = _rig.ConnectsAfter(connectsBefore); Assert.Equal(Backup, after[0]); // the probe Assert.Equal(Primary, after[1]); // then the normal retry Assert.False(LiveOn(Backup)); Assert.Equal("Primary", ActiveEndpoint(actor)); } /// /// Non-trigger (anti-flap): a link lost inside the dwell of the previous ServiceLevel switch /// takes the normal retry cycle — no probe. Positive control: the identical sequence after the /// dwell () /// does probe; here the backup it sits on is retried first. /// [Fact] public void LinkLost_InsideTheDwell_TakesTheRetryCycle() { _rig.SetLevel(Primary, 90); _rig.SetLevel(Backup, 250); var actor = Start("ga-lost-dwell", Config(Primary, windowSeconds: 30, dwellSeconds: 120), Config(Backup)); AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5)); // connect-time switch starts the dwell AwaitSamples(Backup, 3); _rig.SetLevel(Backup, 90); _rig.SetLevel(Primary, 250); AwaitSamples(Backup, 3); var connectsBefore = _rig.Connects.Count; _rig.Stop(Backup); AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Length >= 2, TimeSpan.FromSeconds(10)); var after = _rig.ConnectsAfter(connectsBefore); Assert.Equal(Backup, after[0]); Assert.Equal(Backup, after[1]); Assert.Equal("Backup", ActiveEndpoint(actor)); } /// /// Non-trigger: ServiceLevel failover disabled — a lost link never probes, whatever the levels. /// Positive control: the retry cycle runs (the stopped node is attempted again). /// [Fact] public void LinkLost_FeatureDisabled_NeverProbes() { _rig.SetLevel(Primary, 0); _rig.SetLevel(Backup, 250); var actor = Start("ga-lost-disabled", Config(Primary, enabled: false), Config(Backup)); AwaitCondition(() => LiveOn(Primary), TimeSpan.FromSeconds(5)); var connectsBefore = _rig.Connects.Count; _rig.Stop(Primary); AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Length >= 2, TimeSpan.FromSeconds(10)); Assert.All(_rig.ConnectsAfter(connectsBefore).Take(2), e => Assert.Equal(Primary, e)); Assert.Equal(0, _rig.ReadCount(Backup)); Assert.Equal("Primary", ActiveEndpoint(actor)); } }