ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionGoingAwayFailoverTests.cs

using System.Diagnostics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;

/// <summary>
/// ScadaBridge#141 — a redundant OPC UA server that is going away. On the QA Warm gate a graceful
/// stop of the Primary reported ServiceLevel 0 for ~6 s (shorter than the 10 s degraded window), the
/// next sample was unreadable and cleared the streak, and the connection then retried the stopped
/// node <c>FailoverRetryCount</c> times before failing over: 66 s, while the peer was Active in 0.2 s.
///
/// <para>
/// Two triggers, each with its non-trigger: (1) ServiceLevel 0 probes the other endpoint without the
/// window (other Degraded levels still wait it out); (2) a link lost while the session's last KNOWN
/// level was Degraded probes the other endpoint before the first reconnect and switches if it is
/// Healthy (a link lost after Healthy levels — today's 250/240 pair — retries exactly as before).
/// Anti-flap is kept: Healthy-and-higher and the dwell. The "no shortcut" assertions are made on the
/// order of sessions opened (the artifact), each paired with a positive control.
/// </para>
/// </summary>
public class DataConnectionGoingAwayFailoverTests : TestKit
{
    private const string Primary = "opc.tcp://otopcua-a:4840";
    private const string Backup = "opc.tcp://otopcua-b:4840";
    private const string Tag = "nsu=https://zb.com/otopcua/uns;s=Line1.Pump.Speed";
    private const int RetryCount = 3;

    private readonly ServiceLevelRig _rig = new();
    private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
    private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();

    // ReconnectInterval 1 s: the old path to the backup after a lost link needs RetryCount failed
    // reconnects of the stopped node, i.e. at least 3 s and three Primary sessions attempted.
    private readonly DataConnectionOptions _options = new()
    {
        ReconnectInterval = TimeSpan.FromSeconds(1),
        TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
        WriteTimeout = TimeSpan.FromSeconds(5),
        SeedReadMaxAttempts = 1,
        SeedReadRetryDelay = TimeSpan.FromMilliseconds(10),
        ServiceLevelPollInterval = TimeSpan.FromMilliseconds(50),
        ServiceLevelProbeTimeout = TimeSpan.FromSeconds(2),
    };

    /// <summary>Wires the factory to build a rig-bound fake for whichever endpoint it is asked for.</summary>
    public DataConnectionGoingAwayFailoverTests()
        : base(@"akka.loglevel = DEBUG")
    {
        _factory.Create(Arg.Any<string>(), Arg.Any<IDictionary<string, string>>())
            .Returns(ci => new FakeServiceLevelConnection(_rig, ((IDictionary<string, string>)ci[1])["Endpoint"]));
    }

    private static Dictionary<string, string> Config(
        string endpoint, bool? enabled = null, int? windowSeconds = null, int? dwellSeconds = null)
    {
        var config = new Dictionary<string, string> { ["Endpoint"] = endpoint };
        if (enabled is { } e) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey] = e.ToString();
        if (windowSeconds is { } w) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey] = w.ToString();
        if (dwellSeconds is { } d) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey] = d.ToString();
        return config;
    }

    private IActorRef Start(string name, IDictionary<string, string> primaryConfig, IDictionary<string, string>? backupConfig)
    {
        var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
            name, new FakeServiceLevelConnection(_rig, Primary), _options, _health, _factory, "OpcUa",
            primaryConfig, backupConfig, RetryCount, null)), name);
        actor.Tell(new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        ExpectMsg<SubscribeTagsResponse>(TimeSpan.FromSeconds(5));
        return actor;
    }

    private string ActiveEndpoint(IActorRef actor) =>
        actor.Ask<DataConnectionHealthReport>(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(3))
            .GetAwaiter().GetResult().ActiveEndpoint;

    private bool LiveOn(string endpoint) => _rig.Subscribes.Contains($"{endpoint}|{Tag}");

    /// <summary>Waits until the connected endpoint has been sampled <paramref name="more"/> more times.</summary>
    private void AwaitSamples(string endpoint, int more)
    {
        var target = _rig.ReadCount(endpoint) + more;
        // Generous: under a full parallel test run the 50 ms poll can be starved for seconds.
        AwaitCondition(() => _rig.ReadCount(endpoint) >= target, TimeSpan.FromSeconds(15));
    }

    // ── Trigger 1: ServiceLevel 0 skips the degraded window ──

    /// <summary>
    /// The graceful stop as seen by the poll: the Primary drops from 250 to 0 while the peer is
    /// promoted to 250. With a 30 s window the connection still moves within a couple of seconds.
    /// </summary>
    [Fact]
    public void ServiceLevel0_SkipsTheWindow_AndFailsOverToTheHealthyPeer()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 90);
        var actor = Start("ga-zero", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);

        var sw = Stopwatch.StartNew();
        _rig.SetLevel(Primary, 0);
        _rig.SetLevel(Backup, 250);

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
        Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"took {sw.Elapsed.TotalSeconds:F1}s against a 30s window");
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Non-trigger: any other Degraded level (Warm standby 90, booting 1) still waits out the window —
    /// the shortcut is for 0 only. Positive control in the same connection: the level was sampled
    /// across the wait, and the moment it reads 0 the connection does move.
    /// </summary>
    /// <param name="degraded">A Degraded level other than 0.</param>
    [Theory]
    [InlineData(90)]
    [InlineData(1)]
    public void OtherDegradedLevels_StillWaitTheWindow_Until0IsReported(byte degraded)
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 250);
        var actor = Start($"ga-window-{degraded}", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);

        _rig.SetLevel(Primary, degraded);
        AwaitSamples(Primary, 10); // well inside the 30 s window
        Assert.Equal(0, _rig.ConnectCount(Backup)); // not even probed
        Assert.Equal("Primary", ActiveEndpoint(actor));

        _rig.SetLevel(Primary, 0);
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
    }

    /// <summary>
    /// Anti-flap kept: ServiceLevel 0 with the peer Degraded too stays put (positive control: the
    /// peer WAS probed). The shortcut is taken once per streak — the peer is then re-probed once per
    /// window, not on every poll — and once the peer turns Healthy the next window moves the connection.
    /// </summary>
    [Fact]
    public void ServiceLevel0_PeerNotHealthy_StaysPut_AndReprobesOncePerWindow()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 90);
        var actor = Start("ga-zero-both", Config(Primary, windowSeconds: 3, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);

        _rig.SetLevel(Primary, 0);
        AwaitCondition(() => _rig.ReadCount(Backup) >= 1, TimeSpan.FromSeconds(5));
        AwaitSamples(Primary, 20); // ~1 s of polls at 50 ms, inside the 3 s window
        Assert.Equal(1, _rig.ReadCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
        Assert.False(LiveOn(Backup));

        _rig.SetLevel(Backup, 250);
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
    }

    /// <summary>
    /// Anti-flap kept: ServiceLevel 0 inside the dwell of a previous ServiceLevel switch does not move
    /// the connection back; after the dwell it does.
    /// </summary>
    [Fact]
    public void ServiceLevel0_InsideTheDwell_Waits_ThenSwitches()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var actor = Start("ga-zero-dwell", Config(Primary, windowSeconds: 30, dwellSeconds: 6), Config(Backup));
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5)); // connect-time switch starts the dwell
        var switchedAt = Stopwatch.StartNew();

        _rig.SetLevel(Backup, 0);
        _rig.SetLevel(Primary, 250);
        var primaryConnects = _rig.ConnectCount(Primary);
        Thread.Sleep(2500);
        Assert.Equal(primaryConnects, _rig.ConnectCount(Primary));
        Assert.Equal("Backup", ActiveEndpoint(actor));

        AwaitCondition(() => ActiveEndpoint(actor) == "Primary", TimeSpan.FromSeconds(15));
        Assert.True(switchedAt.Elapsed >= TimeSpan.FromSeconds(4),
            $"switched back after {switchedAt.Elapsed.TotalSeconds:F1}s, inside the 6s dwell");
    }

    // ── Trigger 2: a link lost after a Degraded level ──

    /// <summary>
    /// The QA sequence: Degraded, then unreadable (the server is closing — the streak is cleared),
    /// then the link drops. The last KNOWN level was Degraded, so the peer is probed first and the
    /// stopped node is never retried.
    /// </summary>
    [Fact]
    public void LinkLost_AfterDegradedThenUnknown_ProbesThePeer_AndNeverRetriesTheStoppedNode()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 240);
        var actor = Start("ga-lost", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);

        _rig.SetLevel(Primary, 90);
        AwaitSamples(Primary, 3);
        _rig.SetLevel(Primary, (byte?)null);
        AwaitSamples(Primary, 3);
        Assert.Equal(0, _rig.ConnectCount(Backup));

        var connectsBefore = _rig.Connects.Count;
        var sw = Stopwatch.StartNew();
        _rig.Stop(Primary);

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
        Assert.True(sw.Elapsed < TimeSpan.FromSeconds(RetryCount * _options.ReconnectInterval.TotalSeconds),
            $"took {sw.Elapsed.TotalSeconds:F1}s — as long as the retry cycle");
        Assert.DoesNotContain(Primary, _rig.ConnectsAfter(connectsBefore));
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Non-trigger: today's 250/240 pair. A link lost after Healthy levels retries the same node
    /// <c>FailoverRetryCount</c> times before the connectivity failover — exactly as before #141.
    /// Positive control: the connectivity failover itself still happens afterwards.
    /// </summary>
    [Fact]
    public void LinkLost_AfterHealthyLevels_RetriesAsBefore_ThenFailsOver()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 240);
        var actor = Start("ga-lost-healthy", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);

        var connectsBefore = _rig.Connects.Count;
        _rig.Stop(Primary);

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(15));
        var after = _rig.ConnectsAfter(connectsBefore);
        Assert.Equal(Enumerable.Repeat(Primary, RetryCount), after.Take(RetryCount));
        Assert.Equal(Backup, after[RetryCount]);
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Non-trigger (anti-flap): link lost after a Degraded level, but the peer is Degraded too. The
    /// peer is probed (positive control), then the ordinary retry cycle runs on the same node.
    /// </summary>
    [Fact]
    public void LinkLost_AfterDegraded_PeerNotHealthy_FallsBackToTheRetryCycle()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 50);
        var actor = Start("ga-lost-both", Config(Primary, windowSeconds: 30, dwellSeconds: 0), Config(Backup));
        AwaitSamples(Primary, 3);
        _rig.SetLevel(Primary, 90);
        AwaitSamples(Primary, 3);

        var connectsBefore = _rig.Connects.Count;
        _rig.Stop(Primary);

        AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Count(e => e == Primary) >= 2, TimeSpan.FromSeconds(10));
        var after = _rig.ConnectsAfter(connectsBefore);
        Assert.Equal(Backup, after[0]);   // the probe
        Assert.Equal(Primary, after[1]);  // then the normal retry
        Assert.False(LiveOn(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Non-trigger (anti-flap): a link lost inside the dwell of the previous ServiceLevel switch
    /// takes the normal retry cycle — no probe. Positive control: the identical sequence after the
    /// dwell (<see cref="LinkLost_AfterDegradedThenUnknown_ProbesThePeer_AndNeverRetriesTheStoppedNode"/>)
    /// does probe; here the backup it sits on is retried first.
    /// </summary>
    [Fact]
    public void LinkLost_InsideTheDwell_TakesTheRetryCycle()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var actor = Start("ga-lost-dwell", Config(Primary, windowSeconds: 30, dwellSeconds: 120), Config(Backup));
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5)); // connect-time switch starts the dwell
        AwaitSamples(Backup, 3);
        _rig.SetLevel(Backup, 90);
        _rig.SetLevel(Primary, 250);
        AwaitSamples(Backup, 3);

        var connectsBefore = _rig.Connects.Count;
        _rig.Stop(Backup);

        AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Length >= 2, TimeSpan.FromSeconds(10));
        var after = _rig.ConnectsAfter(connectsBefore);
        Assert.Equal(Backup, after[0]);
        Assert.Equal(Backup, after[1]);
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Non-trigger: ServiceLevel failover disabled — a lost link never probes, whatever the levels.
    /// Positive control: the retry cycle runs (the stopped node is attempted again).
    /// </summary>
    [Fact]
    public void LinkLost_FeatureDisabled_NeverProbes()
    {
        _rig.SetLevel(Primary, 0);
        _rig.SetLevel(Backup, 250);
        var actor = Start("ga-lost-disabled", Config(Primary, enabled: false), Config(Backup));
        AwaitCondition(() => LiveOn(Primary), TimeSpan.FromSeconds(5));

        var connectsBefore = _rig.Connects.Count;
        _rig.Stop(Primary);

        AwaitCondition(() => _rig.ConnectsAfter(connectsBefore).Length >= 2, TimeSpan.FromSeconds(10));
        Assert.All(_rig.ConnectsAfter(connectsBefore).Take(2), e => Assert.Equal(Primary, e));
        Assert.Equal(0, _rig.ReadCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }
}