ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionServiceLevelFailoverTests.cs

using System.Diagnostics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;

/// <summary>
/// ScadaBridge#138 — ServiceLevel-aware failover for a primary/backup OPC UA connection, the
/// ScadaBridge prerequisite for OtOpcUa Warm redundancy (lmxopcua#717). Under Warm the standby keeps
/// a healthy session but serves <c>BadOutOfService</c> data at a standby ServiceLevel (90 / 50 /
/// boot 1); the Active node is 200 or more (250 Primary / 240 Secondary today).
///
/// <para>
/// Each trigger has a matching non-trigger case, and every "no switch" assertion is paired with a
/// positive control proving the evaluation it relies on actually ran (the ServiceLevel was read, or
/// the other endpoint was probed) — an inert feature would otherwise pass them all.
/// </para>
/// </summary>
public class DataConnectionServiceLevelFailoverTests : TestKit
{
    private const string Primary = "opc.tcp://otopcua-a:4840";
    private const string Backup = "opc.tcp://otopcua-b:4840";
    private const string Tag = "nsu=https://zb.com/otopcua/uns;s=Line1.Pump.Speed";

    private readonly ServiceLevelRig _rig = new();
    private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
    private readonly IDataConnectionFactory _factory = Substitute.For<IDataConnectionFactory>();
    private readonly DataConnectionOptions _options = new()
    {
        ReconnectInterval = TimeSpan.FromMilliseconds(100),
        TagResolutionRetryInterval = TimeSpan.FromMilliseconds(200),
        WriteTimeout = TimeSpan.FromSeconds(5),
        SeedReadMaxAttempts = 1,
        SeedReadRetryDelay = TimeSpan.FromMilliseconds(10),
        ServiceLevelPollInterval = TimeSpan.FromMilliseconds(50),
        ServiceLevelProbeTimeout = TimeSpan.FromSeconds(2),
    };

    /// <summary>Wires the factory to build a rig-bound fake for whichever endpoint it is asked for.</summary>
    public DataConnectionServiceLevelFailoverTests()
        : base(@"akka.loglevel = DEBUG")
    {
        _factory.Create(Arg.Any<string>(), Arg.Any<IDictionary<string, string>>())
            .Returns(ci => new FakeServiceLevelConnection(_rig, ((IDictionary<string, string>)ci[1])["Endpoint"]));
    }

    private static Dictionary<string, string> Config(
        string endpoint, bool? enabled = null, int? windowSeconds = null, int? dwellSeconds = null)
    {
        var config = new Dictionary<string, string> { ["Endpoint"] = endpoint };
        if (enabled is { } e) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverEnabledKey] = e.ToString();
        if (windowSeconds is { } w) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverWindowKey] = w.ToString();
        if (dwellSeconds is { } d) config[OpcUaEndpointConfigSerializer.ServiceLevelFailoverDwellKey] = d.ToString();
        return config;
    }

    private IActorRef Start(
        string name, IDictionary<string, string> primaryConfig, IDictionary<string, string>? backupConfig,
        IDataConnection? initialAdapter = null)
    {
        var adapter = initialAdapter ?? new FakeServiceLevelConnection(_rig, Primary);
        var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
            name, adapter, _options, _health, _factory, "OpcUa", primaryConfig, backupConfig, 3, null)), name);
        actor.Tell(new SubscribeTagsRequest("c1", "inst1", name, [Tag], DateTimeOffset.UtcNow));
        ExpectMsg<SubscribeTagsResponse>(TimeSpan.FromSeconds(5));
        return actor;
    }

    private string ActiveEndpoint(IActorRef actor) =>
        actor.Ask<DataConnectionHealthReport>(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(3))
            .GetAwaiter().GetResult().ActiveEndpoint;

    private bool LiveOn(string endpoint) => _rig.Subscribes.Contains($"{endpoint}|{Tag}");

    /// <summary>
    /// Warm promotion seen from a connection on the old Active: it was Healthy (250), then the
    /// node becomes the Standby (90) while the other node is promoted (250). The switch waits for
    /// the degraded window, then lands on the backup, where every subscription is re-issued.
    /// </summary>
    [Fact]
    public void Sustained_Degraded_FailsOver_AfterTheWindow_AndResubscribesOnTheOtherEndpoint()
    {
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 240);
        var actor = Start("sl-window", Config(Primary, windowSeconds: 2, dwellSeconds: 0), Config(Backup));
        AwaitCondition(() => _rig.ReadCount(Primary) >= 3, TimeSpan.FromSeconds(5));

        var degradedAt = Stopwatch.StartNew();
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);

        // Not before the window: half-way through, still on the primary and nothing probed.
        Thread.Sleep(1000);
        Assert.Equal("Primary", ActiveEndpoint(actor));
        Assert.Equal(0, _rig.ConnectCount(Backup));

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
        Assert.True(degradedAt.Elapsed >= TimeSpan.FromSeconds(1.9),
            $"switched after {degradedAt.Elapsed.TotalSeconds:F2}s, before the 2s window");
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Connect-time preference: the connection starts on a primary that is the Warm standby (90)
    /// while the backup is Active (250). It leaves at once — no 30 s window — because the first
    /// sample of a session is evaluated immediately.
    /// </summary>
    [Fact]
    public void Connect_OnDegradedEndpoint_PrefersTheHealthierOther_WithoutWaitingTheWindow()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var sw = Stopwatch.StartNew();
        var actor = Start("sl-connect", Config(Primary, windowSeconds: 30), Config(Backup));

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
        Assert.True(sw.Elapsed < TimeSpan.FromSeconds(10), $"took {sw.Elapsed.TotalSeconds:F1}s");
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>Booting node (ServiceLevel 1) on connect, Active other → leave it too.</summary>
    [Fact]
    public void Connect_OnBootingEndpoint_ServiceLevel1_PrefersTheActiveOther()
    {
        _rig.SetLevel(Primary, 1);
        _rig.SetLevel(Backup, 240);
        var actor = Start("sl-boot", Config(Primary, windowSeconds: 30), Config(Backup));

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
        Assert.Equal("Backup", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Today's OtOpcUa pair: both nodes Active-band, 250/240 in either order. Nothing is ever
    /// probed and nothing switches — a 240 Secondary never triggers. Positive control: the
    /// connected endpoint's ServiceLevel WAS read many times across more than a full window.
    /// </summary>
    /// <param name="primaryLevel">The connected (primary) endpoint's ServiceLevel.</param>
    /// <param name="backupLevel">The backup endpoint's ServiceLevel.</param>
    [Theory]
    [InlineData(240, 250)]
    [InlineData(250, 240)]
    [InlineData(200, 250)]
    public void ActiveBandPair_NeverSwitches_NorProbes(byte primaryLevel, byte backupLevel)
    {
        _rig.SetLevel(Primary, primaryLevel);
        _rig.SetLevel(Backup, backupLevel);
        var actor = Start($"sl-today-{primaryLevel}-{backupLevel}", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));

        Thread.Sleep(2000);
        Assert.True(_rig.ReadCount(Primary) >= 10, $"only {_rig.ReadCount(Primary)} reads — the watch did not run");
        Assert.Equal(0, _rig.ConnectCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Both endpoints degraded (Warm standby 90 vs standby-degraded 50, either way round): stay
    /// put. Positive control: the other endpoint WAS probed, so the verdict, not a dead watch,
    /// kept the connection where it is.
    /// </summary>
    /// <param name="primaryLevel">The connected (primary) endpoint's ServiceLevel.</param>
    /// <param name="backupLevel">The backup endpoint's ServiceLevel.</param>
    [Theory]
    [InlineData(50, 90)]
    [InlineData(90, 50)]
    [InlineData(90, 199)]
    public void BothDegraded_StaysPut_AfterProbingTheOther(byte primaryLevel, byte backupLevel)
    {
        _rig.SetLevel(Primary, primaryLevel);
        _rig.SetLevel(Backup, backupLevel);
        var actor = Start($"sl-both-{primaryLevel}-{backupLevel}", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));

        AwaitCondition(() => _rig.ReadCount(Backup) >= 2, TimeSpan.FromSeconds(5));
        Assert.Equal("Primary", ActiveEndpoint(actor));
        Assert.False(LiveOn(Backup));
    }

    /// <summary>
    /// The other endpoint is down: stay on the degraded one (it still answers). Positive control:
    /// sessions to the backup were attempted.
    /// </summary>
    [Fact]
    public void OtherEndpointUnreachable_StaysPut()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        _rig.SetReachable(Backup, false);
        var actor = Start("sl-unreachable", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));

        AwaitCondition(() => _rig.ConnectCount(Backup) >= 2, TimeSpan.FromSeconds(5));
        Assert.Equal("Primary", ActiveEndpoint(actor));
        Assert.False(LiveOn(Backup));

        // ...and once it comes back, the next window moves the connection.
        _rig.SetReachable(Backup, true);
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
    }

    /// <summary>
    /// An unreadable ServiceLevel (Bad status, server without the node) is unknown, never
    /// Degraded: no probe, no switch. Positive control: the reads happened.
    /// </summary>
    [Fact]
    public void UnknownServiceLevel_IsNotDegraded()
    {
        _rig.SetLevel(Primary, (byte?)null);
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-unknown", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));

        Thread.Sleep(2000);
        Assert.True(_rig.ReadCount(Primary) >= 10);
        Assert.Equal(0, _rig.ConnectCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Hysteresis: a level that keeps dipping below 200 but recovers between samples never forms
    /// an unbroken window, so it never switches. Positive control: Degraded samples were seen.
    /// </summary>
    [Fact]
    public void FlickeringLevel_NeverCompletesTheWindow()
    {
        var reads = 0;
        // Healthy for the first samples, so the connect-time check is not what is exercised; then
        // three Degraded samples out of every four.
        _rig.SetLevel(Primary, () =>
        {
            var n = Interlocked.Increment(ref reads);
            return n <= 3 || n % 4 == 0 ? (byte)250 : (byte)90;
        });
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-flicker", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));

        Thread.Sleep(2500);
        Assert.True(Volatile.Read(ref reads) >= 20);
        Assert.Equal("Primary", ActiveEndpoint(actor));
        Assert.False(LiveOn(Backup));
    }

    /// <summary>
    /// Anti-flap: right after a ServiceLevel switch the roles flip again. The connection does not
    /// bounce straight back — not even through the connect-time check — until the dwell has passed;
    /// then it does.
    /// </summary>
    [Fact]
    public void Dwell_BlocksAnImmediateSwitchBack_ThenAllowsIt()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-dwell", Config(Primary, windowSeconds: 1, dwellSeconds: 8), Config(Backup));
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(10));
        var switchedAt = Stopwatch.StartNew();

        // Roles flip back at once. The dwell runs from the switch decision, which comes before the
        // observed re-subscribe on the backup by a reconnect + subscribe round trip, so the
        // "blocked" window checked here (3 s) leaves wide slack inside the 8 s dwell under load.
        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 90);
        var primaryConnectsAfterSwitch = _rig.ConnectCount(Primary);

        Thread.Sleep(3000);
        Assert.Equal("Backup", ActiveEndpoint(actor));
        Assert.Equal(primaryConnectsAfterSwitch, _rig.ConnectCount(Primary)); // not even probed

        // Positive control: once the dwell is over, it does switch back.
        AwaitCondition(() => ActiveEndpoint(actor) == "Primary", TimeSpan.FromSeconds(20));
        Assert.True(switchedAt.Elapsed >= TimeSpan.FromSeconds(5),
            $"switched back after {switchedAt.Elapsed.TotalSeconds:F2}s, well inside the 8s dwell");
    }

    /// <summary>
    /// <c>ServiceLevelFailover.Enabled=False</c> on the primary: the ServiceLevel is never even
    /// read, whatever it is.
    /// </summary>
    [Fact]
    public void Disabled_NeverReadsTheServiceLevel()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-disabled", Config(Primary, enabled: false, windowSeconds: 1), Config(Backup));

        Thread.Sleep(1500);
        Assert.Equal(0, _rig.ReadCount(Primary));
        Assert.Equal(0, _rig.ConnectCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>Only the PRIMARY's copy of the setting counts: disabling it on the backup changes nothing.</summary>
    [Fact]
    public void BackupCopyOfTheSetting_IsIgnored()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        Start("sl-backup-copy", Config(Primary, windowSeconds: 1), Config(Backup, enabled: false));

        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
    }

    /// <summary>A connection without a backup has nowhere to go: the ServiceLevel is never read.</summary>
    [Fact]
    public void NoBackup_NeverReadsTheServiceLevel()
    {
        _rig.SetLevel(Primary, 90);
        var actor = Start("sl-nobackup", Config(Primary, windowSeconds: 1), backupConfig: null);

        Thread.Sleep(1500);
        Assert.True(LiveOn(Primary)); // positive control: the session is up and subscribed
        Assert.Equal(0, _rig.ReadCount(Primary));
        Assert.Equal("Primary (no backup)", ActiveEndpoint(actor));
    }

    /// <summary>
    /// Symmetry: sitting on the backup (after a connectivity failover) when the backup becomes the
    /// Warm standby moves the connection back to the primary.
    /// </summary>
    [Fact]
    public void OnBackup_DegradedBackup_FailsBackToAHealthyPrimary()
    {
        _rig.SetLevel(Primary, 90);
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-symmetric", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup));
        AwaitCondition(() => LiveOn(Backup), TimeSpan.FromSeconds(5));
        var primarySubscribes = _rig.Subscribes.Count(s => s.StartsWith(Primary + "|", StringComparison.Ordinal));

        _rig.SetLevel(Primary, 250);
        _rig.SetLevel(Backup, 90);

        AwaitCondition(() => _rig.Subscribes.Count(s => s.StartsWith(Primary + "|", StringComparison.Ordinal)) > primarySubscribes,
            TimeSpan.FromSeconds(6));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>
    /// An adapter without <see cref="IServiceLevelReadable"/> (MxGateway, or any plain adapter) is
    /// never watched and never probed, even with a backup and the feature enabled.
    /// </summary>
    [Fact]
    public void AdapterWithoutTheCapability_IsNeverWatched()
    {
        var plain = Substitute.For<IDataConnection>();
        plain.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
        plain.SubscribeAsync(Arg.Any<string>(), Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>())
            .Returns("sub-1");
        plain.ReadAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
            .Returns(new ReadResult(true, new TagValue(1, QualityCode.Good, DateTimeOffset.UtcNow), null));
        _rig.SetLevel(Backup, 250);
        var actor = Start("sl-plain", Config(Primary, windowSeconds: 1, dwellSeconds: 0), Config(Backup), plain);

        Thread.Sleep(1500);
        Assert.Equal(0, _rig.ConnectCount(Backup));
        Assert.Equal("Primary", ActiveEndpoint(actor));
    }

    /// <summary>The policy parser: defaults when absent, and clamps nonsense to safe values.</summary>
    [Fact]
    public void Policy_DefaultsAndClamps()
    {
        var defaults = ServiceLevelFailoverPolicy.From(new Dictionary<string, string> { ["Endpoint"] = Primary });
        Assert.True(defaults.Enabled);
        Assert.Equal(TimeSpan.FromSeconds(10), defaults.DegradedWindow);
        Assert.Equal(TimeSpan.FromSeconds(60), defaults.MinDwell);

        var clamped = ServiceLevelFailoverPolicy.From(Config(Primary, enabled: false, windowSeconds: 0, dwellSeconds: -5));
        Assert.False(clamped.Enabled);
        Assert.Equal(TimeSpan.FromSeconds(1), clamped.DegradedWindow);
        Assert.Equal(TimeSpan.Zero, clamped.MinDwell);

        Assert.True(ServiceLevelFailoverPolicy.From(null).Enabled);
    }
}