ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/LeaderAutoDownFailoverTests.cs

using Akka.Actor;
using Xunit;
using ZB.MOM.WW.ScadaBridge.Host.Actors;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;

/// <summary>
/// ScadaBridge#106: the in-repo <see cref="LeaderAutoDowningProvider"/> (strategy
/// <c>leader-auto-down</c>) must give a REAL two-node cluster exactly the outcomes Akka's
/// <c>AutoDowning</c> (<c>auto-down</c>) gives in <see cref="SbrFailoverTests"/>: a hard crash
/// of EITHER node — the oldest included — is downed and removed by the survivor and the
/// singleton ends up answering on the survivor. The two failover cases are clones of
/// <c>SbrFailoverTests.AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton</c> and
/// <c>…AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton</c>, differing only in the strategy.
/// Under a provider that never downs, the member-removal step times out (the crashed member
/// stays Unreachable forever) — that is what gives them teeth.
/// </summary>
public class LeaderAutoDownFailoverTests
{
    private const string Strategy = "leader-auto-down";

    /// <summary>
    /// The configured provider is the one Akka actually instantiated and started: the Cluster
    /// extension holds a <see cref="LeaderAutoDowningProvider"/> and its downing actor is running
    /// under the cluster daemon on both nodes.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task LeaderAutoDown_ProviderIsWhatAkkaLoads_OnBothNodes()
    {
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);

        foreach (var node in new[] { cluster.NodeA, cluster.NodeB })
        {
            var akkaCluster = Akka.Cluster.Cluster.Get(node);
            Assert.IsType<LeaderAutoDowningProvider>(akkaCluster.DowningProvider);

            // ClusterCoreDaemon creates the provider's DowningActorProps as "downingProvider".
            var identity = await node.ActorSelection("/system/cluster/core/daemon/downingProvider")
                .Ask<ActorIdentity>(new Identify("downing"), TimeSpan.FromSeconds(5));
            Assert.NotNull(identity.Subject);
        }
    }

    /// <summary>
    /// With leader-auto-down, a hard crash of the OLDEST node lets the younger survivor down it
    /// and take over the singleton — the direction two-node keep-oldest cannot survive.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task LeaderAutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
    {
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
        SbrFailoverTests.StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
        var (_, proxyB) = SbrFailoverTests.StartSingleton(cluster.NodeB);

        var echo = await proxyB.Ask<string>("ping", TimeSpan.FromSeconds(20));
        Assert.Equal("ping", echo);

        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
        await TwoNodeClusterFixture.CrashNode(cluster.NodeA);

        // 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member.
        await TwoNodeClusterFixture.WaitForMemberRemoved(
            cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));

        // 2) B must still be a functioning cluster member (not self-downed) …
        var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
        Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");

        // 3) … and the singleton must migrate to B and answer again.
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        Exception? last = null;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var echo2 = await proxyB.Ask<string>("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
                Assert.Equal("ping-after-oldest-crash", echo2);
                return;
            }
            catch (Exception ex) { last = ex; }
        }
        throw new Xunit.Sdk.XunitException(
            $"Singleton never migrated to the younger survivor after the oldest crashed under leader-auto-down: {last}");
    }

    /// <summary>With leader-auto-down, a hard crash of the younger node leaves the singleton on the oldest node.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task LeaderAutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
    {
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
        var (_, proxyA) = SbrFailoverTests.StartSingleton(cluster.NodeA);
        SbrFailoverTests.StartSingleton(cluster.NodeB);

        Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20)));

        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);

        await TwoNodeClusterFixture.WaitForMemberRemoved(
            cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));

        Assert.False(Akka.Cluster.Cluster.Get(cluster.NodeA).IsTerminated,
            "surviving oldest's Cluster extension terminated — it downed itself");

        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        Exception? last = null;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
                Assert.Equal("ping-after-crash", echo2);
                return;
            }
            catch (Exception ex) { last = ex; }
        }
        throw new Xunit.Sdk.XunitException(
            $"Singleton stopped answering on the surviving oldest node under leader-auto-down: {last}");
    }
}