ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/SbrFailoverTests.cs

using Akka.Actor;
using Akka.Cluster.Tools.Singleton;
using Xunit;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;

/// <summary>
/// Behavioral proof that the downing provider enabled in
/// <c>AkkaHostedService.BuildHocon</c> (arch-review 01 Critical) is actually active:
/// after a hard crash, the surviving node DOWNS and REMOVES the crashed member. Under
/// the pre-fix Akka default (NoDowning) the crashed member lingers <c>Unreachable</c>
/// forever, so the member-removal assertions here are impossible to satisfy without the
/// fix — that is what gives the tests teeth.
///
/// TWO-NODE SEMANTICS (verified against Akka.NET 1.5.62 source + live on the docker
/// rig, 2026-07-21):
/// <list type="bullet">
/// <item><c>keep-oldest</c> — downs the side that does NOT contain the oldest member,
/// and its <c>down-if-alone</c> escape only fires when the surviving side has ≥2
/// members (<c>KeepOldest.OldestDecision</c>: <c>otherSide == 1 &amp;&amp; thisSide &gt;= 2</c>).
/// With 1-vs-1 the younger survivor therefore takes <c>DownReachable</c> — it downs
/// ITSELF — so only a YOUNGER-node crash is survivable.</item>
/// <item><c>auto-down</c> (production default, decision 2026-07-21) — the leader among
/// the reachable members downs the unreachable peer after the stability window, so a
/// crash of EITHER node fails over to the survivor; the accepted trade is dual-active
/// during a real network partition.</item>
/// </list>
/// </summary>
public class SbrFailoverTests
{
    private sealed class EchoActor : ReceiveActor
    {
        public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
    }

    private static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys)
    {
        var manager = sys.ActorOf(ClusterSingletonManager.Props(
            Props.Create(() => new EchoActor()),
            PoisonPill.Instance,
            ClusterSingletonManagerSettings.Create(sys).WithSingletonName("failover-probe")),
            "failover-probe-singleton");
        var proxy = sys.ActorOf(ClusterSingletonProxy.Props(
            "/user/failover-probe-singleton",
            ClusterSingletonProxySettings.Create(sys).WithSingletonName("failover-probe")),
            "failover-probe-proxy");
        return (manager, proxy);
    }

    [Fact]
    public async Task HardCrashOfYoungerNode_SbrDownsIt_AndOldestKeepsSingleton()
    {
        // Pinned to keep-oldest: this is the SBR path's (only) survivable direction.
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "keep-oldest");
        var (_, proxyA) = StartSingleton(cluster.NodeA); // oldest hosts the singleton
        StartSingleton(cluster.NodeB);

        // Singleton is reachable from A (the oldest / singleton host).
        var echo = await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20));
        Assert.Equal("ping", echo);

        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);

        // 1) SBR must DOWN and REMOVE the crashed younger member (NoDowning => this
        //    times out; the member stays unreachable forever).
        // Budget: failure detection (~2s) + stable-after (3s) + gossip margin.
        await TwoNodeClusterFixture.WaitForMemberRemoved(
            cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));

        // 2) The oldest survivor stays up and its singleton keeps answering.
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        Exception? last = null;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
                Assert.Equal("ping-after-crash", echo2);
                return;
            }
            catch (Exception ex) { last = ex; }
        }
        throw new Xunit.Sdk.XunitException($"Singleton stopped answering on the surviving oldest node after SBR downing: {last}");
    }

    [Fact]
    public async Task AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
    {
        // Decision 2026-07-21: the direction two-node keep-oldest can NEVER survive
        // (proven live on the docker rig — the younger survivor took DownReachable and
        // self-downed). Under auto-down the survivor must instead down the crashed
        // oldest and TAKE OVER its singleton.
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
        StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
        var (_, proxyB) = StartSingleton(cluster.NodeB);

        // Singleton reachable from B while A is alive (proxy routes to the oldest).
        var echo = await proxyB.Ask<string>("ping", TimeSpan.FromSeconds(20));
        Assert.Equal("ping", echo);

        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
        await TwoNodeClusterFixture.CrashNode(cluster.NodeA);

        // 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member —
        //    the exact step keep-oldest refuses (it downs itself instead).
        await TwoNodeClusterFixture.WaitForMemberRemoved(
            cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));

        // 2) B must still be a functioning cluster member (not self-downed) …
        var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
        Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");

        // 3) … and the singleton must migrate to B and answer again.
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        Exception? last = null;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var echo2 = await proxyB.Ask<string>("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
                Assert.Equal("ping-after-oldest-crash", echo2);
                return;
            }
            catch (Exception ex) { last = ex; }
        }
        throw new Xunit.Sdk.XunitException(
            $"Singleton never migrated to the younger survivor after the oldest crashed under auto-down: {last}");
    }

    [Fact]
    public async Task AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
    {
        // The previously-survivable direction must STAY survivable under auto-down.
        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
        var (_, proxyA) = StartSingleton(cluster.NodeA);
        StartSingleton(cluster.NodeB);

        Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20)));

        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);

        await TwoNodeClusterFixture.WaitForMemberRemoved(
            cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));

        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        Exception? last = null;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
                Assert.Equal("ping-after-crash", echo2);
                return;
            }
            catch (Exception ex) { last = ex; }
        }
        throw new Xunit.Sdk.XunitException(
            $"Singleton stopped answering on the surviving oldest node under auto-down: {last}");
    }
}