using Akka.Actor; using Akka.Cluster.Tools.Singleton; using Xunit; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// /// Behavioral proof that the downing provider enabled in /// AkkaHostedService.BuildHocon (arch-review 01 Critical) is actually active: /// after a hard crash, the surviving node DOWNS and REMOVES the crashed member. Under /// the pre-fix Akka default (NoDowning) the crashed member lingers Unreachable /// forever, so the member-removal assertions here are impossible to satisfy without the /// fix — that is what gives the tests teeth. /// /// TWO-NODE SEMANTICS (verified against Akka.NET 1.5.62 source + live on the docker /// rig, 2026-07-21): /// /// keep-oldest — downs the side that does NOT contain the oldest member, /// and its down-if-alone escape only fires when the surviving side has ≥2 /// members (KeepOldest.OldestDecision: otherSide == 1 && thisSide >= 2). /// With 1-vs-1 the younger survivor therefore takes DownReachable — it downs /// ITSELF — so only a YOUNGER-node crash is survivable. /// auto-down (production default, decision 2026-07-21) — the leader among /// the reachable members downs the unreachable peer after the stability window, so a /// crash of EITHER node fails over to the survivor; the accepted trade is dual-active /// during a real network partition. /// /// public class SbrFailoverTests { private sealed class EchoActor : ReceiveActor { public EchoActor() => ReceiveAny(msg => Sender.Tell(msg)); } private static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys) { var manager = sys.ActorOf(ClusterSingletonManager.Props( Props.Create(() => new EchoActor()), PoisonPill.Instance, ClusterSingletonManagerSettings.Create(sys).WithSingletonName("failover-probe")), "failover-probe-singleton"); var proxy = sys.ActorOf(ClusterSingletonProxy.Props( "/user/failover-probe-singleton", ClusterSingletonProxySettings.Create(sys).WithSingletonName("failover-probe")), "failover-probe-proxy"); return (manager, proxy); } [Fact] public async Task HardCrashOfYoungerNode_SbrDownsIt_AndOldestKeepsSingleton() { // Pinned to keep-oldest: this is the SBR path's (only) survivable direction. await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "keep-oldest"); var (_, proxyA) = StartSingleton(cluster.NodeA); // oldest hosts the singleton StartSingleton(cluster.NodeB); // Singleton is reachable from A (the oldest / singleton host). var echo = await proxyA.Ask("ping", TimeSpan.FromSeconds(20)); Assert.Equal("ping", echo); var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; await TwoNodeClusterFixture.CrashNode(cluster.NodeB); // 1) SBR must DOWN and REMOVE the crashed younger member (NoDowning => this // times out; the member stays unreachable forever). // Budget: failure detection (~2s) + stable-after (3s) + gossip margin. await TwoNodeClusterFixture.WaitForMemberRemoved( cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30)); // 2) The oldest survivor stays up and its singleton keeps answering. var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); Exception? last = null; while (DateTime.UtcNow < deadline) { try { var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3)); Assert.Equal("ping-after-crash", echo2); return; } catch (Exception ex) { last = ex; } } throw new Xunit.Sdk.XunitException($"Singleton stopped answering on the surviving oldest node after SBR downing: {last}"); } [Fact] public async Task AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton() { // Decision 2026-07-21: the direction two-node keep-oldest can NEVER survive // (proven live on the docker rig — the younger survivor took DownReachable and // self-downed). Under auto-down the survivor must instead down the crashed // oldest and TAKE OVER its singleton. await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down"); StartSingleton(cluster.NodeA); // oldest hosts the singleton initially var (_, proxyB) = StartSingleton(cluster.NodeB); // Singleton reachable from B while A is alive (proxy routes to the oldest). var echo = await proxyB.Ask("ping", TimeSpan.FromSeconds(20)); Assert.Equal("ping", echo); var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress; await TwoNodeClusterFixture.CrashNode(cluster.NodeA); // 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member — // the exact step keep-oldest refuses (it downs itself instead). await TwoNodeClusterFixture.WaitForMemberRemoved( cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30)); // 2) B must still be a functioning cluster member (not self-downed) … var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB); Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself"); // 3) … and the singleton must migrate to B and answer again. var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); Exception? last = null; while (DateTime.UtcNow < deadline) { try { var echo2 = await proxyB.Ask("ping-after-oldest-crash", TimeSpan.FromSeconds(3)); Assert.Equal("ping-after-oldest-crash", echo2); return; } catch (Exception ex) { last = ex; } } throw new Xunit.Sdk.XunitException( $"Singleton never migrated to the younger survivor after the oldest crashed under auto-down: {last}"); } [Fact] public async Task AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton() { // The previously-survivable direction must STAY survivable under auto-down. await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down"); var (_, proxyA) = StartSingleton(cluster.NodeA); StartSingleton(cluster.NodeB); Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(20))); var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; await TwoNodeClusterFixture.CrashNode(cluster.NodeB); await TwoNodeClusterFixture.WaitForMemberRemoved( cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30)); var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); Exception? last = null; while (DateTime.UtcNow < deadline) { try { var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3)); Assert.Equal("ping-after-crash", echo2); return; } catch (Exception ex) { last = ex; } } throw new Xunit.Sdk.XunitException( $"Singleton stopped answering on the surviving oldest node under auto-down: {last}"); } }