using Akka.Actor; using Xunit; using ZB.MOM.WW.ScadaBridge.Host.Actors; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// /// ScadaBridge#106: the in-repo (strategy /// leader-auto-down) must give a REAL two-node cluster exactly the outcomes Akka's /// AutoDowning (auto-down) gives in : a hard crash /// of EITHER node — the oldest included — is downed and removed by the survivor and the /// singleton ends up answering on the survivor. The two failover cases are clones of /// SbrFailoverTests.AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton and /// …AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton, differing only in the strategy. /// Under a provider that never downs, the member-removal step times out (the crashed member /// stays Unreachable forever) — that is what gives them teeth. /// public class LeaderAutoDownFailoverTests { private const string Strategy = "leader-auto-down"; /// /// The configured provider is the one Akka actually instantiated and started: the Cluster /// extension holds a and its downing actor is running /// under the cluster daemon on both nodes. /// /// A task that represents the asynchronous operation. [Fact] public async Task LeaderAutoDown_ProviderIsWhatAkkaLoads_OnBothNodes() { await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy); foreach (var node in new[] { cluster.NodeA, cluster.NodeB }) { var akkaCluster = Akka.Cluster.Cluster.Get(node); Assert.IsType(akkaCluster.DowningProvider); // ClusterCoreDaemon creates the provider's DowningActorProps as "downingProvider". var identity = await node.ActorSelection("/system/cluster/core/daemon/downingProvider") .Ask(new Identify("downing"), TimeSpan.FromSeconds(5)); Assert.NotNull(identity.Subject); } } /// /// With leader-auto-down, a hard crash of the OLDEST node lets the younger survivor down it /// and take over the singleton — the direction two-node keep-oldest cannot survive. /// /// A task that represents the asynchronous operation. [Fact] public async Task LeaderAutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton() { await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy); SbrFailoverTests.StartSingleton(cluster.NodeA); // oldest hosts the singleton initially var (_, proxyB) = SbrFailoverTests.StartSingleton(cluster.NodeB); var echo = await proxyB.Ask("ping", TimeSpan.FromSeconds(20)); Assert.Equal("ping", echo); var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress; await TwoNodeClusterFixture.CrashNode(cluster.NodeA); // 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member. await TwoNodeClusterFixture.WaitForMemberRemoved( cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30)); // 2) B must still be a functioning cluster member (not self-downed) … var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB); Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself"); // 3) … and the singleton must migrate to B and answer again. var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); Exception? last = null; while (DateTime.UtcNow < deadline) { try { var echo2 = await proxyB.Ask("ping-after-oldest-crash", TimeSpan.FromSeconds(3)); Assert.Equal("ping-after-oldest-crash", echo2); return; } catch (Exception ex) { last = ex; } } throw new Xunit.Sdk.XunitException( $"Singleton never migrated to the younger survivor after the oldest crashed under leader-auto-down: {last}"); } /// With leader-auto-down, a hard crash of the younger node leaves the singleton on the oldest node. /// A task that represents the asynchronous operation. [Fact] public async Task LeaderAutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton() { await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy); var (_, proxyA) = SbrFailoverTests.StartSingleton(cluster.NodeA); SbrFailoverTests.StartSingleton(cluster.NodeB); Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(20))); var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; await TwoNodeClusterFixture.CrashNode(cluster.NodeB); await TwoNodeClusterFixture.WaitForMemberRemoved( cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30)); Assert.False(Akka.Cluster.Cluster.Get(cluster.NodeA).IsTerminated, "surviving oldest's Cluster extension terminated — it downed itself"); var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); Exception? last = null; while (DateTime.UtcNow < deadline) { try { var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3)); Assert.Equal("ping-after-crash", echo2); return; } catch (Exception ex) { last = ex; } } throw new Xunit.Sdk.XunitException( $"Singleton stopped answering on the surviving oldest node under leader-auto-down: {last}"); } }