test(cluster): prove SBR downs a hard-crashed node and the oldest survivor keeps its singleton

Rewritten from the plan's oldest-crash scenario: empirically, 2-node keep-oldest downs the non-oldest partition, so a hard crash of the OLDEST makes the younger survivor down itself (total cluster loss) — the survivable case is a crash of the younger node. The member-removal assertion has teeth (impossible under the pre-fix NoDowning default). See the test's XML doc for the active/oldest-node-crash gap.
2026-07-08 15:06:54 -04:00
parent e3a6603c74
commit b8d91dcc5b
1 changed files with 81 additions and 0 deletions
@@ -0,0 +1,81 @@
+using Akka.Actor;
+using Akka.Cluster.Tools.Singleton;
+using Xunit;
+
+namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
+
+/// <summary>
+/// Behavioral proof that the SBR downing provider enabled in
+/// <c>AkkaHostedService.BuildHocon</c> (arch-review 01 Critical) is actually active:
+/// after a hard crash, the surviving node DOWNS and REMOVES the crashed member. Under
+/// the pre-fix Akka default (NoDowning) the crashed member lingers <c>Unreachable</c>
+/// forever, so the member-removal assertions here are impossible to satisfy without the
+/// fix — that is what gives the test teeth.
+///
+/// IMPORTANT — <c>keep-oldest</c> two-node semantics (verified empirically, Akka 1.5.62):
+/// SBR downs the partition that does NOT contain the oldest member. So the crash that
+/// SBR can recover from in a two-node cluster is the crash of the YOUNGER node — the
+/// oldest survives and keeps its singletons. Crashing the OLDEST node instead makes the
+/// younger survivor down ITSELF (total cluster loss); <c>down-if-alone=on</c> does not
+/// change this on a hard crash because the alone-oldest is no longer running to down
+/// itself. That asymmetry (active/oldest-node crash is NOT covered by two-node
+/// keep-oldest) is a design-level gap tracked separately, not something this test can
+/// assert as a success path.
+/// </summary>
+public class SbrFailoverTests
+{
+    private sealed class EchoActor : ReceiveActor
+    {
+        public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
+    }
+
+    private static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys)
+    {
+        var manager = sys.ActorOf(ClusterSingletonManager.Props(
+            Props.Create(() => new EchoActor()),
+            PoisonPill.Instance,
+            ClusterSingletonManagerSettings.Create(sys).WithSingletonName("failover-probe")),
+            "failover-probe-singleton");
+        var proxy = sys.ActorOf(ClusterSingletonProxy.Props(
+            "/user/failover-probe-singleton",
+            ClusterSingletonProxySettings.Create(sys).WithSingletonName("failover-probe")),
+            "failover-probe-proxy");
+        return (manager, proxy);
+    }
+
+    [Fact]
+    public async Task HardCrashOfYoungerNode_SbrDownsIt_AndOldestKeepsSingleton()
+    {
+        await using var cluster = await TwoNodeClusterFixture.StartAsync();
+        var (_, proxyA) = StartSingleton(cluster.NodeA); // oldest hosts the singleton
+        StartSingleton(cluster.NodeB);
+
+        // Singleton is reachable from A (the oldest / singleton host).
+        var echo = await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20));
+        Assert.Equal("ping", echo);
+
+        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
+        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
+
+        // 1) SBR must DOWN and REMOVE the crashed younger member (NoDowning => this
+        //    times out; the member stays unreachable forever).
+        // Budget: failure detection (~2s) + stable-after (3s) + gossip margin.
+        await TwoNodeClusterFixture.WaitForMemberRemoved(
+            cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
+
+        // 2) The oldest survivor stays up and its singleton keeps answering.
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+        Exception? last = null;
+        while (DateTime.UtcNow < deadline)
+        {
+            try
+            {
+                var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
+                Assert.Equal("ping-after-crash", echo2);
+                return;
+            }
+            catch (Exception ex) { last = ex; }
+        }
+        throw new Xunit.Sdk.XunitException($"Singleton stopped answering on the surviving oldest node after SBR downing: {last}");
+    }
+}