test(cluster): restarted original node stays standby — oldest-member active semantics proven (graceful-leave variant per 2-node keep-oldest gap)

2026-07-08 15:57:45 -04:00
parent f164f62a8b
commit 7138d47630
1 changed files with 69 additions and 0 deletions
@@ -0,0 +1,69 @@
+using Akka.Cluster;
+using Xunit;
+using ZB.MOM.WW.ScadaBridge.Host.Health;
+
+namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
+
+/// <summary>
+/// The exact scenario from arch-review 01 [High]: after the original first node
+/// (oldest = singleton host) leaves and rejoins, it comes back as the YOUNGEST
+/// member while the peer is now oldest — yet by Akka's address-ordered rule the
+/// rejoiner may regain cluster LEADERSHIP. The two definitions permanently
+/// diverge, which is why every product active/standby decision must use
+/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> (oldest
+/// member = singleton host) and never <c>cluster.State.Leader</c>.
+///
+/// DEVIATION FROM THE PLAN (documented): the plan drafted this as a hard CRASH of
+/// the oldest node. Under two-node <c>keep-oldest</c> (verified empirically in
+/// <c>SbrFailoverTests</c>, Akka 1.5.62) a hard crash of the oldest makes the
+/// younger survivor down ITSELF — total cluster loss — so the survivor could not
+/// be asserted to stay up. A graceful cluster LEAVE of the oldest instead models a
+/// rolling restart / redeploy of the first node (the realistic way the original
+/// oldest cycles out) and hands oldest-ship to the survivor cleanly, without any
+/// unreachable-partition SBR decision. The divergence assertions are identical and
+/// leadership-independent either way.
+/// </summary>
+public class ActiveNodeSemanticsTests
+{
+    [Fact]
+    public async Task AfterOldestNodeRestarts_SurvivorStaysActive_RejoinerIsStandby()
+    {
+        await using var fixture = await TwoNodeClusterFixture.StartAsync();
+
+        // Gracefully remove A (oldest); B takes over as oldest. A graceful Leave
+        // (not a hard crash) is required because two-node keep-oldest downs the
+        // survivor on an oldest-node partition — see the class-doc deviation note.
+        var clusterA = Akka.Cluster.Cluster.Get(fixture.NodeA);
+        var victim = clusterA.SelfAddress;
+        clusterA.Leave(victim);
+        await TwoNodeClusterFixture.WaitForMemberRemoved(fixture.NodeB, victim, TimeSpan.FromSeconds(30));
+
+        // Release the port so the "restarted" original node can rebind it.
+        await fixture.NodeA.Terminate();
+
+        // Restart A on the same port — it rejoins as the YOUNGEST member but
+        // (lowest address) may regain Akka leadership.
+        var rejoined = fixture.StartNode(fixture.PortA, "Central");
+        try
+        {
+            await TwoNodeClusterFixture.WaitForMembersUp(rejoined, 2, TimeSpan.FromSeconds(30));
+            await TwoNodeClusterFixture.WaitForMembersUp(fixture.NodeB, 2, TimeSpan.FromSeconds(30));
+
+            var clusterB = Akka.Cluster.Cluster.Get(fixture.NodeB);
+            var clusterA2 = Akka.Cluster.Cluster.Get(rejoined);
+
+            // Product active-node semantics: survivor (oldest, singleton host)
+            // is active; the rejoined original node is standby — even when it
+            // holds Akka leadership.
+            Assert.True(ClusterActivityEvaluator.SelfIsOldest(clusterB),
+                "Survivor B must be the oldest Up member (the active/singleton-host node).");
+            Assert.False(ClusterActivityEvaluator.SelfIsOldest(clusterA2),
+                "Rejoined original node is the youngest member — it must NOT be active, " +
+                "even if it holds address-ordered Akka leadership (that is the divergence).");
+        }
+        finally
+        {
+            await rejoined.Terminate();
+        }
+    }
+}