From 7138d47630af644d4df8322aea52633dbdae199d Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Wed, 8 Jul 2026 15:57:45 -0400
Subject: [PATCH] =?UTF-8?q?test(cluster):=20restarted=20original=20node=20?=
 =?UTF-8?q?stays=20standby=20=E2=80=94=20oldest-member=20active=20semantic?=
 =?UTF-8?q?s=20proven=20(graceful-leave=20variant=20per=202-node=20keep-ol?=
 =?UTF-8?q?dest=20gap)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../Cluster/ActiveNodeSemanticsTests.cs       | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ActiveNodeSemanticsTests.cs

diff --git a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ActiveNodeSemanticsTests.cs b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ActiveNodeSemanticsTests.cs
new file mode 100644
index 00000000..9778ecf1
--- /dev/null
+++ b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ActiveNodeSemanticsTests.cs
@@ -0,0 +1,69 @@
+using Akka.Cluster;
+using Xunit;
+using ZB.MOM.WW.ScadaBridge.Host.Health;
+
+namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
+
+/// <summary>
+/// The exact scenario from arch-review 01 [High]: after the original first node
+/// (oldest = singleton host) leaves and rejoins, it comes back as the YOUNGEST
+/// member while the peer is now oldest — yet by Akka's address-ordered rule the
+/// rejoiner may regain cluster LEADERSHIP. The two definitions permanently
+/// diverge, which is why every product active/standby decision must use
+/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> (oldest
+/// member = singleton host) and never <c>cluster.State.Leader</c>.
+///
+/// DEVIATION FROM THE PLAN (documented): the plan drafted this as a hard CRASH of
+/// the oldest node. Under two-node <c>keep-oldest</c> (verified empirically in
+/// <c>SbrFailoverTests</c>, Akka 1.5.62) a hard crash of the oldest makes the
+/// younger survivor down ITSELF — total cluster loss — so the survivor could not
+/// be asserted to stay up. A graceful cluster LEAVE of the oldest instead models a
+/// rolling restart / redeploy of the first node (the realistic way the original
+/// oldest cycles out) and hands oldest-ship to the survivor cleanly, without any
+/// unreachable-partition SBR decision. The divergence assertions are identical and
+/// leadership-independent either way.
+/// </summary>
+public class ActiveNodeSemanticsTests
+{
+    [Fact]
+    public async Task AfterOldestNodeRestarts_SurvivorStaysActive_RejoinerIsStandby()
+    {
+        await using var fixture = await TwoNodeClusterFixture.StartAsync();
+
+        // Gracefully remove A (oldest); B takes over as oldest. A graceful Leave
+        // (not a hard crash) is required because two-node keep-oldest downs the
+        // survivor on an oldest-node partition — see the class-doc deviation note.
+        var clusterA = Akka.Cluster.Cluster.Get(fixture.NodeA);
+        var victim = clusterA.SelfAddress;
+        clusterA.Leave(victim);
+        await TwoNodeClusterFixture.WaitForMemberRemoved(fixture.NodeB, victim, TimeSpan.FromSeconds(30));
+
+        // Release the port so the "restarted" original node can rebind it.
+        await fixture.NodeA.Terminate();
+
+        // Restart A on the same port — it rejoins as the YOUNGEST member but
+        // (lowest address) may regain Akka leadership.
+        var rejoined = fixture.StartNode(fixture.PortA, "Central");
+        try
+        {
+            await TwoNodeClusterFixture.WaitForMembersUp(rejoined, 2, TimeSpan.FromSeconds(30));
+            await TwoNodeClusterFixture.WaitForMembersUp(fixture.NodeB, 2, TimeSpan.FromSeconds(30));
+
+            var clusterB = Akka.Cluster.Cluster.Get(fixture.NodeB);
+            var clusterA2 = Akka.Cluster.Cluster.Get(rejoined);
+
+            // Product active-node semantics: survivor (oldest, singleton host)
+            // is active; the rejoined original node is standby — even when it
+            // holds Akka leadership.
+            Assert.True(ClusterActivityEvaluator.SelfIsOldest(clusterB),
+                "Survivor B must be the oldest Up member (the active/singleton-host node).");
+            Assert.False(ClusterActivityEvaluator.SelfIsOldest(clusterA2),
+                "Rejoined original node is the youngest member — it must NOT be active, " +
+                "even if it holds address-ordered Akka leadership (that is the divergence).");
+        }
+        finally
+        {
+            await rejoined.Terminate();
+        }
+    }
+}