Joseph Doherty
70 lines
3.4 KiB
C#
70 lines
3.4 KiB
C#
using Akka.Cluster;
|
|
using Xunit;
|
|
using ZB.MOM.WW.ScadaBridge.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
|
|
|
/// <summary>
|
|
/// The exact scenario from arch-review 01 [High]: after the original first node
|
|
/// (oldest = singleton host) leaves and rejoins, it comes back as the YOUNGEST
|
|
/// member while the peer is now oldest — yet by Akka's address-ordered rule the
|
|
/// rejoiner may regain cluster LEADERSHIP. The two definitions permanently
|
|
/// diverge, which is why every product active/standby decision must use
|
|
/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> (oldest
|
|
/// member = singleton host) and never <c>cluster.State.Leader</c>.
|
|
///
|
|
/// DEVIATION FROM THE PLAN (documented): the plan drafted this as a hard CRASH of
|
|
/// the oldest node. Under two-node <c>keep-oldest</c> (verified empirically in
|
|
/// <c>SbrFailoverTests</c>, Akka 1.5.62) a hard crash of the oldest makes the
|
|
/// younger survivor down ITSELF — total cluster loss — so the survivor could not
|
|
/// be asserted to stay up. A graceful cluster LEAVE of the oldest instead models a
|
|
/// rolling restart / redeploy of the first node (the realistic way the original
|
|
/// oldest cycles out) and hands oldest-ship to the survivor cleanly, without any
|
|
/// unreachable-partition SBR decision. The divergence assertions are identical and
|
|
/// leadership-independent either way.
|
|
/// </summary>
|
|
public class ActiveNodeSemanticsTests
|
|
{
|
|
[Fact]
|
|
public async Task AfterOldestNodeRestarts_SurvivorStaysActive_RejoinerIsStandby()
|
|
{
|
|
await using var fixture = await TwoNodeClusterFixture.StartAsync();
|
|
|
|
// Gracefully remove A (oldest); B takes over as oldest. A graceful Leave
|
|
// (not a hard crash) is required because two-node keep-oldest downs the
|
|
// survivor on an oldest-node partition — see the class-doc deviation note.
|
|
var clusterA = Akka.Cluster.Cluster.Get(fixture.NodeA);
|
|
var victim = clusterA.SelfAddress;
|
|
clusterA.Leave(victim);
|
|
await TwoNodeClusterFixture.WaitForMemberRemoved(fixture.NodeB, victim, TimeSpan.FromSeconds(30));
|
|
|
|
// Release the port so the "restarted" original node can rebind it.
|
|
await fixture.NodeA.Terminate();
|
|
|
|
// Restart A on the same port — it rejoins as the YOUNGEST member but
|
|
// (lowest address) may regain Akka leadership.
|
|
var rejoined = fixture.StartNode(fixture.PortA, "Central");
|
|
try
|
|
{
|
|
await TwoNodeClusterFixture.WaitForMembersUp(rejoined, 2, TimeSpan.FromSeconds(30));
|
|
await TwoNodeClusterFixture.WaitForMembersUp(fixture.NodeB, 2, TimeSpan.FromSeconds(30));
|
|
|
|
var clusterB = Akka.Cluster.Cluster.Get(fixture.NodeB);
|
|
var clusterA2 = Akka.Cluster.Cluster.Get(rejoined);
|
|
|
|
// Product active-node semantics: survivor (oldest, singleton host)
|
|
// is active; the rejoined original node is standby — even when it
|
|
// holds Akka leadership.
|
|
Assert.True(ClusterActivityEvaluator.SelfIsOldest(clusterB),
|
|
"Survivor B must be the oldest Up member (the active/singleton-host node).");
|
|
Assert.False(ClusterActivityEvaluator.SelfIsOldest(clusterA2),
|
|
"Rejoined original node is the youngest member — it must NOT be active, " +
|
|
"even if it holds address-ordered Akka leadership (that is the divergence).");
|
|
}
|
|
finally
|
|
{
|
|
await rejoined.Terminate();
|
|
}
|
|
}
|
|
}
|