diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs new file mode 100644 index 00000000..19a7349c --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs @@ -0,0 +1,65 @@ +using Akka.Cluster; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// The load-bearing integration half of arch-review Critical 1 (03/S1) — proves the cluster actually +/// FAILS OVER when the oldest node hard-crashes: the split-brain resolver downs the alone crashed node +/// and the survivor takes over as driver-role leader. The unit guard only asserts the resolver is +/// wired; only a real hard-kill proves failover happens. +/// +/// The distinction the graceful path can NOT +/// exercise: a graceful Cluster.Leave removes a member with no downing decision at all, so it +/// passes even with downing disabled. A hard KILL leaves the survivor staring at an Unreachable +/// member — and with no active downing provider that member stays Unreachable forever, never failing +/// over. Only an active resolver downs the alone oldest node and lets the survivor take over. +/// +/// What the negative control established (2026-07-08): removing the typed +/// ClusterOptions.SplitBrainResolver did NOT break this test — Akka.Cluster.Hosting's +/// WithClustering applies SplitBrainResolverOption.Default when the option is null, which +/// enables the SBR downing provider that then reads the akka.conf keep-oldest block (present +/// on master before Critical 1). So the resolver is active via the framework default + HOCON; the typed +/// option is reinforcing, not the sole activator. This test was verified to genuinely require an active +/// downing provider by a separate run forcing explicit NoDowning +/// (akka.cluster.downing-provider-class = ""), under which it correctly times out with the node +/// stuck Up-but-Unreachable. It therefore guards the failover OUTCOME regardless of activation path. +/// +/// Traited Failover — it is deliberately slow (~acceptable-heartbeat-pause 10s + +/// stable-after 15s before the resolver acts). +/// +[Trait("Category", "Failover")] +public sealed class HardKillFailoverTests +{ + // acceptable-heartbeat-pause (10s) + stable-after (15s) + convergence/takeover margin. + private static readonly TimeSpan FailoverTimeout = TimeSpan.FromSeconds(60); + + /// + /// Hard-killing the oldest node (abrupt ActorSystem terminate, no graceful Leave) must leave the + /// survivor as the sole driver-role leader — proving the split-brain resolver downed the alone + /// oldest node and failed over. Without an active resolver the survivor keeps A Unreachable and this + /// times out. + /// + [Fact] + public async Task Hard_kill_of_oldest_node_fails_over_to_survivor_via_split_brain_resolver() + { + await using var harness = await TwoNodeClusterHarness.StartAsync(); + + // Precondition: a converged 2-member cluster from node B's own view. + var clusterB = Akka.Cluster.Cluster.Get(harness.NodeBSystem); + clusterB.State.Members.Count(m => m.Status == MemberStatus.Up).ShouldBe(2); + + // Crash the seed / oldest member with no graceful Leave — B now sees it Unreachable. + await harness.HardKillNodeAAsync(); + + // Only an ACTIVE split-brain resolver turns that Unreachable into a downing + takeover. This + // waits for the settled outcome — B as the sole Up member AND driver-role leader, held stable + // across consecutive polls — and THROWS if it never happens (the signature of an inactive + // resolver: A stuck Up-but-Unreachable, no takeover). A clean return IS the SBR proof. + var elapsed = await harness.WaitForNodeBSoleDriverLeaderAsync(FailoverTimeout); + elapsed.ShouldBeLessThan(FailoverTimeout); + + TestContext.Current.SendDiagnosticMessage($"Failover completed in {elapsed.TotalSeconds:F1}s"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs index a80f36a5..a4f120ad 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs @@ -175,6 +175,74 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable formationTimeout ?? TimeSpan.FromSeconds(20)); } + /// + /// Abruptly crashes node A (the seed / oldest member) by shutting down its remoting transport — the + /// canonical in-process crash simulation. All associations drop and heartbeats/gossip stop instantly + /// with NO graceful Cluster.Leave (the transport is dead, so A cannot gossip anything). Node B's + /// failure detector therefore marks A Unreachable (not Removed), which is the only state the + /// split-brain resolver acts on: with an active resolver (keep-oldest + down-if-alone, active here via + /// both the akka.conf block and Akka.Cluster.Hosting's default downing provider) B downs the alone + /// oldest node and takes over; with downing disabled (explicit NoDowning) B keeps A Unreachable forever + /// and never fails over. This is the crash the graceful path (a real + /// Cluster.Leave, which removes a member with no downing decision) can NOT simulate. + /// The ActorSystem itself stays alive with a dead transport until harness teardown disposes it. + /// + public async Task HardKillNodeAAsync() + { + if (NodeA is null) return; + var provider = (Akka.Remote.IRemoteActorRefProvider)((ExtendedActorSystem)NodeASystem).Provider; + await provider.Transport.Shutdown(); + } + + /// + /// Waits until node B's own cluster view has the crashed node A removed (B is the sole Up member) AND + /// B has become the driver-role leader — i.e. the split-brain resolver downed the alone oldest + /// node and B took over. Queries (node A is dead after a hard kill, so its + /// view can't be read). Returns the elapsed time to takeover; throws if + /// B never becomes the sole driver-leader within (the failure signature of + /// an INACTIVE resolver — A stays Unreachable and B never fails over). + /// + /// Upper bound; must exceed acceptable-heartbeat-pause (10s) + stable-after (15s). + public async Task WaitForNodeBSoleDriverLeaderAsync(TimeSpan timeout) + { + var start = DateTime.UtcNow; + var deadline = start + timeout; + // Require the outcome to hold across several consecutive polls so a transient during A's + // Up -> Down convergence (the members list can briefly flap while gossip settles) can't be + // mistaken for a settled takeover. + const int requiredStableStreak = 4; + var streak = 0; + while (DateTime.UtcNow < deadline) + { + var clusterB = Akka.Cluster.Cluster.Get(NodeBSystem); + var state = clusterB.State; + var upCount = state.Members.Count(m => m.Status == MemberStatus.Up); + var driverLeader = state.RoleLeader("driver"); + // SBR proof: the crashed oldest node has been DOWNED (no longer Up, so B is the sole Up + // member) and B has become the driver-role leader. With the resolver INACTIVE (NoDowning) A + // would instead stay Up-but-Unreachable — B would still see 2 Up members and never gain a + // stable role leader. (A lingers as a Down member in the Unreachable set until reaped; that + // reaping is not part of the failover contract, so it is NOT required here.) + if (upCount == 1 && driverLeader == clusterB.SelfAddress) + { + if (++streak >= requiredStableStreak) + return DateTime.UtcNow - start; + } + else + { + streak = 0; + } + await Task.Delay(250); + } + var finalState = Akka.Cluster.Cluster.Get(NodeBSystem).State; + throw new TimeoutException( + $"Node B did not become the sole driver-role leader within {timeout} after node A was hard-killed. " + + $"Up={finalState.Members.Count(m => m.Status == MemberStatus.Up)}, " + + $"Unreachable={finalState.Unreachable.Count}, " + + $"driverLeader={finalState.RoleLeader("driver")}, selfB={Akka.Cluster.Cluster.Get(NodeBSystem).SelfAddress}. " + + "A stuck-Unreachable A with no takeover is the signature of an INACTIVE split-brain resolver."); + } + /// /// Waits for node A's cluster view to reach members in /// . Used for asserting shrink-after-stop or grow-after-restart.