From a81dea102c09d7ac274f14f60292b3931db30f73 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 16:14:26 -0400 Subject: [PATCH 1/3] fix(cluster): activate split-brain resolver (arch-review 03/S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split-brain-resolver HOCON block in akka.conf was inert — nothing registered a downing provider, so the cluster ran Akka's default NoDowning: a hard-crashed node was never downed, cluster singletons + the driver role-leader never failed over, and a partition left both redundancy sides at ServiceLevel 240 indefinitely. Activate the resolver via the typed ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) in a new testable BuildClusterOptions helper. keep-oldest + down-if-alone is correct for a 2-node warm-redundancy pair. HOCON keeps stable-after (typed option can't express it) + a cross-reference comment. Deterministic unit guard added; end-to-end verified the resolved ActorSystem config carries the provider and stable-after survives the HOCON/Hosting merge. Redundancy.md corrected. Hard-kill failover integration test tracked as a follow-up (effort-M harness). --- docs/Redundancy.md | 18 +++++- .../Resources/akka.conf | 7 +++ .../ServiceCollectionExtensions.cs | 38 +++++++++++-- .../SplitBrainResolverActivationTests.cs | 55 +++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 7a2252c5..7d78c1f9 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -154,7 +154,23 @@ Node A lists Node B's `ApplicationUri` and vice-versa. Validated by `DualEndpoin ## Split-brain -`akka.conf` configures Akka's split-brain resolver with `active-strategy = keep-oldest`, `stable-after = 15s`, and `failure-detector.threshold = 10.0`. Under a clean partition: the oldest member stays up + the smaller (or younger) side downs itself within ~15 seconds. The `RedundancyStateActor` on the surviving partition re-computes from the post-partition `Cluster.State`. +The split-brain resolver is **activated in code** by the typed `ClusterOptions.SplitBrainResolver` +(`KeepOldestOption { DownIfAlone = true }`) set in +`ServiceCollectionExtensions.BuildClusterOptions` — this is what registers +`Akka.Cluster.SBR.SplitBrainResolverProvider`. **Without that typed registration the `split-brain-resolver` +HOCON block in `akka.conf` is inert and the cluster runs Akka's default `NoDowning`**, meaning a +hard-crashed node is never downed and neither the cluster singletons nor the `driver` role-leader ever fail +over (a partition would leave both redundancy sides at ServiceLevel 240 indefinitely). The HOCON block is the +tuning source only: `active-strategy = keep-oldest`, `stable-after = 15s`, `keep-oldest.down-if-alone = on`, +`failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone` must stay consistent with the +typed option; `stable-after` lives only in HOCON because the typed option can't express it). + +`keep-oldest` is the correct strategy for a 2-node warm-redundancy pair: under a clean partition the oldest +member (typically the long-running primary) stays up and the smaller (or younger) side downs itself within +~`stable-after` seconds; `down-if-alone` downs a node that loses its only peer. `keep-majority`/`static-quorum` +are wrong for two nodes (no majority in a 1-1 split). The `RedundancyStateActor` on the surviving partition +re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node now +triggers the same failover. There is no operator-driven role swap during a partition. Failover is what the cluster does automatically. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf index d64cbf11..4735fa89 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf @@ -37,6 +37,13 @@ akka { roles = [] min-nr-of-members = 1 + # Split-brain resolver. This HOCON block is the TUNING source only — it does NOT + # activate the resolver. Activation happens in code via the typed + # ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) set in + # ServiceCollectionExtensions.BuildClusterOptions; without that the cluster runs Akka's + # default NoDowning and hard-crashed nodes never fail over (arch-review 03/S1). + # active-strategy + keep-oldest.down-if-alone below MUST match the typed option. + # stable-after must stay >= failure-detector.acceptable-heartbeat-pause (10s) + margin. split-brain-resolver { active-strategy = "keep-oldest" stable-after = 15s diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index 2d4fa44c..8d4f1fab 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Akka.Cluster.Hosting; +using Akka.Cluster.Hosting.SBR; using Akka.Event; using Akka.Hosting; using Akka.Logger.Serilog; @@ -77,12 +78,39 @@ public static class ServiceCollectionExtensions PublicHostName = options.PublicHostname, }); - builder.WithClustering(new ClusterOptions - { - SeedNodes = options.SeedNodes, - Roles = options.Roles, - }); + builder.WithClustering(BuildClusterOptions(options)); return builder; } + + /// + /// Builds the for the fused-host cluster, including the + /// activating split-brain-resolver registration. + /// + /// The split-brain-resolver HOCON block in Resources/akka.conf is inert on its own — + /// Akka.NET only runs the resolver when a downing provider is registered, otherwise the cluster + /// falls back to NoDowning and a hard-crashed node is never downed (singletons and the + /// driver role-leader never fail over, and a partition leaves both redundancy sides at + /// ServiceLevel 240 forever). Setting is what + /// activates Akka.Cluster.SBR.SplitBrainResolverProvider under the hood — this is the piece + /// that was missing (arch-review 03/S1). + /// + /// with DownIfAlone=true mirrors the HOCON intent and is the + /// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically + /// the long-running primary) survives, and down-if-alone downs a singleton that loses its + /// peer. keep-majority/static-quorum are wrong for two nodes. The strategy + down-if-alone + /// here MUST stay consistent with the HOCON block; stable-after lives only in HOCON because the + /// typed option cannot express it (it must stay ≥ failure-detector.acceptable-heartbeat-pause). + /// + /// The bound cluster options carrying seed nodes and roles. + /// The cluster options with seed nodes, roles, and the activated split-brain resolver. + public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options) + { + return new ClusterOptions + { + SeedNodes = options.SeedNodes, + Roles = options.Roles, + SplitBrainResolver = new KeepOldestOption { DownIfAlone = true }, + }; + } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs new file mode 100644 index 00000000..9f404027 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs @@ -0,0 +1,55 @@ +using Akka.Cluster.Hosting; +using Akka.Cluster.Hosting.SBR; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Guards the split-brain-resolver ACTIVATION (arch-review 03/S1). The split-brain-resolver +/// HOCON block is inert on its own; the resolver only runs when the typed +/// is set. These tests assert +/// registers it, so a future refactor +/// cannot silently drop back to Akka's default NoDowning (the exact "built-but-never-wired" failure +/// mode this finding described). +/// +public sealed class SplitBrainResolverActivationTests +{ + private static AkkaClusterOptions SampleOptions() => new() + { + SeedNodes = new[] { "akka.tcp://otopcua@127.0.0.1:4053" }, + Roles = new[] { "admin", "driver" }, + }; + + /// Verifies the cluster options carry a split-brain resolver (provider gets activated). + [Fact] + public void BuildClusterOptions_activates_a_split_brain_resolver() + { + var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions()); + + options.SplitBrainResolver.ShouldNotBeNull( + "SplitBrainResolver must be set — without it the cluster runs default NoDowning and never fails over on a hard crash."); + } + + /// Verifies the resolver is keep-oldest with down-if-alone (correct for a 2-node pair). + [Fact] + public void BuildClusterOptions_uses_keep_oldest_with_down_if_alone() + { + var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions()); + + var keepOldest = options.SplitBrainResolver.ShouldBeOfType(); + keepOldest.DownIfAlone.ShouldBe(true); + } + + /// Verifies seed nodes and roles are still threaded through unchanged. + [Fact] + public void BuildClusterOptions_preserves_seed_nodes_and_roles() + { + var input = SampleOptions(); + + var options = ServiceCollectionExtensions.BuildClusterOptions(input); + + options.SeedNodes.ShouldBe(input.SeedNodes); + options.Roles.ShouldBe(input.Roles); + } +} From a25c9ed097e20c723bf3fefadc2529932cf2875d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 18:24:49 -0400 Subject: [PATCH 2/3] test(cluster): hard-kill failover integration test on the 2-node harness (arch-review 03/S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The load-bearing integration half of Critical 1. Adds TwoNodeClusterHarness.HardKillNodeAAsync (canonical in-process crash sim: shut down node A's remoting transport via IRemoteActorRefProvider.Transport.Shutdown() — associations drop, heartbeats/gossip stop, NO graceful Cluster.Leave, so node B's failure detector marks A Unreachable) + a WaitForNodeBSoleDriverLeaderAsync helper (waits for a STABLE takeover: B sole Up member + driver role-leader). HardKillFailoverTests crashes the oldest node and asserts the survivor takes over — the exact scenario the graceful StopNodeBAsync path (Cluster.Leave, no downing decision) cannot exercise. Live-verified in-process (~35s: acceptable-heartbeat-pause 10s + stable-after 15s + convergence). Negative control confirmed the test BITES: forcing explicit NoDowning (akka.cluster.downing-provider-class = "") makes it time out with the node stuck Up-but-Unreachable. FINDING surfaced by the negative control: removing Critical 1's typed ClusterOptions.SplitBrainResolver did NOT break failover, because Akka.Cluster.Hosting's WithClustering applies SplitBrainResolverOption.Default when the option is null — enabling the SBR downing provider that reads the akka.conf keep-oldest block (present on master BEFORE Critical 1). So the cluster was NOT running NoDowning before Critical 1; the typed option reinforces the akka.conf strategy rather than being the sole activator, and Critical 1's 'HOCON inert / NoDowning' premise is inaccurate. The test guards the failover OUTCOME regardless of activation path. Comments corrected to reflect this; flagged for review. --- .../HardKillFailoverTests.cs | 65 ++++++++++++++++++ .../TwoNodeClusterHarness.cs | 68 +++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs new file mode 100644 index 00000000..19a7349c --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/HardKillFailoverTests.cs @@ -0,0 +1,65 @@ +using Akka.Cluster; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// The load-bearing integration half of arch-review Critical 1 (03/S1) — proves the cluster actually +/// FAILS OVER when the oldest node hard-crashes: the split-brain resolver downs the alone crashed node +/// and the survivor takes over as driver-role leader. The unit guard only asserts the resolver is +/// wired; only a real hard-kill proves failover happens. +/// +/// The distinction the graceful path can NOT +/// exercise: a graceful Cluster.Leave removes a member with no downing decision at all, so it +/// passes even with downing disabled. A hard KILL leaves the survivor staring at an Unreachable +/// member — and with no active downing provider that member stays Unreachable forever, never failing +/// over. Only an active resolver downs the alone oldest node and lets the survivor take over. +/// +/// What the negative control established (2026-07-08): removing the typed +/// ClusterOptions.SplitBrainResolver did NOT break this test — Akka.Cluster.Hosting's +/// WithClustering applies SplitBrainResolverOption.Default when the option is null, which +/// enables the SBR downing provider that then reads the akka.conf keep-oldest block (present +/// on master before Critical 1). So the resolver is active via the framework default + HOCON; the typed +/// option is reinforcing, not the sole activator. This test was verified to genuinely require an active +/// downing provider by a separate run forcing explicit NoDowning +/// (akka.cluster.downing-provider-class = ""), under which it correctly times out with the node +/// stuck Up-but-Unreachable. It therefore guards the failover OUTCOME regardless of activation path. +/// +/// Traited Failover — it is deliberately slow (~acceptable-heartbeat-pause 10s + +/// stable-after 15s before the resolver acts). +/// +[Trait("Category", "Failover")] +public sealed class HardKillFailoverTests +{ + // acceptable-heartbeat-pause (10s) + stable-after (15s) + convergence/takeover margin. + private static readonly TimeSpan FailoverTimeout = TimeSpan.FromSeconds(60); + + /// + /// Hard-killing the oldest node (abrupt ActorSystem terminate, no graceful Leave) must leave the + /// survivor as the sole driver-role leader — proving the split-brain resolver downed the alone + /// oldest node and failed over. Without an active resolver the survivor keeps A Unreachable and this + /// times out. + /// + [Fact] + public async Task Hard_kill_of_oldest_node_fails_over_to_survivor_via_split_brain_resolver() + { + await using var harness = await TwoNodeClusterHarness.StartAsync(); + + // Precondition: a converged 2-member cluster from node B's own view. + var clusterB = Akka.Cluster.Cluster.Get(harness.NodeBSystem); + clusterB.State.Members.Count(m => m.Status == MemberStatus.Up).ShouldBe(2); + + // Crash the seed / oldest member with no graceful Leave — B now sees it Unreachable. + await harness.HardKillNodeAAsync(); + + // Only an ACTIVE split-brain resolver turns that Unreachable into a downing + takeover. This + // waits for the settled outcome — B as the sole Up member AND driver-role leader, held stable + // across consecutive polls — and THROWS if it never happens (the signature of an inactive + // resolver: A stuck Up-but-Unreachable, no takeover). A clean return IS the SBR proof. + var elapsed = await harness.WaitForNodeBSoleDriverLeaderAsync(FailoverTimeout); + elapsed.ShouldBeLessThan(FailoverTimeout); + + TestContext.Current.SendDiagnosticMessage($"Failover completed in {elapsed.TotalSeconds:F1}s"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs index a80f36a5..a4f120ad 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs @@ -175,6 +175,74 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable formationTimeout ?? TimeSpan.FromSeconds(20)); } + /// + /// Abruptly crashes node A (the seed / oldest member) by shutting down its remoting transport — the + /// canonical in-process crash simulation. All associations drop and heartbeats/gossip stop instantly + /// with NO graceful Cluster.Leave (the transport is dead, so A cannot gossip anything). Node B's + /// failure detector therefore marks A Unreachable (not Removed), which is the only state the + /// split-brain resolver acts on: with an active resolver (keep-oldest + down-if-alone, active here via + /// both the akka.conf block and Akka.Cluster.Hosting's default downing provider) B downs the alone + /// oldest node and takes over; with downing disabled (explicit NoDowning) B keeps A Unreachable forever + /// and never fails over. This is the crash the graceful path (a real + /// Cluster.Leave, which removes a member with no downing decision) can NOT simulate. + /// The ActorSystem itself stays alive with a dead transport until harness teardown disposes it. + /// + public async Task HardKillNodeAAsync() + { + if (NodeA is null) return; + var provider = (Akka.Remote.IRemoteActorRefProvider)((ExtendedActorSystem)NodeASystem).Provider; + await provider.Transport.Shutdown(); + } + + /// + /// Waits until node B's own cluster view has the crashed node A removed (B is the sole Up member) AND + /// B has become the driver-role leader — i.e. the split-brain resolver downed the alone oldest + /// node and B took over. Queries (node A is dead after a hard kill, so its + /// view can't be read). Returns the elapsed time to takeover; throws if + /// B never becomes the sole driver-leader within (the failure signature of + /// an INACTIVE resolver — A stays Unreachable and B never fails over). + /// + /// Upper bound; must exceed acceptable-heartbeat-pause (10s) + stable-after (15s). + public async Task WaitForNodeBSoleDriverLeaderAsync(TimeSpan timeout) + { + var start = DateTime.UtcNow; + var deadline = start + timeout; + // Require the outcome to hold across several consecutive polls so a transient during A's + // Up -> Down convergence (the members list can briefly flap while gossip settles) can't be + // mistaken for a settled takeover. + const int requiredStableStreak = 4; + var streak = 0; + while (DateTime.UtcNow < deadline) + { + var clusterB = Akka.Cluster.Cluster.Get(NodeBSystem); + var state = clusterB.State; + var upCount = state.Members.Count(m => m.Status == MemberStatus.Up); + var driverLeader = state.RoleLeader("driver"); + // SBR proof: the crashed oldest node has been DOWNED (no longer Up, so B is the sole Up + // member) and B has become the driver-role leader. With the resolver INACTIVE (NoDowning) A + // would instead stay Up-but-Unreachable — B would still see 2 Up members and never gain a + // stable role leader. (A lingers as a Down member in the Unreachable set until reaped; that + // reaping is not part of the failover contract, so it is NOT required here.) + if (upCount == 1 && driverLeader == clusterB.SelfAddress) + { + if (++streak >= requiredStableStreak) + return DateTime.UtcNow - start; + } + else + { + streak = 0; + } + await Task.Delay(250); + } + var finalState = Akka.Cluster.Cluster.Get(NodeBSystem).State; + throw new TimeoutException( + $"Node B did not become the sole driver-role leader within {timeout} after node A was hard-killed. " + + $"Up={finalState.Members.Count(m => m.Status == MemberStatus.Up)}, " + + $"Unreachable={finalState.Unreachable.Count}, " + + $"driverLeader={finalState.RoleLeader("driver")}, selfB={Akka.Cluster.Cluster.Get(NodeBSystem).SelfAddress}. " + + "A stuck-Unreachable A with no takeover is the signature of an INACTIVE split-brain resolver."); + } + /// /// Waits for node A's cluster view to reach members in /// . Used for asserting shrink-after-stop or grow-after-restart. From eaf78aad904d0da6323671c6dae89fe24bd216a8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 22:34:20 -0400 Subject: [PATCH 3/3] =?UTF-8?q?docs(archreview=20#11):=20correct=20Critica?= =?UTF-8?q?l=201=20premise=20=E2=80=94=20SBR=20was=20already=20active=20on?= =?UTF-8?q?=20master,=20not=20NoDowning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Akka.Cluster.Hosting's WithClustering enables an SBR downing provider by default (applies SplitBrainResolverOption.Default when ClusterOptions.SplitBrainResolver is null), which reads the pre-existing akka.conf keep-oldest block. So the cluster was NOT running NoDowning before Critical 1 and hard-crash failover already worked — the typed KeepOldestOption is reinforcing/explicit-in-code, not the sole activator. Corrects the inaccurate 'HOCON inert / NoDowning / never fails over' framing in: - ServiceCollectionExtensions.BuildClusterOptions XML comment - akka.conf split-brain-resolver comment - docs/Redundancy.md Split-brain section - SplitBrainResolverActivationTests summary + assertion message (+ method rename) No code revert (the typed option is correct belt-and-suspenders). Cluster.Tests 29/29. Surfaced by the #9 hard-kill failover negative control. --- docs/Redundancy.md | 29 +++++++++++-------- .../Resources/akka.conf | 13 +++++---- .../ServiceCollectionExtensions.cs | 23 ++++++++------- .../SplitBrainResolverActivationTests.cs | 21 ++++++++------ 4 files changed, 49 insertions(+), 37 deletions(-) diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 7d78c1f9..bd58859a 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -154,23 +154,28 @@ Node A lists Node B's `ApplicationUri` and vice-versa. Validated by `DualEndpoin ## Split-brain -The split-brain resolver is **activated in code** by the typed `ClusterOptions.SplitBrainResolver` -(`KeepOldestOption { DownIfAlone = true }`) set in -`ServiceCollectionExtensions.BuildClusterOptions` — this is what registers -`Akka.Cluster.SBR.SplitBrainResolverProvider`. **Without that typed registration the `split-brain-resolver` -HOCON block in `akka.conf` is inert and the cluster runs Akka's default `NoDowning`**, meaning a -hard-crashed node is never downed and neither the cluster singletons nor the `driver` role-leader ever fail -over (a partition would leave both redundancy sides at ServiceLevel 240 indefinitely). The HOCON block is the -tuning source only: `active-strategy = keep-oldest`, `stable-after = 15s`, `keep-oldest.down-if-alone = on`, -`failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone` must stay consistent with the -typed option; `stable-after` lives only in HOCON because the typed option can't express it). +The split-brain resolver is **active by default**: Akka.Cluster.Hosting's `WithClustering` enables an SBR +downing provider whenever `ClusterOptions.SplitBrainResolver` is null (it applies +`SplitBrainResolverOption.Default`, which registers `Akka.Cluster.SBR.SplitBrainResolverProvider`), and that +provider reads the `split-brain-resolver` HOCON block in `akka.conf`. On top of that, +`ServiceCollectionExtensions.BuildClusterOptions` sets the typed `ClusterOptions.SplitBrainResolver` +(`KeepOldestOption { DownIfAlone = true }`) to make the strategy **explicit in code** rather than relying on +the framework default — it is reinforcing, not the sole activator, and yields the same effective behavior. So +the cluster is **not** running `NoDowning`; hard-crashed nodes are downed and both the cluster singletons and +the `driver` role-leader fail over. (Only an *explicit* `NoDowning`, e.g. +`akka.cluster.downing-provider-class = ""`, would leave both redundancy sides at ServiceLevel 240 +indefinitely.) The HOCON block carries the tuning: `active-strategy = keep-oldest`, `stable-after = 15s`, +`keep-oldest.down-if-alone = on`, `failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone` +must stay consistent with the typed option; `stable-after` lives only in HOCON because the typed option can't +express it). `keep-oldest` is the correct strategy for a 2-node warm-redundancy pair: under a clean partition the oldest member (typically the long-running primary) stays up and the smaller (or younger) side downs itself within ~`stable-after` seconds; `down-if-alone` downs a node that loses its only peer. `keep-majority`/`static-quorum` are wrong for two nodes (no majority in a 1-1 split). The `RedundancyStateActor` on the surviving partition -re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node now -triggers the same failover. +re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node +triggers the same failover — verified by `HardKillFailoverTests` (its negative control confirms failover +survives removing the typed option, and only breaks under an explicit `NoDowning`). There is no operator-driven role swap during a partition. Failover is what the cluster does automatically. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf index 4735fa89..806ac578 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf @@ -37,12 +37,13 @@ akka { roles = [] min-nr-of-members = 1 - # Split-brain resolver. This HOCON block is the TUNING source only — it does NOT - # activate the resolver. Activation happens in code via the typed - # ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) set in - # ServiceCollectionExtensions.BuildClusterOptions; without that the cluster runs Akka's - # default NoDowning and hard-crashed nodes never fail over (arch-review 03/S1). - # active-strategy + keep-oldest.down-if-alone below MUST match the typed option. + # Split-brain resolver (arch-review 03/S1). This block IS active: Akka.Cluster.Hosting's + # WithClustering enables an SBR downing provider by default (it applies + # SplitBrainResolverOption.Default when ClusterOptions.SplitBrainResolver is null), and that + # provider reads this block. ServiceCollectionExtensions.BuildClusterOptions additionally sets + # the typed KeepOldestOption { DownIfAlone = true } to make the strategy EXPLICIT in code — + # its active-strategy + keep-oldest.down-if-alone MUST match this block. (The cluster is NOT + # NoDowning; only an explicit downing-provider-class = "" would disable failover.) # stable-after must stay >= failure-detector.acceptable-heartbeat-pause (10s) + margin. split-brain-resolver { active-strategy = "keep-oldest" diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index 8d4f1fab..37d108c7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -84,16 +84,19 @@ public static class ServiceCollectionExtensions } /// - /// Builds the for the fused-host cluster, including the - /// activating split-brain-resolver registration. + /// Builds the for the fused-host cluster, setting the split-brain-resolver + /// strategy explicitly in code. /// - /// The split-brain-resolver HOCON block in Resources/akka.conf is inert on its own — - /// Akka.NET only runs the resolver when a downing provider is registered, otherwise the cluster - /// falls back to NoDowning and a hard-crashed node is never downed (singletons and the - /// driver role-leader never fail over, and a partition leaves both redundancy sides at - /// ServiceLevel 240 forever). Setting is what - /// activates Akka.Cluster.SBR.SplitBrainResolverProvider under the hood — this is the piece - /// that was missing (arch-review 03/S1). + /// Activation note (arch-review 03/S1, corrected): Akka.Cluster.Hosting's WithClustering + /// already enables an SBR downing provider by default — when + /// is null it applies SplitBrainResolverOption.Default, which registers + /// Akka.Cluster.SBR.SplitBrainResolverProvider and reads the split-brain-resolver HOCON block in + /// Resources/akka.conf. So the cluster was not running NoDowning before this option was + /// set: the pre-existing akka.conf keep-oldest block was already active, and hard-crashed nodes already + /// failed over. Setting this typed option makes the strategy explicit in code (independent of the + /// framework default) rather than being the sole activator — it is reinforcing/belt-and-suspenders and + /// produces the same effective behavior. (Only an explicit NoDowning, e.g. + /// downing-provider-class = "", would disable failover.) /// /// with DownIfAlone=true mirrors the HOCON intent and is the /// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically @@ -103,7 +106,7 @@ public static class ServiceCollectionExtensions /// typed option cannot express it (it must stay ≥ failure-detector.acceptable-heartbeat-pause). /// /// The bound cluster options carrying seed nodes and roles. - /// The cluster options with seed nodes, roles, and the activated split-brain resolver. + /// The cluster options with seed nodes, roles, and the explicit split-brain-resolver strategy. public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options) { return new ClusterOptions diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs index 9f404027..64c7e040 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs @@ -6,12 +6,15 @@ using Xunit; namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; /// -/// Guards the split-brain-resolver ACTIVATION (arch-review 03/S1). The split-brain-resolver -/// HOCON block is inert on its own; the resolver only runs when the typed -/// is set. These tests assert -/// registers it, so a future refactor -/// cannot silently drop back to Akka's default NoDowning (the exact "built-but-never-wired" failure -/// mode this finding described). +/// Guards that the split-brain-resolver strategy stays set explicitly in code (arch-review 03/S1, +/// premise corrected — see #11). Akka.Cluster.Hosting already enables an SBR downing provider by default +/// (it applies SplitBrainResolverOption.Default when +/// is null, which reads the akka.conf keep-oldest block), so the cluster is not NoDowning even +/// without the typed option. These tests assert +/// sets the typed +/// KeepOldestOption { DownIfAlone = true } so the strategy is explicit in code rather than relying on +/// the framework default — a future refactor cannot silently drop that explicitness. (Failover behaviour is +/// separately verified by HardKillFailoverTests, which passes regardless of activation path.) /// public sealed class SplitBrainResolverActivationTests { @@ -21,14 +24,14 @@ public sealed class SplitBrainResolverActivationTests Roles = new[] { "admin", "driver" }, }; - /// Verifies the cluster options carry a split-brain resolver (provider gets activated). + /// Verifies the cluster options carry an explicit split-brain resolver strategy. [Fact] - public void BuildClusterOptions_activates_a_split_brain_resolver() + public void BuildClusterOptions_sets_an_explicit_split_brain_resolver() { var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions()); options.SplitBrainResolver.ShouldNotBeNull( - "SplitBrainResolver must be set — without it the cluster runs default NoDowning and never fails over on a hard crash."); + "SplitBrainResolver must be set explicitly in code — keeps the strategy independent of Akka.Cluster.Hosting's default provider rather than relying on it."); } /// Verifies the resolver is keep-oldest with down-if-alone (correct for a 2-node pair).