From a81dea102c09d7ac274f14f60292b3931db30f73 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 16:14:26 -0400 Subject: [PATCH] fix(cluster): activate split-brain resolver (arch-review 03/S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split-brain-resolver HOCON block in akka.conf was inert — nothing registered a downing provider, so the cluster ran Akka's default NoDowning: a hard-crashed node was never downed, cluster singletons + the driver role-leader never failed over, and a partition left both redundancy sides at ServiceLevel 240 indefinitely. Activate the resolver via the typed ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) in a new testable BuildClusterOptions helper. keep-oldest + down-if-alone is correct for a 2-node warm-redundancy pair. HOCON keeps stable-after (typed option can't express it) + a cross-reference comment. Deterministic unit guard added; end-to-end verified the resolved ActorSystem config carries the provider and stable-after survives the HOCON/Hosting merge. Redundancy.md corrected. Hard-kill failover integration test tracked as a follow-up (effort-M harness). --- docs/Redundancy.md | 18 +++++- .../Resources/akka.conf | 7 +++ .../ServiceCollectionExtensions.cs | 38 +++++++++++-- .../SplitBrainResolverActivationTests.cs | 55 +++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs diff --git a/docs/Redundancy.md b/docs/Redundancy.md index 7a2252c5..7d78c1f9 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -154,7 +154,23 @@ Node A lists Node B's `ApplicationUri` and vice-versa. Validated by `DualEndpoin ## Split-brain -`akka.conf` configures Akka's split-brain resolver with `active-strategy = keep-oldest`, `stable-after = 15s`, and `failure-detector.threshold = 10.0`. Under a clean partition: the oldest member stays up + the smaller (or younger) side downs itself within ~15 seconds. The `RedundancyStateActor` on the surviving partition re-computes from the post-partition `Cluster.State`. +The split-brain resolver is **activated in code** by the typed `ClusterOptions.SplitBrainResolver` +(`KeepOldestOption { DownIfAlone = true }`) set in +`ServiceCollectionExtensions.BuildClusterOptions` — this is what registers +`Akka.Cluster.SBR.SplitBrainResolverProvider`. **Without that typed registration the `split-brain-resolver` +HOCON block in `akka.conf` is inert and the cluster runs Akka's default `NoDowning`**, meaning a +hard-crashed node is never downed and neither the cluster singletons nor the `driver` role-leader ever fail +over (a partition would leave both redundancy sides at ServiceLevel 240 indefinitely). The HOCON block is the +tuning source only: `active-strategy = keep-oldest`, `stable-after = 15s`, `keep-oldest.down-if-alone = on`, +`failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone` must stay consistent with the +typed option; `stable-after` lives only in HOCON because the typed option can't express it). + +`keep-oldest` is the correct strategy for a 2-node warm-redundancy pair: under a clean partition the oldest +member (typically the long-running primary) stays up and the smaller (or younger) side downs itself within +~`stable-after` seconds; `down-if-alone` downs a node that loses its only peer. `keep-majority`/`static-quorum` +are wrong for two nodes (no majority in a 1-1 split). The `RedundancyStateActor` on the surviving partition +re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node now +triggers the same failover. There is no operator-driven role swap during a partition. Failover is what the cluster does automatically. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf index d64cbf11..4735fa89 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf @@ -37,6 +37,13 @@ akka { roles = [] min-nr-of-members = 1 + # Split-brain resolver. This HOCON block is the TUNING source only — it does NOT + # activate the resolver. Activation happens in code via the typed + # ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) set in + # ServiceCollectionExtensions.BuildClusterOptions; without that the cluster runs Akka's + # default NoDowning and hard-crashed nodes never fail over (arch-review 03/S1). + # active-strategy + keep-oldest.down-if-alone below MUST match the typed option. + # stable-after must stay >= failure-detector.acceptable-heartbeat-pause (10s) + margin. split-brain-resolver { active-strategy = "keep-oldest" stable-after = 15s diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index 2d4fa44c..8d4f1fab 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Akka.Cluster.Hosting; +using Akka.Cluster.Hosting.SBR; using Akka.Event; using Akka.Hosting; using Akka.Logger.Serilog; @@ -77,12 +78,39 @@ public static class ServiceCollectionExtensions PublicHostName = options.PublicHostname, }); - builder.WithClustering(new ClusterOptions - { - SeedNodes = options.SeedNodes, - Roles = options.Roles, - }); + builder.WithClustering(BuildClusterOptions(options)); return builder; } + + /// + /// Builds the for the fused-host cluster, including the + /// activating split-brain-resolver registration. + /// + /// The split-brain-resolver HOCON block in Resources/akka.conf is inert on its own — + /// Akka.NET only runs the resolver when a downing provider is registered, otherwise the cluster + /// falls back to NoDowning and a hard-crashed node is never downed (singletons and the + /// driver role-leader never fail over, and a partition leaves both redundancy sides at + /// ServiceLevel 240 forever). Setting is what + /// activates Akka.Cluster.SBR.SplitBrainResolverProvider under the hood — this is the piece + /// that was missing (arch-review 03/S1). + /// + /// with DownIfAlone=true mirrors the HOCON intent and is the + /// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically + /// the long-running primary) survives, and down-if-alone downs a singleton that loses its + /// peer. keep-majority/static-quorum are wrong for two nodes. The strategy + down-if-alone + /// here MUST stay consistent with the HOCON block; stable-after lives only in HOCON because the + /// typed option cannot express it (it must stay ≥ failure-detector.acceptable-heartbeat-pause). + /// + /// The bound cluster options carrying seed nodes and roles. + /// The cluster options with seed nodes, roles, and the activated split-brain resolver. + public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options) + { + return new ClusterOptions + { + SeedNodes = options.SeedNodes, + Roles = options.Roles, + SplitBrainResolver = new KeepOldestOption { DownIfAlone = true }, + }; + } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs new file mode 100644 index 00000000..9f404027 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SplitBrainResolverActivationTests.cs @@ -0,0 +1,55 @@ +using Akka.Cluster.Hosting; +using Akka.Cluster.Hosting.SBR; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Guards the split-brain-resolver ACTIVATION (arch-review 03/S1). The split-brain-resolver +/// HOCON block is inert on its own; the resolver only runs when the typed +/// is set. These tests assert +/// registers it, so a future refactor +/// cannot silently drop back to Akka's default NoDowning (the exact "built-but-never-wired" failure +/// mode this finding described). +/// +public sealed class SplitBrainResolverActivationTests +{ + private static AkkaClusterOptions SampleOptions() => new() + { + SeedNodes = new[] { "akka.tcp://otopcua@127.0.0.1:4053" }, + Roles = new[] { "admin", "driver" }, + }; + + /// Verifies the cluster options carry a split-brain resolver (provider gets activated). + [Fact] + public void BuildClusterOptions_activates_a_split_brain_resolver() + { + var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions()); + + options.SplitBrainResolver.ShouldNotBeNull( + "SplitBrainResolver must be set — without it the cluster runs default NoDowning and never fails over on a hard crash."); + } + + /// Verifies the resolver is keep-oldest with down-if-alone (correct for a 2-node pair). + [Fact] + public void BuildClusterOptions_uses_keep_oldest_with_down_if_alone() + { + var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions()); + + var keepOldest = options.SplitBrainResolver.ShouldBeOfType(); + keepOldest.DownIfAlone.ShouldBe(true); + } + + /// Verifies seed nodes and roles are still threaded through unchanged. + [Fact] + public void BuildClusterOptions_preserves_seed_nodes_and_roles() + { + var input = SampleOptions(); + + var options = ServiceCollectionExtensions.BuildClusterOptions(input); + + options.SeedNodes.ShouldBe(input.SeedNodes); + options.Roles.ShouldBe(input.Roles); + } +}