feat(cluster): auto-down downing strategy — either-node crash now fails over (owner decision 2026-07-21: availability over partition-safety)
Two-node keep-oldest could NEVER survive a crash of the oldest/active node:
Akka.NET 1.5.62 KeepOldest.OldestDecision only lets down-if-alone rescue a
side with >= 2 members, so the 1-vs-1 survivor takes DownReachable and downs
ITSELF — proven live on the rig ('SBR took decision ... including myself')
before this change. static-quorum(1) is worse (IsTooManyMembers -> DownAll);
keep-majority just re-keys the fatal crash to the lowest address.
SplitBrainResolverStrategy gains 'auto-down' (new default): BuildHocon emits
Akka's AutoDowning provider with auto-down-unreachable-after = StableAfter.
The leader among the REACHABLE members downs the unreachable peer, so the
survivor takes over singletons and /health/active in ~25s regardless of which
node died. Accepted trade (explicit owner decision): a real network partition
runs dual-active until an operator restarts one side. keep-oldest remains
supported; DownIfAlone validation is now scoped to it.
Live drill on the rebuilt rig: active-crash TAKEOVER in 28s (victim still
down; all 7 singletons Younger->Oldest), standby-crash removal 27s with 0
routing blips; victims rejoin as standby in 2s. New real-cluster tests pin
both directions (SbrFailoverTests.AutoDown_*); TwoNodeClusterFixture gains a
strategy knob. All 16 appsettings flipped (src, docker, docker-env2, and the
gitignored wonder-app-vd03 overlay on disk — owner must sync to the host).
Docs: decision record docs/plans/2026-07-21-auto-down-availability-decision.md,
Component-ClusterInfrastructure downing section rewritten, drill + README
reworked (active mode now asserts takeover), deferred-work SBR row resolved.
This commit is contained in:
@@ -200,8 +200,9 @@ public class AkkaHostedService : IHostedService
|
||||
_communicationOptions.TransportHeartbeatInterval.TotalSeconds,
|
||||
_communicationOptions.TransportFailureThreshold.TotalSeconds);
|
||||
|
||||
// Down-if-alone recovery watchdog: SBR's keep-oldest down-if-alone plus
|
||||
// run-coordinated-shutdown-when-down means a self-downed node terminates
|
||||
// Downed-node recovery watchdog: any downing decision against this node
|
||||
// (auto-down by the peer, or an SBR self-down under keep-oldest) plus
|
||||
// run-coordinated-shutdown-when-down means a downed node terminates
|
||||
// its own ActorSystem. If that happens outside our StopAsync, the Host
|
||||
// process must exit so the service supervisor (docker
|
||||
// `restart: unless-stopped` / Windows service recovery) restarts it and
|
||||
@@ -228,14 +229,21 @@ public class AkkaHostedService : IHostedService
|
||||
/// seed-node URI, role or split-brain strategy containing a quote, backslash or
|
||||
/// whitespace cannot corrupt the document or be silently misparsed.
|
||||
///
|
||||
/// The <c>keep-oldest down-if-alone</c> flag is emitted from
|
||||
/// The downing block branches on <see cref="ClusterOptions.SplitBrainResolverStrategy"/>:
|
||||
/// <c>auto-down</c> (default; decision 2026-07-21) installs Akka's
|
||||
/// <c>AutoDowning</c> provider with <c>auto-down-unreachable-after</c> =
|
||||
/// <see cref="ClusterOptions.StableAfter"/> — the leader among the REACHABLE members
|
||||
/// downs the unreachable peer, so a crash of either node (oldest included) fails
|
||||
/// over to the survivor; the accepted trade is dual-active during a real network
|
||||
/// partition. Any other value takes the SBR path, where the
|
||||
/// <c>keep-oldest down-if-alone</c> flag is emitted from
|
||||
/// <see cref="ClusterOptions.DownIfAlone"/> rather than hard-coded, so the bound
|
||||
/// configuration value is actually consumed.
|
||||
///
|
||||
/// The split-brain-resolver <c>downing-provider-class</c> is installed
|
||||
/// explicitly: Akka defaults to <c>NoDowning</c>, under which the entire
|
||||
/// split-brain-resolver section is inert and singletons never migrate on a hard
|
||||
/// crash or partition. Naming the SBR provider is what activates automatic downing.
|
||||
/// A <c>downing-provider-class</c> is always installed explicitly: Akka defaults
|
||||
/// to <c>NoDowning</c>, under which the downing configuration is inert and
|
||||
/// singletons never migrate on a hard crash or partition. Naming the provider is
|
||||
/// what activates automatic downing.
|
||||
///
|
||||
/// Every duration is rendered via <see cref="DurationHocon"/> in
|
||||
/// milliseconds, so sub-second cluster timing values (e.g. a 750ms heartbeat) are
|
||||
@@ -258,6 +266,25 @@ public class AkkaHostedService : IHostedService
|
||||
clusterOptions.SeedNodes.Select(QuoteHocon));
|
||||
var rolesStr = string.Join(",", roles.Select(QuoteHocon));
|
||||
|
||||
// auto-down (default): AutoDowning provider — the leader among the reachable
|
||||
// members downs the unreachable peer after StableAfter, so a crash of EITHER
|
||||
// node fails over to the survivor (dual-active during a real partition is the
|
||||
// accepted trade — decision 2026-07-21). Anything else: the SBR provider with
|
||||
// the configured active-strategy (keep-oldest), which is partition-safe but
|
||||
// cannot survive a crash of the oldest node in a two-node cluster.
|
||||
var downingBlock = string.Equals(
|
||||
clusterOptions.SplitBrainResolverStrategy, "auto-down", StringComparison.OrdinalIgnoreCase)
|
||||
? $@"downing-provider-class = ""Akka.Cluster.AutoDowning, Akka.Cluster""
|
||||
auto-down-unreachable-after = {DurationHocon(clusterOptions.StableAfter)}"
|
||||
: $@"downing-provider-class = ""Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster""
|
||||
split-brain-resolver {{
|
||||
active-strategy = {QuoteHocon(clusterOptions.SplitBrainResolverStrategy)}
|
||||
stable-after = {DurationHocon(clusterOptions.StableAfter)}
|
||||
keep-oldest {{
|
||||
down-if-alone = {(clusterOptions.DownIfAlone ? "on" : "off")}
|
||||
}}
|
||||
}}";
|
||||
|
||||
return $@"
|
||||
audit-telemetry-dispatcher {{
|
||||
type = ForkJoinDispatcher
|
||||
@@ -287,14 +314,7 @@ akka {{
|
||||
seed-nodes = [{seedNodesStr}]
|
||||
roles = [{rolesStr}]
|
||||
min-nr-of-members = {clusterOptions.MinNrOfMembers}
|
||||
downing-provider-class = ""Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster""
|
||||
split-brain-resolver {{
|
||||
active-strategy = {QuoteHocon(clusterOptions.SplitBrainResolverStrategy)}
|
||||
stable-after = {DurationHocon(clusterOptions.StableAfter)}
|
||||
keep-oldest {{
|
||||
down-if-alone = {(clusterOptions.DownIfAlone ? "on" : "off")}
|
||||
}}
|
||||
}}
|
||||
{downingBlock}
|
||||
failure-detector {{
|
||||
heartbeat-interval = {DurationHocon(clusterOptions.HeartbeatInterval)}
|
||||
acceptable-heartbeat-pause = {DurationHocon(clusterOptions.FailureDetectionThreshold)}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"akka.tcp://scadabridge@localhost:8081",
|
||||
"akka.tcp://scadabridge@localhost:8082"
|
||||
],
|
||||
"SplitBrainResolverStrategy": "keep-oldest",
|
||||
"SplitBrainResolverStrategy": "auto-down",
|
||||
"StableAfter": "00:00:15",
|
||||
"HeartbeatInterval": "00:00:02",
|
||||
"FailureDetectionThreshold": "00:00:10",
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"akka.tcp://scadabridge@localhost:8082",
|
||||
"akka.tcp://scadabridge@localhost:8085"
|
||||
],
|
||||
"SplitBrainResolverStrategy": "keep-oldest",
|
||||
"SplitBrainResolverStrategy": "auto-down",
|
||||
"StableAfter": "00:00:15",
|
||||
"HeartbeatInterval": "00:00:02",
|
||||
"FailureDetectionThreshold": "00:00:10",
|
||||
|
||||
Reference in New Issue
Block a user