fix(cluster): activate split-brain resolver (arch-review 03/S1)
The split-brain-resolver HOCON block in akka.conf was inert — nothing
registered a downing provider, so the cluster ran Akka's default NoDowning:
a hard-crashed node was never downed, cluster singletons + the driver
role-leader never failed over, and a partition left both redundancy sides
at ServiceLevel 240 indefinitely.
Activate the resolver via the typed ClusterOptions.SplitBrainResolver
(KeepOldestOption { DownIfAlone = true }) in a new testable
BuildClusterOptions helper. keep-oldest + down-if-alone is correct for a
2-node warm-redundancy pair. HOCON keeps stable-after (typed option can't
express it) + a cross-reference comment. Deterministic unit guard added;
end-to-end verified the resolved ActorSystem config carries the provider
and stable-after survives the HOCON/Hosting merge. Redundancy.md corrected.
Hard-kill failover integration test tracked as a follow-up (effort-M harness).
This commit is contained in:
@@ -37,6 +37,13 @@ akka {
|
||||
roles = []
|
||||
min-nr-of-members = 1
|
||||
|
||||
# Split-brain resolver. This HOCON block is the TUNING source only — it does NOT
|
||||
# activate the resolver. Activation happens in code via the typed
|
||||
# ClusterOptions.SplitBrainResolver (KeepOldestOption { DownIfAlone = true }) set in
|
||||
# ServiceCollectionExtensions.BuildClusterOptions; without that the cluster runs Akka's
|
||||
# default NoDowning and hard-crashed nodes never fail over (arch-review 03/S1).
|
||||
# active-strategy + keep-oldest.down-if-alone below MUST match the typed option.
|
||||
# stable-after must stay >= failure-detector.acceptable-heartbeat-pause (10s) + margin.
|
||||
split-brain-resolver {
|
||||
active-strategy = "keep-oldest"
|
||||
stable-after = 15s
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using Akka.Cluster.Hosting;
|
||||
using Akka.Cluster.Hosting.SBR;
|
||||
using Akka.Event;
|
||||
using Akka.Hosting;
|
||||
using Akka.Logger.Serilog;
|
||||
@@ -77,12 +78,39 @@ public static class ServiceCollectionExtensions
|
||||
PublicHostName = options.PublicHostname,
|
||||
});
|
||||
|
||||
builder.WithClustering(new ClusterOptions
|
||||
{
|
||||
SeedNodes = options.SeedNodes,
|
||||
Roles = options.Roles,
|
||||
});
|
||||
builder.WithClustering(BuildClusterOptions(options));
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <see cref="ClusterOptions"/> for the fused-host cluster, including the
|
||||
/// <b>activating</b> split-brain-resolver registration.
|
||||
///
|
||||
/// The <c>split-brain-resolver</c> HOCON block in <c>Resources/akka.conf</c> is inert on its own —
|
||||
/// Akka.NET only runs the resolver when a downing provider is registered, otherwise the cluster
|
||||
/// falls back to <b>NoDowning</b> and a hard-crashed node is never downed (singletons and the
|
||||
/// <c>driver</c> role-leader never fail over, and a partition leaves both redundancy sides at
|
||||
/// ServiceLevel 240 forever). Setting <see cref="ClusterOptions.SplitBrainResolver"/> is what
|
||||
/// activates <c>Akka.Cluster.SBR.SplitBrainResolverProvider</c> under the hood — this is the piece
|
||||
/// that was missing (arch-review 03/S1).
|
||||
///
|
||||
/// <see cref="KeepOldestOption"/> with <c>DownIfAlone=true</c> mirrors the HOCON intent and is the
|
||||
/// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically
|
||||
/// the long-running primary) survives, and <c>down-if-alone</c> downs a singleton that loses its
|
||||
/// peer. <c>keep-majority</c>/<c>static-quorum</c> are wrong for two nodes. The strategy + down-if-alone
|
||||
/// here MUST stay consistent with the HOCON block; <c>stable-after</c> lives only in HOCON because the
|
||||
/// typed option cannot express it (it must stay ≥ <c>failure-detector.acceptable-heartbeat-pause</c>).
|
||||
/// </summary>
|
||||
/// <param name="options">The bound cluster options carrying seed nodes and roles.</param>
|
||||
/// <returns>The cluster options with seed nodes, roles, and the activated split-brain resolver.</returns>
|
||||
public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options)
|
||||
{
|
||||
return new ClusterOptions
|
||||
{
|
||||
SeedNodes = options.SeedNodes,
|
||||
Roles = options.Roles,
|
||||
SplitBrainResolver = new KeepOldestOption { DownIfAlone = true },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user