Merge branch 'fix/archreview-crit1-split-brain-resolver'
This commit is contained in:
Joseph Doherty
+22
-1
@@ -154,7 +154,28 @@ Node A lists Node B's `ApplicationUri` and vice-versa. Validated by `DualEndpoin
|
||||
|
||||
## Split-brain
|
||||
|
||||
`akka.conf` configures Akka's split-brain resolver with `active-strategy = keep-oldest`, `stable-after = 15s`, and `failure-detector.threshold = 10.0`. Under a clean partition: the oldest member stays up + the smaller (or younger) side downs itself within ~15 seconds. The `RedundancyStateActor` on the surviving partition re-computes from the post-partition `Cluster.State`.
|
||||
The split-brain resolver is **active by default**: Akka.Cluster.Hosting's `WithClustering` enables an SBR
|
||||
downing provider whenever `ClusterOptions.SplitBrainResolver` is null (it applies
|
||||
`SplitBrainResolverOption.Default`, which registers `Akka.Cluster.SBR.SplitBrainResolverProvider`), and that
|
||||
provider reads the `split-brain-resolver` HOCON block in `akka.conf`. On top of that,
|
||||
`ServiceCollectionExtensions.BuildClusterOptions` sets the typed `ClusterOptions.SplitBrainResolver`
|
||||
(`KeepOldestOption { DownIfAlone = true }`) to make the strategy **explicit in code** rather than relying on
|
||||
the framework default — it is reinforcing, not the sole activator, and yields the same effective behavior. So
|
||||
the cluster is **not** running `NoDowning`; hard-crashed nodes are downed and both the cluster singletons and
|
||||
the `driver` role-leader fail over. (Only an *explicit* `NoDowning`, e.g.
|
||||
`akka.cluster.downing-provider-class = ""`, would leave both redundancy sides at ServiceLevel 240
|
||||
indefinitely.) The HOCON block carries the tuning: `active-strategy = keep-oldest`, `stable-after = 15s`,
|
||||
`keep-oldest.down-if-alone = on`, `failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone`
|
||||
must stay consistent with the typed option; `stable-after` lives only in HOCON because the typed option can't
|
||||
express it).
|
||||
|
||||
`keep-oldest` is the correct strategy for a 2-node warm-redundancy pair: under a clean partition the oldest
|
||||
member (typically the long-running primary) stays up and the smaller (or younger) side downs itself within
|
||||
~`stable-after` seconds; `down-if-alone` downs a node that loses its only peer. `keep-majority`/`static-quorum`
|
||||
are wrong for two nodes (no majority in a 1-1 split). The `RedundancyStateActor` on the surviving partition
|
||||
re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node
|
||||
triggers the same failover — verified by `HardKillFailoverTests` (its negative control confirms failover
|
||||
survives removing the typed option, and only breaks under an explicit `NoDowning`).
|
||||
|
||||
There is no operator-driven role swap during a partition. Failover is what the cluster does automatically.
|
||||
|
||||
|
||||
@@ -37,6 +37,14 @@ akka {
|
||||
roles = []
|
||||
min-nr-of-members = 1
|
||||
|
||||
# Split-brain resolver (arch-review 03/S1). This block IS active: Akka.Cluster.Hosting's
|
||||
# WithClustering enables an SBR downing provider by default (it applies
|
||||
# SplitBrainResolverOption.Default when ClusterOptions.SplitBrainResolver is null), and that
|
||||
# provider reads this block. ServiceCollectionExtensions.BuildClusterOptions additionally sets
|
||||
# the typed KeepOldestOption { DownIfAlone = true } to make the strategy EXPLICIT in code —
|
||||
# its active-strategy + keep-oldest.down-if-alone MUST match this block. (The cluster is NOT
|
||||
# NoDowning; only an explicit downing-provider-class = "" would disable failover.)
|
||||
# stable-after must stay >= failure-detector.acceptable-heartbeat-pause (10s) + margin.
|
||||
split-brain-resolver {
|
||||
active-strategy = "keep-oldest"
|
||||
stable-after = 15s
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using Akka.Cluster.Hosting;
|
||||
using Akka.Cluster.Hosting.SBR;
|
||||
using Akka.Event;
|
||||
using Akka.Hosting;
|
||||
using Akka.Logger.Serilog;
|
||||
@@ -77,12 +78,42 @@ public static class ServiceCollectionExtensions
|
||||
PublicHostName = options.PublicHostname,
|
||||
});
|
||||
|
||||
builder.WithClustering(new ClusterOptions
|
||||
{
|
||||
SeedNodes = options.SeedNodes,
|
||||
Roles = options.Roles,
|
||||
});
|
||||
builder.WithClustering(BuildClusterOptions(options));
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <see cref="ClusterOptions"/> for the fused-host cluster, setting the split-brain-resolver
|
||||
/// strategy <b>explicitly in code</b>.
|
||||
///
|
||||
/// <b>Activation note (arch-review 03/S1, corrected):</b> Akka.Cluster.Hosting's <c>WithClustering</c>
|
||||
/// already enables an SBR downing provider <b>by default</b> — when <see cref="ClusterOptions.SplitBrainResolver"/>
|
||||
/// is <c>null</c> it applies <c>SplitBrainResolverOption.Default</c>, which registers
|
||||
/// <c>Akka.Cluster.SBR.SplitBrainResolverProvider</c> and reads the <c>split-brain-resolver</c> HOCON block in
|
||||
/// <c>Resources/akka.conf</c>. So the cluster was <b>not</b> running <c>NoDowning</c> before this option was
|
||||
/// set: the pre-existing akka.conf <c>keep-oldest</c> block was already active, and hard-crashed nodes already
|
||||
/// failed over. Setting this typed option makes the strategy <b>explicit in code</b> (independent of the
|
||||
/// framework default) rather than being the sole activator — it is reinforcing/belt-and-suspenders and
|
||||
/// produces the same effective behavior. (Only an <i>explicit</i> <c>NoDowning</c>, e.g.
|
||||
/// <c>downing-provider-class = ""</c>, would disable failover.)
|
||||
///
|
||||
/// <see cref="KeepOldestOption"/> with <c>DownIfAlone=true</c> mirrors the HOCON intent and is the
|
||||
/// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically
|
||||
/// the long-running primary) survives, and <c>down-if-alone</c> downs a singleton that loses its
|
||||
/// peer. <c>keep-majority</c>/<c>static-quorum</c> are wrong for two nodes. The strategy + down-if-alone
|
||||
/// here MUST stay consistent with the HOCON block; <c>stable-after</c> lives only in HOCON because the
|
||||
/// typed option cannot express it (it must stay ≥ <c>failure-detector.acceptable-heartbeat-pause</c>).
|
||||
/// </summary>
|
||||
/// <param name="options">The bound cluster options carrying seed nodes and roles.</param>
|
||||
/// <returns>The cluster options with seed nodes, roles, and the explicit split-brain-resolver strategy.</returns>
|
||||
public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options)
|
||||
{
|
||||
return new ClusterOptions
|
||||
{
|
||||
SeedNodes = options.SeedNodes,
|
||||
Roles = options.Roles,
|
||||
SplitBrainResolver = new KeepOldestOption { DownIfAlone = true },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
using Akka.Cluster.Hosting;
|
||||
using Akka.Cluster.Hosting.SBR;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Guards that the split-brain-resolver strategy stays set <b>explicitly in code</b> (arch-review 03/S1,
|
||||
/// premise corrected — see #11). Akka.Cluster.Hosting already enables an SBR downing provider by default
|
||||
/// (it applies <c>SplitBrainResolverOption.Default</c> when <see cref="ClusterOptions.SplitBrainResolver"/>
|
||||
/// is null, which reads the akka.conf <c>keep-oldest</c> block), so the cluster is not NoDowning even
|
||||
/// without the typed option. These tests assert
|
||||
/// <see cref="ServiceCollectionExtensions.BuildClusterOptions"/> sets the typed
|
||||
/// <c>KeepOldestOption { DownIfAlone = true }</c> so the strategy is explicit in code rather than relying on
|
||||
/// the framework default — a future refactor cannot silently drop that explicitness. (Failover behaviour is
|
||||
/// separately verified by <c>HardKillFailoverTests</c>, which passes regardless of activation path.)
|
||||
/// </summary>
|
||||
public sealed class SplitBrainResolverActivationTests
|
||||
{
|
||||
private static AkkaClusterOptions SampleOptions() => new()
|
||||
{
|
||||
SeedNodes = new[] { "akka.tcp://otopcua@127.0.0.1:4053" },
|
||||
Roles = new[] { "admin", "driver" },
|
||||
};
|
||||
|
||||
/// <summary>Verifies the cluster options carry an explicit split-brain resolver strategy.</summary>
|
||||
[Fact]
|
||||
public void BuildClusterOptions_sets_an_explicit_split_brain_resolver()
|
||||
{
|
||||
var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions());
|
||||
|
||||
options.SplitBrainResolver.ShouldNotBeNull(
|
||||
"SplitBrainResolver must be set explicitly in code — keeps the strategy independent of Akka.Cluster.Hosting's default provider rather than relying on it.");
|
||||
}
|
||||
|
||||
/// <summary>Verifies the resolver is keep-oldest with down-if-alone (correct for a 2-node pair).</summary>
|
||||
[Fact]
|
||||
public void BuildClusterOptions_uses_keep_oldest_with_down_if_alone()
|
||||
{
|
||||
var options = ServiceCollectionExtensions.BuildClusterOptions(SampleOptions());
|
||||
|
||||
var keepOldest = options.SplitBrainResolver.ShouldBeOfType<KeepOldestOption>();
|
||||
keepOldest.DownIfAlone.ShouldBe(true);
|
||||
}
|
||||
|
||||
/// <summary>Verifies seed nodes and roles are still threaded through unchanged.</summary>
|
||||
[Fact]
|
||||
public void BuildClusterOptions_preserves_seed_nodes_and_roles()
|
||||
{
|
||||
var input = SampleOptions();
|
||||
|
||||
var options = ServiceCollectionExtensions.BuildClusterOptions(input);
|
||||
|
||||
options.SeedNodes.ShouldBe(input.SeedNodes);
|
||||
options.Roles.ShouldBe(input.Roles);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using Akka.Cluster;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// The load-bearing integration half of arch-review Critical 1 (03/S1) — proves the cluster actually
|
||||
/// FAILS OVER when the oldest node hard-crashes: the split-brain resolver downs the alone crashed node
|
||||
/// and the survivor takes over as <c>driver</c>-role leader. The unit guard only asserts the resolver is
|
||||
/// wired; only a real hard-kill proves failover happens.
|
||||
///
|
||||
/// <para>The distinction the graceful <see cref="TwoNodeClusterHarness.StopNodeBAsync"/> path can NOT
|
||||
/// exercise: a graceful <c>Cluster.Leave</c> removes a member with no downing decision at all, so it
|
||||
/// passes even with downing disabled. A hard KILL leaves the survivor staring at an <b>Unreachable</b>
|
||||
/// member — and with no active downing provider that member stays Unreachable forever, never failing
|
||||
/// over. Only an active resolver downs the alone oldest node and lets the survivor take over.</para>
|
||||
///
|
||||
/// <para><b>What the negative control established (2026-07-08):</b> removing the typed
|
||||
/// <c>ClusterOptions.SplitBrainResolver</c> did NOT break this test — Akka.Cluster.Hosting's
|
||||
/// <c>WithClustering</c> applies <c>SplitBrainResolverOption.Default</c> when the option is null, which
|
||||
/// enables the SBR downing provider that then reads the <c>akka.conf</c> <c>keep-oldest</c> block (present
|
||||
/// on master before Critical 1). So the resolver is active via the framework default + HOCON; the typed
|
||||
/// option is reinforcing, not the sole activator. This test was verified to genuinely require an active
|
||||
/// downing provider by a separate run forcing explicit NoDowning
|
||||
/// (<c>akka.cluster.downing-provider-class = ""</c>), under which it correctly times out with the node
|
||||
/// stuck Up-but-Unreachable. It therefore guards the failover OUTCOME regardless of activation path.</para>
|
||||
///
|
||||
/// <para>Traited <c>Failover</c> — it is deliberately slow (~acceptable-heartbeat-pause 10s +
|
||||
/// stable-after 15s before the resolver acts).</para>
|
||||
/// </summary>
|
||||
[Trait("Category", "Failover")]
|
||||
public sealed class HardKillFailoverTests
|
||||
{
|
||||
// acceptable-heartbeat-pause (10s) + stable-after (15s) + convergence/takeover margin.
|
||||
private static readonly TimeSpan FailoverTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// Hard-killing the oldest node (abrupt ActorSystem terminate, no graceful Leave) must leave the
|
||||
/// survivor as the sole <c>driver</c>-role leader — proving the split-brain resolver downed the alone
|
||||
/// oldest node and failed over. Without an active resolver the survivor keeps A Unreachable and this
|
||||
/// times out.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Hard_kill_of_oldest_node_fails_over_to_survivor_via_split_brain_resolver()
|
||||
{
|
||||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||||
|
||||
// Precondition: a converged 2-member cluster from node B's own view.
|
||||
var clusterB = Akka.Cluster.Cluster.Get(harness.NodeBSystem);
|
||||
clusterB.State.Members.Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
|
||||
|
||||
// Crash the seed / oldest member with no graceful Leave — B now sees it Unreachable.
|
||||
await harness.HardKillNodeAAsync();
|
||||
|
||||
// Only an ACTIVE split-brain resolver turns that Unreachable into a downing + takeover. This
|
||||
// waits for the settled outcome — B as the sole Up member AND driver-role leader, held stable
|
||||
// across consecutive polls — and THROWS if it never happens (the signature of an inactive
|
||||
// resolver: A stuck Up-but-Unreachable, no takeover). A clean return IS the SBR proof.
|
||||
var elapsed = await harness.WaitForNodeBSoleDriverLeaderAsync(FailoverTimeout);
|
||||
elapsed.ShouldBeLessThan(FailoverTimeout);
|
||||
|
||||
TestContext.Current.SendDiagnosticMessage($"Failover completed in {elapsed.TotalSeconds:F1}s");
|
||||
}
|
||||
}
|
||||
@@ -175,6 +175,74 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable
|
||||
formationTimeout ?? TimeSpan.FromSeconds(20));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Abruptly crashes node A (the seed / oldest member) by shutting down its remoting transport — the
|
||||
/// canonical in-process crash simulation. All associations drop and heartbeats/gossip stop instantly
|
||||
/// with NO graceful <c>Cluster.Leave</c> (the transport is dead, so A cannot gossip anything). Node B's
|
||||
/// failure detector therefore marks A <b>Unreachable</b> (not Removed), which is the only state the
|
||||
/// split-brain resolver acts on: with an active resolver (keep-oldest + down-if-alone, active here via
|
||||
/// both the akka.conf block and Akka.Cluster.Hosting's default downing provider) B downs the alone
|
||||
/// oldest node and takes over; with downing disabled (explicit NoDowning) B keeps A Unreachable forever
|
||||
/// and never fails over. This is the crash the graceful <see cref="StopNodeBAsync"/> path (a real
|
||||
/// <c>Cluster.Leave</c>, which removes a member with no downing decision) can NOT simulate.
|
||||
/// The ActorSystem itself stays alive with a dead transport until harness teardown disposes it.
|
||||
/// </summary>
|
||||
public async Task HardKillNodeAAsync()
|
||||
{
|
||||
if (NodeA is null) return;
|
||||
var provider = (Akka.Remote.IRemoteActorRefProvider)((ExtendedActorSystem)NodeASystem).Provider;
|
||||
await provider.Transport.Shutdown();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits until node B's own cluster view has the crashed node A removed (B is the sole Up member) AND
|
||||
/// B has become the <c>driver</c>-role leader — i.e. the split-brain resolver downed the alone oldest
|
||||
/// node and B took over. Queries <see cref="NodeBSystem"/> (node A is dead after a hard kill, so its
|
||||
/// view can't be read). Returns the elapsed time to takeover; throws <see cref="TimeoutException"/> if
|
||||
/// B never becomes the sole driver-leader within <paramref name="timeout"/> (the failure signature of
|
||||
/// an INACTIVE resolver — A stays Unreachable and B never fails over).
|
||||
/// </summary>
|
||||
/// <param name="timeout">Upper bound; must exceed acceptable-heartbeat-pause (10s) + stable-after (15s).</param>
|
||||
public async Task<TimeSpan> WaitForNodeBSoleDriverLeaderAsync(TimeSpan timeout)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var deadline = start + timeout;
|
||||
// Require the outcome to hold across several consecutive polls so a transient during A's
|
||||
// Up -> Down convergence (the members list can briefly flap while gossip settles) can't be
|
||||
// mistaken for a settled takeover.
|
||||
const int requiredStableStreak = 4;
|
||||
var streak = 0;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var clusterB = Akka.Cluster.Cluster.Get(NodeBSystem);
|
||||
var state = clusterB.State;
|
||||
var upCount = state.Members.Count(m => m.Status == MemberStatus.Up);
|
||||
var driverLeader = state.RoleLeader("driver");
|
||||
// SBR proof: the crashed oldest node has been DOWNED (no longer Up, so B is the sole Up
|
||||
// member) and B has become the driver-role leader. With the resolver INACTIVE (NoDowning) A
|
||||
// would instead stay Up-but-Unreachable — B would still see 2 Up members and never gain a
|
||||
// stable role leader. (A lingers as a Down member in the Unreachable set until reaped; that
|
||||
// reaping is not part of the failover contract, so it is NOT required here.)
|
||||
if (upCount == 1 && driverLeader == clusterB.SelfAddress)
|
||||
{
|
||||
if (++streak >= requiredStableStreak)
|
||||
return DateTime.UtcNow - start;
|
||||
}
|
||||
else
|
||||
{
|
||||
streak = 0;
|
||||
}
|
||||
await Task.Delay(250);
|
||||
}
|
||||
var finalState = Akka.Cluster.Cluster.Get(NodeBSystem).State;
|
||||
throw new TimeoutException(
|
||||
$"Node B did not become the sole driver-role leader within {timeout} after node A was hard-killed. " +
|
||||
$"Up={finalState.Members.Count(m => m.Status == MemberStatus.Up)}, " +
|
||||
$"Unreachable={finalState.Unreachable.Count}, " +
|
||||
$"driverLeader={finalState.RoleLeader("driver")}, selfB={Akka.Cluster.Cluster.Get(NodeBSystem).SelfAddress}. " +
|
||||
"A stuck-Unreachable A with no takeover is the signature of an INACTIVE split-brain resolver.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits for node A's cluster view to reach <paramref name="expectedUpMembers"/> members in
|
||||
/// <see cref="MemberStatus.Up"/>. Used for asserting shrink-after-stop or grow-after-restart.
|
||||
|
||||
Reference in New Issue
Block a user