Joseph Doherty
cf3bd52f93
Two-node keep-oldest could NEVER survive a crash of the oldest/active node:
Akka.NET 1.5.62 KeepOldest.OldestDecision only lets down-if-alone rescue a
side with >= 2 members, so the 1-vs-1 survivor takes DownReachable and downs
ITSELF — proven live on the rig ('SBR took decision ... including myself')
before this change. static-quorum(1) is worse (IsTooManyMembers -> DownAll);
keep-majority just re-keys the fatal crash to the lowest address.
SplitBrainResolverStrategy gains 'auto-down' (new default): BuildHocon emits
Akka's AutoDowning provider with auto-down-unreachable-after = StableAfter.
The leader among the REACHABLE members downs the unreachable peer, so the
survivor takes over singletons and /health/active in ~25s regardless of which
node died. Accepted trade (explicit owner decision): a real network partition
runs dual-active until an operator restarts one side. keep-oldest remains
supported; DownIfAlone validation is now scoped to it.
Live drill on the rebuilt rig: active-crash TAKEOVER in 28s (victim still
down; all 7 singletons Younger->Oldest), standby-crash removal 27s with 0
routing blips; victims rejoin as standby in 2s. New real-cluster tests pin
both directions (SbrFailoverTests.AutoDown_*); TwoNodeClusterFixture gains a
strategy knob. All 16 appsettings flipped (src, docker, docker-env2, and the
gitignored wonder-app-vd03 overlay on disk — owner must sync to the host).
Docs: decision record docs/plans/2026-07-21-auto-down-availability-decision.md,
Component-ClusterInfrastructure downing section rewritten, drill + README
reworked (active mode now asserts takeover), deferred-work SBR row resolved.
70 lines
3.2 KiB
C#
70 lines
3.2 KiB
C#
using System.Diagnostics;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Tools.Singleton;
|
|
using Xunit.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
|
|
|
|
/// <summary>
|
|
/// Failover-timing measurement on the real two-node in-process rig
|
|
/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
|
|
/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
|
|
/// the CLAUDE.md "total failover ~25s" design envelope.
|
|
///
|
|
/// Runs under the production default downing strategy (auto-down, decision
|
|
/// 2026-07-21 — either-direction crash fails over; see SbrFailoverTests XML
|
|
/// doc). Measures a hard-crash of the YOUNGER node, timed to the survivor's
|
|
/// member REMOVAL (detection + stability window + gossip) with singleton
|
|
/// continuity asserted on the oldest; the oldest-crash direction is covered
|
|
/// behaviorally by SbrFailoverTests.AutoDown_HardCrashOfOldestNode_* and by
|
|
/// the docker failover drill. Covers overall review P2-10 / report-08 NF2
|
|
/// and report-01 round-2 N1's measurement ask.
|
|
/// </summary>
|
|
public class FailoverTimingTests(ITestOutputHelper output)
|
|
{
|
|
private sealed class EchoActor : ReceiveActor
|
|
{
|
|
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
|
|
}
|
|
|
|
[Trait("Category", "Performance")]
|
|
[Fact]
|
|
public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
|
|
{
|
|
await using var cluster = await TwoNodeClusterFixture.StartAsync(
|
|
stableAfter: TimeSpan.FromSeconds(15),
|
|
heartbeatInterval: TimeSpan.FromSeconds(2),
|
|
failureDetectionThreshold: TimeSpan.FromSeconds(10));
|
|
|
|
// Singleton on the oldest (NodeA) — the continuity probe.
|
|
var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
|
|
Props.Create(() => new EchoActor()),
|
|
PoisonPill.Instance,
|
|
ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-singleton");
|
|
var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
|
|
"/user/timing-probe-singleton",
|
|
ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-proxy");
|
|
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
|
|
|
|
var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
|
|
var sw = Stopwatch.StartNew();
|
|
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
|
|
await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
|
|
sw.Stop();
|
|
|
|
output.WriteLine(
|
|
$"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
|
|
"(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
|
|
|
|
// Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
|
|
Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
|
|
|
|
// Singleton continuity on the surviving oldest node.
|
|
Assert.Equal("ping-after-crash",
|
|
await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
|
|
}
|
|
}
|