feat(cluster)!: leader-auto-down is the shipped default downing strategy (#116)

SplitBrainResolverStrategy now defaults to leader-auto-down (the in-repo
LeaderAutoDowningProvider, #106, with the #132 shut-out restart on). auto-down and
keep-oldest stay supported. Drilled on the docker rig, docker-env2 and the QA pair
(central + site, kill and stop-and-hold) before the flip.

- ClusterOptions default + validator/HOCON comments updated.
- Shipped Host appsettings.Central/Site.json and every docker/ and docker-env2/
  node appsettings now say leader-auto-down. The leader-auto-down overlays are
  kept as no-ops (drill records reference them); new docker-compose.auto-down.yml
  overlays keep Akka AutoDowning available for comparison drills.
- Tests: default asserts leader-auto-down; new HOCON test for an unset strategy;
  TwoNodeClusterFixture defaults to the production posture.
- Docs: CLAUDE.md, Component-ClusterInfrastructure, components/Host + Cluster,
  README, failover procedures, Traefik, docker + env2 READMEs; register row 70 closed.

No deploy needed: DEV, QA and both rigs set the key explicitly.

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
Joseph Doherty
2026-11-15 19:49:31 -05:00
parent 1213a74353
commit efade2eab0
36 changed files with 153 additions and 83 deletions
@@ -11,10 +11,11 @@ public class ClusterOptionsTests
{
var options = new ClusterOptions();
// 'auto-down' is the default posture (decision 2026-07-21): a crash of either
// node fails over to the survivor; dual-active during a real partition is the
// accepted trade for pairs with no shared lease infrastructure.
Assert.Equal("auto-down", options.SplitBrainResolverStrategy);
// 'leader-auto-down' is the default since 2026-11-15 (ScadaBridge#116): the in-repo port
// of the availability-first 'auto-down' posture (decision 2026-07-21), without Akka's
// deprecated AutoDowning provider. A crash of either node fails over to the survivor.
Assert.Equal("leader-auto-down", options.SplitBrainResolverStrategy);
Assert.True(options.LeaderAutoDown.RestartWhenShutOut);
Assert.Equal(TimeSpan.FromSeconds(15), options.StableAfter);
Assert.Equal(TimeSpan.FromSeconds(2), options.HeartbeatInterval);
Assert.Equal(TimeSpan.FromSeconds(10), options.FailureDetectionThreshold);
@@ -364,4 +364,27 @@ public class HoconBuilderTests
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
}
/// <summary>
/// ScadaBridge#116: an environment that sets no strategy gets <c>leader-auto-down</c> with
/// <c>LeaderAutoDown</c> at its defaults — the in-repo provider, the StableAfter window under its
/// own key, the shut-out restart ON, and no deprecated <c>auto-down-unreachable-after</c> key.
/// </summary>
[Fact]
public void BuildHocon_UnsetStrategy_DefaultsToLeaderAutoDown()
{
var cluster = new ClusterOptions { SeedNodes = DefaultCluster().SeedNodes };
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
Assert.Equal(LeaderAutoDowningProvider.ProviderClassName,
config.GetString("akka.cluster.downing-provider-class"));
Assert.Equal(TimeSpan.FromSeconds(15), config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
Assert.False(config.HasPath("akka.cluster.auto-down-unreachable-after"));
Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
}
}
@@ -20,7 +20,7 @@ namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
/// members (<c>KeepOldest.OldestDecision</c>: <c>otherSide == 1 &amp;&amp; thisSide &gt;= 2</c>).
/// With a two-node cluster the younger survivor therefore takes <c>DownReachable</c> — it downs
/// ITSELF — so only a YOUNGER-node crash is survivable.</item>
/// <item><c>auto-down</c> (production default, decision 2026-07-21) — the leader among
/// <item><c>auto-down</c> (the production default from 2026-07-21 until 2026-11-15; leader-auto-down since, #116) — the leader among
/// the reachable members downs the unreachable peer after the stability window, so a
/// crash of EITHER node fails over to the survivor; the accepted trade is dual-active
/// during a real network partition.</item>
@@ -42,7 +42,7 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
string role = "Central", TimeSpan? stableAfter = null,
int? portA = null, int? portB = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
string strategy = "auto-down")
string strategy = "leader-auto-down")
{
var f = new TwoNodeClusterFixture();
f.PortA = portA ?? GetFreeTcpPort();
@@ -56,8 +56,9 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
}
/// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.
/// <paramref name="strategy"/> defaults to the production posture (auto-down, decision
/// 2026-07-21); pass "keep-oldest" to exercise the legacy SBR path.</summary>
/// <paramref name="strategy"/> defaults to the production posture (leader-auto-down, the default
/// since 2026-11-15, ScadaBridge#116); pass "auto-down" for Akka's AutoDowning or "keep-oldest"
/// to exercise the legacy SBR path.</summary>
/// <param name="port">The remoting port for the node.</param>
/// <param name="role">The Akka role assigned to the node.</param>
/// <param name="stableAfter">How long membership must be stable before downing decisions; defaults to a scaled test value.</param>
@@ -67,7 +68,7 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
/// <returns>The started actor system.</returns>
public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
string strategy = "auto-down")
string strategy = "leader-auto-down")
{
var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port };
var clusterOptions = new ClusterOptions