feat(cluster): auto-down downing strategy — either-node crash now fails over (owner decision 2026-07-21: availability over partition-safety)

Two-node keep-oldest could NEVER survive a crash of the oldest/active node:
Akka.NET 1.5.62 KeepOldest.OldestDecision only lets down-if-alone rescue a
side with >= 2 members, so the 1-vs-1 survivor takes DownReachable and downs
ITSELF — proven live on the rig ('SBR took decision ... including myself')
before this change. static-quorum(1) is worse (IsTooManyMembers -> DownAll);
keep-majority just re-keys the fatal crash to the lowest address.

SplitBrainResolverStrategy gains 'auto-down' (new default): BuildHocon emits
Akka's AutoDowning provider with auto-down-unreachable-after = StableAfter.
The leader among the REACHABLE members downs the unreachable peer, so the
survivor takes over singletons and /health/active in ~25s regardless of which
node died. Accepted trade (explicit owner decision): a real network partition
runs dual-active until an operator restarts one side. keep-oldest remains
supported; DownIfAlone validation is now scoped to it.

Live drill on the rebuilt rig: active-crash TAKEOVER in 28s (victim still
down; all 7 singletons Younger->Oldest), standby-crash removal 27s with 0
routing blips; victims rejoin as standby in 2s. New real-cluster tests pin
both directions (SbrFailoverTests.AutoDown_*); TwoNodeClusterFixture gains a
strategy knob. All 16 appsettings flipped (src, docker, docker-env2, and the
gitignored wonder-app-vd03 overlay on disk — owner must sync to the host).
Docs: decision record docs/plans/2026-07-21-auto-down-availability-decision.md,
Component-ClusterInfrastructure downing section rewritten, drill + README
reworked (active mode now asserts takeover), deferred-work SBR row resolved.

tests/ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests/ClusterOptionsTests.cs

@@ -10,7 +10,10 @@ public class ClusterOptionsTests
     {
         var options = new ClusterOptions();
 
-        Assert.Equal("keep-oldest", options.SplitBrainResolverStrategy);
+        // 'auto-down' is the default posture (decision 2026-07-21): a crash of either
+        // node fails over to the survivor; dual-active during a real partition is the
+        // accepted trade for pairs with no shared lease infrastructure.
+        Assert.Equal("auto-down", options.SplitBrainResolverStrategy);
         Assert.Equal(TimeSpan.FromSeconds(15), options.StableAfter);
         Assert.Equal(TimeSpan.FromSeconds(2), options.HeartbeatInterval);
         Assert.Equal(TimeSpan.FromSeconds(10), options.FailureDetectionThreshold);

tests/ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests/ClusterOptionsValidatorTests.cs

@@ -153,9 +153,10 @@ public class ClusterOptionsValidatorTests
     }
 
     [Fact]
-    public void DownIfAloneFalse_FailsValidation()
+    public void DownIfAloneFalse_UnderKeepOldest_FailsValidation()
     {
         var options = ValidOptions();
+        options.SplitBrainResolverStrategy = "keep-oldest";
         options.DownIfAlone = false;
 
         var result = new ClusterOptionsValidator().Validate(null, options);
@@ -164,6 +165,34 @@ public class ClusterOptionsValidatorTests
         Assert.Contains("DownIfAlone", result.FailureMessage);
     }
 
+    [Fact]
+    public void AutoDownStrategy_Passes()
+    {
+        // Decision 2026-07-21: 'auto-down' is the supported availability-first
+        // posture — either-node crash fails over; dual-active on a real partition
+        // is the accepted trade.
+        var options = ValidOptions();
+        options.SplitBrainResolverStrategy = "auto-down";
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Succeeded, result.FailureMessage);
+    }
+
+    [Fact]
+    public void DownIfAloneFalse_UnderAutoDown_Passes()
+    {
+        // DownIfAlone is a keep-oldest knob; under auto-down it is inert and must
+        // not block startup.
+        var options = ValidOptions();
+        options.SplitBrainResolverStrategy = "auto-down";
+        options.DownIfAlone = false;
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Succeeded, result.FailureMessage);
+    }
+
     [Fact]
     public void Validate_AccumulatesAllFailures()
     {

tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HoconBuilderTests.cs

@@ -195,4 +195,49 @@ public class HoconBuilderTests
             "Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster",
             config.GetString("akka.cluster.downing-provider-class"));
     }
+
+    [Fact]
+    public void BuildHocon_AutoDownStrategy_EmitsAutoDowningProvider()
+    {
+        // Decision 2026-07-21 (availability over partition-safety): 'auto-down' must
+        // swap the downing provider to Akka's AutoDowning so the survivor downs a
+        // crashed peer — including a crashed OLDEST, which two-node keep-oldest
+        // cannot survive — after StableAfter.
+        var cluster = DefaultCluster();
+        cluster.SplitBrainResolverStrategy = "auto-down";
+        cluster.StableAfter = TimeSpan.FromSeconds(15);
+
+        var hocon = AkkaHostedService.BuildHocon(
+            DefaultNode(), cluster, new[] { "Central" },
+            TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
+
+        var config = ConfigurationFactory.ParseString(hocon);
+        Assert.Equal(
+            "Akka.Cluster.AutoDowning, Akka.Cluster",
+            config.GetString("akka.cluster.downing-provider-class"));
+        Assert.Equal(
+            TimeSpan.FromSeconds(15),
+            config.GetTimeSpan("akka.cluster.auto-down-unreachable-after"));
+        // The SBR section must NOT be active alongside AutoDowning.
+        Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
+    }
+
+    [Fact]
+    public void BuildHocon_AutoDownStrategy_IsCaseInsensitive_AndDocumentStaysIntact()
+    {
+        var cluster = DefaultCluster();
+        cluster.SplitBrainResolverStrategy = "Auto-Down";
+
+        var hocon = AkkaHostedService.BuildHocon(
+            DefaultNode(), cluster, new[] { "Central" },
+            TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
+
+        var config = ConfigurationFactory.ParseString(hocon);
+        Assert.Equal(
+            "Akka.Cluster.AutoDowning, Akka.Cluster",
+            config.GetString("akka.cluster.downing-provider-class"));
+        // Keys after the downing block must remain intact (document not corrupted).
+        Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
+        Assert.True(config.GetBoolean("akka.cluster.run-coordinated-shutdown-when-down"));
+    }
 }

tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/SbrFailoverTests.cs

@@ -5,22 +5,26 @@ using Xunit;
 namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
 
 /// <summary>
-/// Behavioral proof that the SBR downing provider enabled in
+/// Behavioral proof that the downing provider enabled in
 /// <c>AkkaHostedService.BuildHocon</c> (arch-review 01 Critical) is actually active:
 /// after a hard crash, the surviving node DOWNS and REMOVES the crashed member. Under
 /// the pre-fix Akka default (NoDowning) the crashed member lingers <c>Unreachable</c>
 /// forever, so the member-removal assertions here are impossible to satisfy without the
-/// fix — that is what gives the test teeth.
+/// fix — that is what gives the tests teeth.
 ///
-/// IMPORTANT — <c>keep-oldest</c> two-node semantics (verified empirically, Akka 1.5.62):
-/// SBR downs the partition that does NOT contain the oldest member. So the crash that
-/// SBR can recover from in a two-node cluster is the crash of the YOUNGER node — the
-/// oldest survives and keeps its singletons. Crashing the OLDEST node instead makes the
-/// younger survivor down ITSELF (total cluster loss); <c>down-if-alone=on</c> does not
-/// change this on a hard crash because the alone-oldest is no longer running to down
-/// itself. That asymmetry (active/oldest-node crash is NOT covered by two-node
-/// keep-oldest) is a design-level gap tracked separately, not something this test can
-/// assert as a success path.
+/// TWO-NODE SEMANTICS (verified against Akka.NET 1.5.62 source + live on the docker
+/// rig, 2026-07-21):
+/// <list type="bullet">
+/// <item><c>keep-oldest</c> — downs the side that does NOT contain the oldest member,
+/// and its <c>down-if-alone</c> escape only fires when the surviving side has ≥2
+/// members (<c>KeepOldest.OldestDecision</c>: <c>otherSide == 1 &amp;&amp; thisSide &gt;= 2</c>).
+/// With 1-vs-1 the younger survivor therefore takes <c>DownReachable</c> — it downs
+/// ITSELF — so only a YOUNGER-node crash is survivable.</item>
+/// <item><c>auto-down</c> (production default, decision 2026-07-21) — the leader among
+/// the reachable members downs the unreachable peer after the stability window, so a
+/// crash of EITHER node fails over to the survivor; the accepted trade is dual-active
+/// during a real network partition.</item>
+/// </list>
 /// </summary>
 public class SbrFailoverTests
 {
@@ -46,7 +50,8 @@ public class SbrFailoverTests
     [Fact]
     public async Task HardCrashOfYoungerNode_SbrDownsIt_AndOldestKeepsSingleton()
     {
-        await using var cluster = await TwoNodeClusterFixture.StartAsync();
+        // Pinned to keep-oldest: this is the SBR path's (only) survivable direction.
+        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "keep-oldest");
         var (_, proxyA) = StartSingleton(cluster.NodeA); // oldest hosts the singleton
         StartSingleton(cluster.NodeB);
 
@@ -78,4 +83,80 @@ public class SbrFailoverTests
         }
         throw new Xunit.Sdk.XunitException($"Singleton stopped answering on the surviving oldest node after SBR downing: {last}");
     }
+
+    [Fact]
+    public async Task AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
+    {
+        // Decision 2026-07-21: the direction two-node keep-oldest can NEVER survive
+        // (proven live on the docker rig — the younger survivor took DownReachable and
+        // self-downed). Under auto-down the survivor must instead down the crashed
+        // oldest and TAKE OVER its singleton.
+        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
+        StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
+        var (_, proxyB) = StartSingleton(cluster.NodeB);
+
+        // Singleton reachable from B while A is alive (proxy routes to the oldest).
+        var echo = await proxyB.Ask<string>("ping", TimeSpan.FromSeconds(20));
+        Assert.Equal("ping", echo);
+
+        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
+        await TwoNodeClusterFixture.CrashNode(cluster.NodeA);
+
+        // 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member —
+        //    the exact step keep-oldest refuses (it downs itself instead).
+        await TwoNodeClusterFixture.WaitForMemberRemoved(
+            cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));
+
+        // 2) B must still be a functioning cluster member (not self-downed) …
+        var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
+        Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");
+
+        // 3) … and the singleton must migrate to B and answer again.
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+        Exception? last = null;
+        while (DateTime.UtcNow < deadline)
+        {
+            try
+            {
+                var echo2 = await proxyB.Ask<string>("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
+                Assert.Equal("ping-after-oldest-crash", echo2);
+                return;
+            }
+            catch (Exception ex) { last = ex; }
+        }
+        throw new Xunit.Sdk.XunitException(
+            $"Singleton never migrated to the younger survivor after the oldest crashed under auto-down: {last}");
+    }
+
+    [Fact]
+    public async Task AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
+    {
+        // The previously-survivable direction must STAY survivable under auto-down.
+        await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
+        var (_, proxyA) = StartSingleton(cluster.NodeA);
+        StartSingleton(cluster.NodeB);
+
+        Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20)));
+
+        var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
+        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
+
+        await TwoNodeClusterFixture.WaitForMemberRemoved(
+            cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
+
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+        Exception? last = null;
+        while (DateTime.UtcNow < deadline)
+        {
+            try
+            {
+                var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
+                Assert.Equal("ping-after-crash", echo2);
+                return;
+            }
+            catch (Exception ex) { last = ex; }
+        }
+        throw new Xunit.Sdk.XunitException(
+            $"Singleton stopped answering on the surviving oldest node under auto-down: {last}");
+    }
 }

tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs

@@ -28,22 +28,26 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
     public static async Task<TwoNodeClusterFixture> StartAsync(
         string role = "Central", TimeSpan? stableAfter = null,
         int? portA = null, int? portB = null,
-        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
+        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
+        string strategy = "auto-down")
     {
         var f = new TwoNodeClusterFixture();
         f.PortA = portA ?? GetFreeTcpPort();
         f.PortB = portB ?? GetFreeTcpPort();
-        f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
+        f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold, strategy);
         await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20));
-        f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
+        f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold, strategy);
         await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20));
         await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20));
         return f;
     }
 
-    /// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.</summary>
+    /// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.
+    /// <paramref name="strategy"/> defaults to the production posture (auto-down, decision
+    /// 2026-07-21); pass "keep-oldest" to exercise the legacy SBR path.</summary>
     public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null,
-        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
+        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
+        string strategy = "auto-down")
     {
         var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port };
         var clusterOptions = new ClusterOptions
@@ -53,7 +57,7 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
                 $"akka.tcp://scadabridge@127.0.0.1:{PortA}",
                 $"akka.tcp://scadabridge@127.0.0.1:{PortB}",
             },
-            SplitBrainResolverStrategy = "keep-oldest",
+            SplitBrainResolverStrategy = strategy,
             StableAfter = stableAfter ?? TimeSpan.FromSeconds(3),
             HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500),
             FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2),

tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs

@@ -12,13 +12,14 @@ namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
 /// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
 /// the CLAUDE.md "total failover ~25s" design envelope.
 ///
-/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node,
-/// timed to the survivor's member REMOVAL (detection + stable-after + gossip)
-/// with singleton continuity asserted on the oldest. The oldest/active-node
-/// crash is NOT a recovery to time — two-node keep-oldest makes the younger
-/// survivor down itself (total outage; registered deferred user decision,
-/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall
-/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask.
+/// Runs under the production default downing strategy (auto-down, decision
+/// 2026-07-21 — either-direction crash fails over; see SbrFailoverTests XML
+/// doc). Measures a hard-crash of the YOUNGER node, timed to the survivor's
+/// member REMOVAL (detection + stability window + gossip) with singleton
+/// continuity asserted on the oldest; the oldest-crash direction is covered
+/// behaviorally by SbrFailoverTests.AutoDown_HardCrashOfOldestNode_* and by
+/// the docker failover drill. Covers overall review P2-10 / report-08 NF2
+/// and report-01 round-2 N1's measurement ask.
 /// </summary>
 public class FailoverTimingTests(ITestOutputHelper output)
 {