test(failover): unskip FailoverTimingTests on the two-node rig at production timings — measured 33.7s vs ~25s design envelope (plan R2-01 T4; covers report-08 NF2)

tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs

@@ -13,6 +13,10 @@ namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
 /// singleton host). CrashNode() terminates a node WITHOUT cluster-leave
 /// (coordinated-shutdown by-terminate disabled on both nodes) to simulate
 /// a hard crash — the scenario review 01 found untested.
+///
+/// Timing knobs default to scaled test values; pass production values
+/// (2s heartbeat / 10s failure-detection / 15s stable-after) to measure the
+/// real failover envelope (FailoverTimingTests).
 /// </summary>
 public sealed class TwoNodeClusterFixture : IAsyncDisposable
 {
@@ -23,21 +27,23 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
 
     public static async Task<TwoNodeClusterFixture> StartAsync(
         string role = "Central", TimeSpan? stableAfter = null,
-        int? portA = null, int? portB = null)
+        int? portA = null, int? portB = null,
+        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
     {
         var f = new TwoNodeClusterFixture();
         f.PortA = portA ?? GetFreeTcpPort();
         f.PortB = portB ?? GetFreeTcpPort();
-        f.NodeA = f.StartNode(f.PortA, role, stableAfter);
+        f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
         await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20));
-        f.NodeB = f.StartNode(f.PortB, role, stableAfter);
+        f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
         await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20));
         await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20));
         return f;
     }
 
     /// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.</summary>
-    public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null)
+    public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null,
+        TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
     {
         var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port };
         var clusterOptions = new ClusterOptions
@@ -49,8 +55,8 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
             },
             SplitBrainResolverStrategy = "keep-oldest",
             StableAfter = stableAfter ?? TimeSpan.FromSeconds(3),
-            HeartbeatInterval = TimeSpan.FromMilliseconds(500),
+            HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500),
-            FailureDetectionThreshold = TimeSpan.FromSeconds(2),
+            FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2),
             MinNrOfMembers = 1,
             DownIfAlone = true,
         };

tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs

@@ -1,33 +1,68 @@
+using System.Diagnostics;
+using Akka.Actor;
+using Akka.Cluster.Tools.Singleton;
+using Xunit.Abstractions;
+using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
+
 namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
 
 /// <summary>
-/// Failover-timing benchmark harness (placeholder).
+/// Failover-timing measurement on the real two-node in-process rig
+/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
+/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
+/// the CLAUDE.md "total failover ~25s" design envelope.
 ///
-/// Design envelope under test (CLAUDE.md "Cluster &amp; Failover"): failure
+/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node,
-/// detection 2s heartbeat / 10s threshold, SBR stable-after 15s, total
+/// timed to the survivor's member REMOVAL (detection + stable-after + gossip)
-/// failover ~25s for a hard node loss.
+/// with singleton continuity asserted on the oldest. The oldest/active-node
-///
+/// crash is NOT a recovery to time — two-node keep-oldest makes the younger
-/// Measurement protocol (to be wired to the two-node cluster rig delivered
+/// survivor down itself (total outage; registered deferred user decision,
-/// by PLAN-01 — see archreview/plans/PLAN-01-*.md; do not duplicate that rig
+/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall
-/// here):
+/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask.
-///   1. Form a real 2-node cluster (docker/ topology or Akka.Remote in-proc
-///      multi-ActorSystem rig from PLAN-01).
-///   2. Confirm a cluster singleton (e.g. site DeploymentManager) is hosted
-///      on node A; record T0.
-///   3. Hard-kill node A (process kill / ActorSystem.Abort — not
-///      CoordinatedShutdown; graceful stop exercises a different path).
-///   4. Poll node B for singleton re-host; record T1 at first successful
-///      response from the migrated singleton.
-///   5. Assert T1 - T0 &lt;= 40s (25s design + margin) and report the number.
 /// </summary>
-public class FailoverTimingTests
+public class FailoverTimingTests(ITestOutputHelper output)
 {
-    [Trait("Category", "Performance")]
+    private sealed class EchoActor : ReceiveActor
-    [Fact(Skip = "Requires the real two-node failover rig delivered by PLAN-01 " +
-                 "(overall review P2-10). This class documents the measurement " +
-                 "protocol; wire it to the rig when PLAN-01 lands.")]
-    public void HardKillFailover_SingletonRehostedWithinDesignEnvelope()
     {
-        // Intentionally empty until the PLAN-01 rig exists.
+        public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
+    }
+
+    [Trait("Category", "Performance")]
+    [Fact]
+    public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
+    {
+        await using var cluster = await TwoNodeClusterFixture.StartAsync(
+            stableAfter: TimeSpan.FromSeconds(15),
+            heartbeatInterval: TimeSpan.FromSeconds(2),
+            failureDetectionThreshold: TimeSpan.FromSeconds(10));
+
+        // Singleton on the oldest (NodeA) — the continuity probe.
+        var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
+            Props.Create(() => new EchoActor()),
+            PoisonPill.Instance,
+            ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
+            "timing-probe-singleton");
+        var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
+            "/user/timing-probe-singleton",
+            ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
+            "timing-probe-proxy");
+        Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
+
+        var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
+        var sw = Stopwatch.StartNew();
+        await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
+        await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
+        sw.Stop();
+
+        output.WriteLine(
+            $"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
+            "(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
+
+        // Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
+        Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
+
+        // Singleton continuity on the surviving oldest node.
+        Assert.Equal("ping-after-crash",
+            await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
     }
 }

tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj

@@ -26,6 +26,8 @@
     <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ZB.MOM.WW.ScadaBridge.HealthMonitoring.csproj" />
     <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ZB.MOM.WW.ScadaBridge.StoreAndForward.csproj" />
     <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
+    <!-- Two-node failover rig (TwoNodeClusterFixture) — brings Host + Akka transitively. -->
+    <ProjectReference Include="../../tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/ZB.MOM.WW.ScadaBridge.IntegrationTests.csproj" />
   </ItemGroup>
 
 </Project>