diff --git a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs index 3d1c94da..88bded93 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs @@ -13,6 +13,10 @@ namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// singleton host). CrashNode() terminates a node WITHOUT cluster-leave /// (coordinated-shutdown by-terminate disabled on both nodes) to simulate /// a hard crash — the scenario review 01 found untested. +/// +/// Timing knobs default to scaled test values; pass production values +/// (2s heartbeat / 10s failure-detection / 15s stable-after) to measure the +/// real failover envelope (FailoverTimingTests). /// public sealed class TwoNodeClusterFixture : IAsyncDisposable { @@ -23,21 +27,23 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable public static async Task StartAsync( string role = "Central", TimeSpan? stableAfter = null, - int? portA = null, int? portB = null) + int? portA = null, int? portB = null, + TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null) { var f = new TwoNodeClusterFixture(); f.PortA = portA ?? GetFreeTcpPort(); f.PortB = portB ?? GetFreeTcpPort(); - f.NodeA = f.StartNode(f.PortA, role, stableAfter); + f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold); await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20)); - f.NodeB = f.StartNode(f.PortB, role, stableAfter); + f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold); await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20)); await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20)); return f; } /// Starts a node from production HOCON; used by StartAsync and by restart-scenarios. - public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null) + public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null, + TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null) { var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port }; var clusterOptions = new ClusterOptions @@ -49,8 +55,8 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable }, SplitBrainResolverStrategy = "keep-oldest", StableAfter = stableAfter ?? TimeSpan.FromSeconds(3), - HeartbeatInterval = TimeSpan.FromMilliseconds(500), - FailureDetectionThreshold = TimeSpan.FromSeconds(2), + HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500), + FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2), MinNrOfMembers = 1, DownIfAlone = true, }; diff --git a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs index 9e0e12db..615532c6 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs @@ -1,33 +1,68 @@ +using System.Diagnostics; +using Akka.Actor; +using Akka.Cluster.Tools.Singleton; +using Xunit.Abstractions; +using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; + namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover; /// -/// Failover-timing benchmark harness (placeholder). +/// Failover-timing measurement on the real two-node in-process rig +/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings: +/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after — +/// the CLAUDE.md "total failover ~25s" design envelope. /// -/// Design envelope under test (CLAUDE.md "Cluster & Failover"): failure -/// detection 2s heartbeat / 10s threshold, SBR stable-after 15s, total -/// failover ~25s for a hard node loss. -/// -/// Measurement protocol (to be wired to the two-node cluster rig delivered -/// by PLAN-01 — see archreview/plans/PLAN-01-*.md; do not duplicate that rig -/// here): -/// 1. Form a real 2-node cluster (docker/ topology or Akka.Remote in-proc -/// multi-ActorSystem rig from PLAN-01). -/// 2. Confirm a cluster singleton (e.g. site DeploymentManager) is hosted -/// on node A; record T0. -/// 3. Hard-kill node A (process kill / ActorSystem.Abort — not -/// CoordinatedShutdown; graceful stop exercises a different path). -/// 4. Poll node B for singleton re-host; record T1 at first successful -/// response from the migrated singleton. -/// 5. Assert T1 - T0 <= 40s (25s design + margin) and report the number. +/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node, +/// timed to the survivor's member REMOVAL (detection + stable-after + gossip) +/// with singleton continuity asserted on the oldest. The oldest/active-node +/// crash is NOT a recovery to time — two-node keep-oldest makes the younger +/// survivor down itself (total outage; registered deferred user decision, +/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall +/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask. /// -public class FailoverTimingTests +public class FailoverTimingTests(ITestOutputHelper output) { - [Trait("Category", "Performance")] - [Fact(Skip = "Requires the real two-node failover rig delivered by PLAN-01 " + - "(overall review P2-10). This class documents the measurement " + - "protocol; wire it to the rig when PLAN-01 lands.")] - public void HardKillFailover_SingletonRehostedWithinDesignEnvelope() + private sealed class EchoActor : ReceiveActor { - // Intentionally empty until the PLAN-01 rig exists. + public EchoActor() => ReceiveAny(msg => Sender.Tell(msg)); + } + + [Trait("Category", "Performance")] + [Fact] + public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope() + { + await using var cluster = await TwoNodeClusterFixture.StartAsync( + stableAfter: TimeSpan.FromSeconds(15), + heartbeatInterval: TimeSpan.FromSeconds(2), + failureDetectionThreshold: TimeSpan.FromSeconds(10)); + + // Singleton on the oldest (NodeA) — the continuity probe. + var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props( + Props.Create(() => new EchoActor()), + PoisonPill.Instance, + ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), + "timing-probe-singleton"); + var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props( + "/user/timing-probe-singleton", + ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), + "timing-probe-proxy"); + Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(30))); + + var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; + var sw = Stopwatch.StartNew(); + await TwoNodeClusterFixture.CrashNode(cluster.NodeB); + await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60)); + sw.Stop(); + + output.WriteLine( + $"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " + + "(design ~25s = 10s detection + 15s stable-after; +gossip margin)"); + + // Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin. + Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40)); + + // Singleton continuity on the surviving oldest node. + Assert.Equal("ping-after-crash", + await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(10))); } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj index bd285f71..7ebabf02 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj +++ b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj @@ -26,6 +26,8 @@ + +