using System.Diagnostics; using Akka.Actor; using Akka.Cluster.Tools.Singleton; using Xunit.Abstractions; using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover; /// /// Failover-timing measurement on the real two-node in-process rig /// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings: /// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after — /// the CLAUDE.md "total failover ~25s" design envelope. /// /// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node, /// timed to the survivor's member REMOVAL (detection + stable-after + gossip) /// with singleton continuity asserted on the oldest. The oldest/active-node /// crash is NOT a recovery to time — two-node keep-oldest makes the younger /// survivor down itself (total outage; registered deferred user decision, /// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall /// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask. /// public class FailoverTimingTests(ITestOutputHelper output) { private sealed class EchoActor : ReceiveActor { public EchoActor() => ReceiveAny(msg => Sender.Tell(msg)); } [Trait("Category", "Performance")] [Fact] public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope() { await using var cluster = await TwoNodeClusterFixture.StartAsync( stableAfter: TimeSpan.FromSeconds(15), heartbeatInterval: TimeSpan.FromSeconds(2), failureDetectionThreshold: TimeSpan.FromSeconds(10)); // Singleton on the oldest (NodeA) — the continuity probe. var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props( Props.Create(() => new EchoActor()), PoisonPill.Instance, ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), "timing-probe-singleton"); var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props( "/user/timing-probe-singleton", ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")), "timing-probe-proxy"); Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(30))); var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress; var sw = Stopwatch.StartNew(); await TwoNodeClusterFixture.CrashNode(cluster.NodeB); await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60)); sw.Stop(); output.WriteLine( $"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " + "(design ~25s = 10s detection + 15s stable-after; +gossip margin)"); // Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin. Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40)); // Singleton continuity on the surviving oldest node. Assert.Equal("ping-after-crash", await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(10))); } }