Joseph Doherty
69 lines
3.2 KiB
C#
69 lines
3.2 KiB
C#
using System.Diagnostics;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Tools.Singleton;
|
|
using Xunit.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
|
|
|
|
/// <summary>
|
|
/// Failover-timing measurement on the real two-node in-process rig
|
|
/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
|
|
/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
|
|
/// the CLAUDE.md "total failover ~25s" design envelope.
|
|
///
|
|
/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node,
|
|
/// timed to the survivor's member REMOVAL (detection + stable-after + gossip)
|
|
/// with singleton continuity asserted on the oldest. The oldest/active-node
|
|
/// crash is NOT a recovery to time — two-node keep-oldest makes the younger
|
|
/// survivor down itself (total outage; registered deferred user decision,
|
|
/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall
|
|
/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask.
|
|
/// </summary>
|
|
public class FailoverTimingTests(ITestOutputHelper output)
|
|
{
|
|
private sealed class EchoActor : ReceiveActor
|
|
{
|
|
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
|
|
}
|
|
|
|
[Trait("Category", "Performance")]
|
|
[Fact]
|
|
public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
|
|
{
|
|
await using var cluster = await TwoNodeClusterFixture.StartAsync(
|
|
stableAfter: TimeSpan.FromSeconds(15),
|
|
heartbeatInterval: TimeSpan.FromSeconds(2),
|
|
failureDetectionThreshold: TimeSpan.FromSeconds(10));
|
|
|
|
// Singleton on the oldest (NodeA) — the continuity probe.
|
|
var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
|
|
Props.Create(() => new EchoActor()),
|
|
PoisonPill.Instance,
|
|
ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-singleton");
|
|
var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
|
|
"/user/timing-probe-singleton",
|
|
ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-proxy");
|
|
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
|
|
|
|
var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
|
|
var sw = Stopwatch.StartNew();
|
|
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
|
|
await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
|
|
sw.Stop();
|
|
|
|
output.WriteLine(
|
|
$"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
|
|
"(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
|
|
|
|
// Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
|
|
Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
|
|
|
|
// Singleton continuity on the surviving oldest node.
|
|
Assert.Equal("ping-after-crash",
|
|
await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
|
|
}
|
|
}
|