dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

test(failover): unskip FailoverTimingTests on the two-node rig at production timings — measured 33.7s vs ~25s design envelope (plan R2-01 T4; covers report-08 NF2)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 10:38:19 -04:00
parent 30356803e7
commit 65e0f770d1
3 changed files with 73 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs
+59 -24
View File
@@ -1,33 +1,68 @@
using System.Diagnostics;
using Akka.Actor;
using Akka.Cluster.Tools.Singleton;
using Xunit.Abstractions;
using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
/// <summary>
/// Failover-timing benchmark harness (placeholder).
/// Failover-timing measurement on the real two-node in-process rig
/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
/// the CLAUDE.md "total failover ~25s" design envelope.
///
/// Design envelope under test (CLAUDE.md "Cluster &amp; Failover"): failure
/// detection 2s heartbeat / 10s threshold, SBR stable-after 15s, total
/// failover ~25s for a hard node loss.
///
/// Measurement protocol (to be wired to the two-node cluster rig delivered
/// by PLAN-01 — see archreview/plans/PLAN-01-*.md; do not duplicate that rig
/// here):
/// 1. Form a real 2-node cluster (docker/ topology or Akka.Remote in-proc
/// multi-ActorSystem rig from PLAN-01).
/// 2. Confirm a cluster singleton (e.g. site DeploymentManager) is hosted
/// on node A; record T0.
/// 3. Hard-kill node A (process kill / ActorSystem.Abort — not
/// CoordinatedShutdown; graceful stop exercises a different path).
/// 4. Poll node B for singleton re-host; record T1 at first successful
/// response from the migrated singleton.
/// 5. Assert T1 - T0 &lt;= 40s (25s design + margin) and report the number.
/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node,
/// timed to the survivor's member REMOVAL (detection + stable-after + gossip)
/// with singleton continuity asserted on the oldest. The oldest/active-node
/// crash is NOT a recovery to time — two-node keep-oldest makes the younger
/// survivor down itself (total outage; registered deferred user decision,
/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall
/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask.
/// </summary>
public class FailoverTimingTests
public class FailoverTimingTests(ITestOutputHelper output)
{
[Trait("Category", "Performance")]
[Fact(Skip = "Requires the real two-node failover rig delivered by PLAN-01 " +
"(overall review P2-10). This class documents the measurement " +
"protocol; wire it to the rig when PLAN-01 lands.")]
public void HardKillFailover_SingletonRehostedWithinDesignEnvelope()
private sealed class EchoActor : ReceiveActor
{
// Intentionally empty until the PLAN-01 rig exists.
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
}
[Trait("Category", "Performance")]
[Fact]
public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
{
await using var cluster = await TwoNodeClusterFixture.StartAsync(
stableAfter: TimeSpan.FromSeconds(15),
heartbeatInterval: TimeSpan.FromSeconds(2),
failureDetectionThreshold: TimeSpan.FromSeconds(10));
// Singleton on the oldest (NodeA) — the continuity probe.
var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
Props.Create(() => new EchoActor()),
PoisonPill.Instance,
ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
"timing-probe-singleton");
var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
"/user/timing-probe-singleton",
ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
"timing-probe-proxy");
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
var sw = Stopwatch.StartNew();
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
sw.Stop();
output.WriteLine(
$"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
"(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
// Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
// Singleton continuity on the surviving oldest node.
Assert.Equal("ping-after-crash",
await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
}
}
tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj
+2
View File
@@ -26,6 +26,8 @@
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ZB.MOM.WW.ScadaBridge.HealthMonitoring.csproj" />
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ZB.MOM.WW.ScadaBridge.StoreAndForward.csproj" />
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
<!-- Two-node failover rig (TwoNodeClusterFixture) — brings Host + Akka transitively. -->
<ProjectReference Include="../../tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/ZB.MOM.WW.ScadaBridge.IntegrationTests.csproj" />
</ItemGroup>
</Project>