dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

test(failover): unskip FailoverTimingTests on the two-node rig at production timings — measured 33.7s vs ~25s design envelope (plan R2-01 T4; covers report-08 NF2)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 10:38:19 -04:00
parent 30356803e7
commit 65e0f770d1
3 changed files with 73 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs
+12 -6
View File
@@ -13,6 +13,10 @@ namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
/// singleton host). CrashNode() terminates a node WITHOUT cluster-leave
/// (coordinated-shutdown by-terminate disabled on both nodes) to simulate
/// a hard crash — the scenario review 01 found untested.
///
/// Timing knobs default to scaled test values; pass production values
/// (2s heartbeat / 10s failure-detection / 15s stable-after) to measure the
/// real failover envelope (FailoverTimingTests).
/// </summary>
public sealed class TwoNodeClusterFixture : IAsyncDisposable
{
@@ -23,21 +27,23 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
public static async Task<TwoNodeClusterFixture> StartAsync(
string role = "Central", TimeSpan? stableAfter = null,
int? portA = null, int? portB = null)
int? portA = null, int? portB = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
{
var f = new TwoNodeClusterFixture();
f.PortA = portA ?? GetFreeTcpPort();
f.PortB = portB ?? GetFreeTcpPort();
f.NodeA = f.StartNode(f.PortA, role, stableAfter);
f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20));
f.NodeB = f.StartNode(f.PortB, role, stableAfter);
f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold);
await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20));
await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20));
return f;
}
/// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.</summary>
public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null)
public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null)
{
var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port };
var clusterOptions = new ClusterOptions
@@ -49,8 +55,8 @@ public sealed class TwoNodeClusterFixture : IAsyncDisposable
},
SplitBrainResolverStrategy = "keep-oldest",
StableAfter = stableAfter ?? TimeSpan.FromSeconds(3),
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500),
FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2),
MinNrOfMembers = 1,
DownIfAlone = true,
};
tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/Failover/FailoverTimingTests.cs
+59 -24
View File
@@ -1,33 +1,68 @@
using System.Diagnostics;
using Akka.Actor;
using Akka.Cluster.Tools.Singleton;
using Xunit.Abstractions;
using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
/// <summary>
/// Failover-timing benchmark harness (placeholder).
/// Failover-timing measurement on the real two-node in-process rig
/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
/// the CLAUDE.md "total failover ~25s" design envelope.
///
/// Design envelope under test (CLAUDE.md "Cluster &amp; Failover"): failure
/// detection 2s heartbeat / 10s threshold, SBR stable-after 15s, total
/// failover ~25s for a hard node loss.
///
/// Measurement protocol (to be wired to the two-node cluster rig delivered
/// by PLAN-01 — see archreview/plans/PLAN-01-*.md; do not duplicate that rig
/// here):
/// 1. Form a real 2-node cluster (docker/ topology or Akka.Remote in-proc
/// multi-ActorSystem rig from PLAN-01).
/// 2. Confirm a cluster singleton (e.g. site DeploymentManager) is hosted
/// on node A; record T0.
/// 3. Hard-kill node A (process kill / ActorSystem.Abort — not
/// CoordinatedShutdown; graceful stop exercises a different path).
/// 4. Poll node B for singleton re-host; record T1 at first successful
/// response from the migrated singleton.
/// 5. Assert T1 - T0 &lt;= 40s (25s design + margin) and report the number.
/// Measures the SURVIVABLE direction only: hard-crash of the YOUNGER node,
/// timed to the survivor's member REMOVAL (detection + stable-after + gossip)
/// with singleton continuity asserted on the oldest. The oldest/active-node
/// crash is NOT a recovery to time — two-node keep-oldest makes the younger
/// survivor down itself (total outage; registered deferred user decision,
/// see SbrFailoverTests XML doc + master tracker 2026-07-08). Covers overall
/// review P2-10 / report-08 NF2 and report-01 round-2 N1's measurement ask.
/// </summary>
public class FailoverTimingTests
public class FailoverTimingTests(ITestOutputHelper output)
{
[Trait("Category", "Performance")]
[Fact(Skip = "Requires the real two-node failover rig delivered by PLAN-01 " +
"(overall review P2-10). This class documents the measurement " +
"protocol; wire it to the rig when PLAN-01 lands.")]
public void HardKillFailover_SingletonRehostedWithinDesignEnvelope()
private sealed class EchoActor : ReceiveActor
{
// Intentionally empty until the PLAN-01 rig exists.
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
}
[Trait("Category", "Performance")]
[Fact]
public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
{
await using var cluster = await TwoNodeClusterFixture.StartAsync(
stableAfter: TimeSpan.FromSeconds(15),
heartbeatInterval: TimeSpan.FromSeconds(2),
failureDetectionThreshold: TimeSpan.FromSeconds(10));
// Singleton on the oldest (NodeA) — the continuity probe.
var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
Props.Create(() => new EchoActor()),
PoisonPill.Instance,
ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
"timing-probe-singleton");
var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
"/user/timing-probe-singleton",
ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
"timing-probe-proxy");
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
var sw = Stopwatch.StartNew();
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
sw.Stop();
output.WriteLine(
$"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
"(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
// Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
// Singleton continuity on the surviving oldest node.
Assert.Equal("ping-after-crash",
await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
}
}
tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/ZB.MOM.WW.ScadaBridge.PerformanceTests.csproj
+2
View File
@@ -26,6 +26,8 @@
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ZB.MOM.WW.ScadaBridge.HealthMonitoring.csproj" />
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.StoreAndForward/ZB.MOM.WW.ScadaBridge.StoreAndForward.csproj" />
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ZB.MOM.WW.ScadaBridge.SiteRuntime.csproj" />
<!-- Two-node failover rig (TwoNodeClusterFixture) — brings Host + Akka transitively. -->
<ProjectReference Include="../../tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/ZB.MOM.WW.ScadaBridge.IntegrationTests.csproj" />
</ItemGroup>
</Project>