Joseph Doherty
513fa1aa16
Comment/XML-doc changes only: - add summaries to ~10k test methods, fixtures, fakes and helpers (+ <param> on theories, <returns> on async Task methods, <inheritdoc/> on interface implementations the checker can resolve) - strip ~2.6k project-tracking references (WP*, P*-*, row-NN, Task N, #NN, Component-0NN) from comments, keeping the explanatory text; reword false positives in comments only - real docs on fakes implementing BCL/library interfaces (IDisposable, ILogger, IOptionsMonitor, IAsyncLifetime) - reword the last 5 src/ false positives (CWE-1236, failures-1, year-0001, M365) Only non-comment edits: four one-line nested classes split across lines so members can carry docs. Build 0 errors / 0 warnings.
72 lines
3.5 KiB
C#
72 lines
3.5 KiB
C#
using System.Diagnostics;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Tools.Singleton;
|
|
using Xunit.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.PerformanceTests.Failover;
|
|
|
|
/// <summary>
|
|
/// Failover-timing measurement on the real two-node in-process rig
|
|
/// (TwoNodeClusterFixture, production BuildHocon) at PRODUCTION timings:
|
|
/// 2s heartbeat / 10s failure-detection threshold / 15s SBR stable-after —
|
|
/// the CLAUDE.md "total failover ~25s" design envelope.
|
|
///
|
|
/// Runs under the production default downing strategy (auto-down, decision
|
|
/// 2026-07-21 — either-direction crash fails over; see SbrFailoverTests XML
|
|
/// doc). Measures a hard-crash of the YOUNGER node, timed to the survivor's
|
|
/// member REMOVAL (detection + stability window + gossip) with singleton
|
|
/// continuity asserted on the oldest; the oldest-crash direction is covered
|
|
/// behaviorally by SbrFailoverTests.AutoDown_HardCrashOfOldestNode_* and by
|
|
/// the docker failover drill. Covers the measurement requested by the overall review.
|
|
/// </summary>
|
|
public class FailoverTimingTests(ITestOutputHelper output)
|
|
{
|
|
private sealed class EchoActor : ReceiveActor
|
|
{
|
|
/// <summary>Initializes a new instance of the <see cref="EchoActor"/> class.</summary>
|
|
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
|
|
}
|
|
|
|
/// <summary>Verifies that a hard kill of the younger node is followed by removal within the design envelope.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Trait("Category", "Performance")]
|
|
[Fact]
|
|
public async Task HardKillOfYoungerNode_SbrRemovalWithinDesignEnvelope()
|
|
{
|
|
await using var cluster = await TwoNodeClusterFixture.StartAsync(
|
|
stableAfter: TimeSpan.FromSeconds(15),
|
|
heartbeatInterval: TimeSpan.FromSeconds(2),
|
|
failureDetectionThreshold: TimeSpan.FromSeconds(10));
|
|
|
|
// Singleton on the oldest (NodeA) — the continuity probe.
|
|
var manager = cluster.NodeA.ActorOf(ClusterSingletonManager.Props(
|
|
Props.Create(() => new EchoActor()),
|
|
PoisonPill.Instance,
|
|
ClusterSingletonManagerSettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-singleton");
|
|
var proxyA = cluster.NodeA.ActorOf(ClusterSingletonProxy.Props(
|
|
"/user/timing-probe-singleton",
|
|
ClusterSingletonProxySettings.Create(cluster.NodeA).WithSingletonName("timing-probe")),
|
|
"timing-probe-proxy");
|
|
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(30)));
|
|
|
|
var victim = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
|
|
var sw = Stopwatch.StartNew();
|
|
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
|
|
await TwoNodeClusterFixture.WaitForMemberRemoved(cluster.NodeA, victim, TimeSpan.FromSeconds(60));
|
|
sw.Stop();
|
|
|
|
output.WriteLine(
|
|
$"MEASURED: crash -> member removed on survivor: {sw.Elapsed.TotalSeconds:F1}s " +
|
|
"(design ~25s = 10s detection + 15s stable-after; +gossip margin)");
|
|
|
|
// Design envelope: >= stable-after (SBR must not act early), <= 25s design + 15s margin.
|
|
Assert.InRange(sw.Elapsed, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(40));
|
|
|
|
// Singleton continuity on the surviving oldest node.
|
|
Assert.Equal("ping-after-crash",
|
|
await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(10)));
|
|
}
|
|
}
|