Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs
T
Joseph Doherty efade2eab0 feat(cluster)!: leader-auto-down is the shipped default downing strategy (#116)
SplitBrainResolverStrategy now defaults to leader-auto-down (the in-repo
LeaderAutoDowningProvider, #106, with the #132 shut-out restart on). auto-down and
keep-oldest stay supported. Drilled on the docker rig, docker-env2 and the QA pair
(central + site, kill and stop-and-hold) before the flip.

- ClusterOptions default + validator/HOCON comments updated.
- Shipped Host appsettings.Central/Site.json and every docker/ and docker-env2/
  node appsettings now say leader-auto-down. The leader-auto-down overlays are
  kept as no-ops (drill records reference them); new docker-compose.auto-down.yml
  overlays keep Akka AutoDowning available for comparison drills.
- Tests: default asserts leader-auto-down; new HOCON test for an unset strategy;
  TwoNodeClusterFixture defaults to the production posture.
- Docs: CLAUDE.md, Component-ClusterInfrastructure, components/Host + Cluster,
  README, failover procedures, Traefik, docker + env2 READMEs; register row 70 closed.

No deploy needed: DEV, QA and both rigs set the key explicitly.

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
2026-11-15 19:57:45 -05:00

169 lines
9.2 KiB
C#

using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Host;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
/// <summary>
/// Forms a REAL two-node Akka cluster in-process from the production
/// BuildHocon output. NodeA is always started first (=> oldest member /
/// singleton host). CrashNode() terminates a node WITHOUT cluster-leave
/// (coordinated-shutdown by-terminate disabled on both nodes) to simulate
/// a hard crash — the scenario review 01 found untested.
///
/// Timing knobs default to scaled test values; pass production values
/// (2s heartbeat / 10s failure-detection / 15s stable-after) to measure the
/// real failover envelope (FailoverTimingTests).
/// </summary>
public sealed class TwoNodeClusterFixture : IAsyncDisposable
{
/// <summary>Gets the first cluster node, which is always started first and is therefore the oldest member.</summary>
public ActorSystem NodeA { get; private set; } = null!;
/// <summary>Gets the second cluster node, which joins after the first.</summary>
public ActorSystem NodeB { get; private set; } = null!;
/// <summary>Gets the remoting port used by the first node.</summary>
public int PortA { get; private set; }
/// <summary>Gets the remoting port used by the second node.</summary>
public int PortB { get; private set; }
/// <summary>Starts both nodes in order and waits until each sees two Up members.</summary>
/// <param name="role">The Akka role assigned to both nodes.</param>
/// <param name="stableAfter">How long membership must be stable before downing decisions; defaults to a scaled test value.</param>
/// <param name="portA">The remoting port for the first node, or null to pick a free port.</param>
/// <param name="portB">The remoting port for the second node, or null to pick a free port.</param>
/// <param name="heartbeatInterval">The failure-detector heartbeat interval; defaults to a scaled test value.</param>
/// <param name="failureDetectionThreshold">The failure-detector threshold; defaults to a scaled test value.</param>
/// <param name="strategy">The split-brain resolver strategy to configure.</param>
/// <returns>A task that yields the started two-node fixture.</returns>
public static async Task<TwoNodeClusterFixture> StartAsync(
string role = "Central", TimeSpan? stableAfter = null,
int? portA = null, int? portB = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
string strategy = "leader-auto-down")
{
var f = new TwoNodeClusterFixture();
f.PortA = portA ?? GetFreeTcpPort();
f.PortB = portB ?? GetFreeTcpPort();
f.NodeA = f.StartNode(f.PortA, role, stableAfter, heartbeatInterval, failureDetectionThreshold, strategy);
await WaitForMembersUp(f.NodeA, 1, TimeSpan.FromSeconds(20));
f.NodeB = f.StartNode(f.PortB, role, stableAfter, heartbeatInterval, failureDetectionThreshold, strategy);
await WaitForMembersUp(f.NodeA, 2, TimeSpan.FromSeconds(20));
await WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(20));
return f;
}
/// <summary>Starts a node from production HOCON; used by StartAsync and by restart-scenarios.
/// <paramref name="strategy"/> defaults to the production posture (leader-auto-down, the default
/// since 2026-11-15, ScadaBridge#116); pass "auto-down" for Akka's AutoDowning or "keep-oldest"
/// to exercise the legacy SBR path.</summary>
/// <param name="port">The remoting port for the node.</param>
/// <param name="role">The Akka role assigned to the node.</param>
/// <param name="stableAfter">How long membership must be stable before downing decisions; defaults to a scaled test value.</param>
/// <param name="heartbeatInterval">The failure-detector heartbeat interval; defaults to a scaled test value.</param>
/// <param name="failureDetectionThreshold">The failure-detector threshold; defaults to a scaled test value.</param>
/// <param name="strategy">The split-brain resolver strategy to configure.</param>
/// <returns>The started actor system.</returns>
public ActorSystem StartNode(int port, string role, TimeSpan? stableAfter = null,
TimeSpan? heartbeatInterval = null, TimeSpan? failureDetectionThreshold = null,
string strategy = "leader-auto-down")
{
var nodeOptions = new NodeOptions { Role = role, NodeHostname = "127.0.0.1", RemotingPort = port };
var clusterOptions = new ClusterOptions
{
SeedNodes = new List<string>
{
$"akka.tcp://scadabridge@127.0.0.1:{PortA}",
$"akka.tcp://scadabridge@127.0.0.1:{PortB}",
},
SplitBrainResolverStrategy = strategy,
StableAfter = stableAfter ?? TimeSpan.FromSeconds(3),
HeartbeatInterval = heartbeatInterval ?? TimeSpan.FromMilliseconds(500),
FailureDetectionThreshold = failureDetectionThreshold ?? TimeSpan.FromSeconds(2),
MinNrOfMembers = 1,
DownIfAlone = true,
};
var hocon = AkkaHostedService.BuildHocon(
nodeOptions, clusterOptions, new[] { role },
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3));
// Crash simulation: Terminate() must NOT run CoordinatedShutdown
// (no Leave gossip) or the test degenerates to the graceful path.
var config = ConfigurationFactory
.ParseString("akka.coordinated-shutdown.run-by-actor-system-terminate = off")
.WithFallback(ConfigurationFactory.ParseString(hocon));
return ActorSystem.Create("scadabridge", config);
}
/// <summary>Hard-crash a node: abrupt terminate, no cluster leave.</summary>
/// <param name="node">The node to crash.</param>
/// <returns>A task that represents the asynchronous termination.</returns>
public static Task CrashNode(ActorSystem node) => node.Terminate();
/// <summary>Waits until the given system sees exactly the given number of Up members and none unreachable.</summary>
/// <param name="system">The actor system whose cluster view is polled.</param>
/// <param name="count">The expected number of Up members.</param>
/// <param name="timeout">The maximum time to wait.</param>
/// <returns>A task that completes when the expected membership is reached, or throws <see cref="TimeoutException"/> on timeout.</returns>
public static async Task WaitForMembersUp(ActorSystem system, int count, TimeSpan timeout)
{
var cluster = Akka.Cluster.Cluster.Get(system);
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) == count
&& !cluster.State.Unreachable.Any())
return;
await Task.Delay(200);
}
throw new TimeoutException(
$"Cluster on {cluster.SelfAddress} did not reach {count} Up members within {timeout}. " +
$"Members: [{string.Join(", ", cluster.State.Members)}], " +
$"Unreachable: [{string.Join(", ", cluster.State.Unreachable)}]");
}
/// <summary>Waits until the given address is neither a member nor unreachable in the survivor's cluster view.</summary>
/// <param name="survivor">The actor system whose cluster view is polled.</param>
/// <param name="removed">The address expected to disappear from the view.</param>
/// <param name="timeout">The maximum time to wait.</param>
/// <returns>A task that completes when the member is gone, or throws <see cref="TimeoutException"/> on timeout.</returns>
public static async Task WaitForMemberRemoved(ActorSystem survivor, Address removed, TimeSpan timeout)
{
var cluster = Akka.Cluster.Cluster.Get(survivor);
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (cluster.State.Members.All(m => m.Address != removed)
&& cluster.State.Unreachable.All(m => m.Address != removed))
return;
await Task.Delay(200);
}
throw new TimeoutException($"Member {removed} was never removed from {cluster.SelfAddress}'s view — SBR did not down it.");
}
/// <summary>Finds a free loopback TCP port.</summary>
/// <returns>A port number that was free when probed.</returns>
public static int GetFreeTcpPort()
{
var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
listener.Start();
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
return port;
}
/// <summary>Releases the resources used by this instance asynchronously.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
foreach (var sys in new[] { NodeA, NodeB })
{
if (sys is { WhenTerminated.IsCompleted: false })
{
try { await sys.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
}
}
}
}