using Akka.Actor;
using Akka.Cluster.Tools.Singleton;
using Xunit;
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
///
/// Behavioral proof that the downing provider enabled in
/// AkkaHostedService.BuildHocon (arch-review 01 Critical) is actually active:
/// after a hard crash, the surviving node DOWNS and REMOVES the crashed member. Under
/// the pre-fix Akka default (NoDowning) the crashed member lingers Unreachable
/// forever, so the member-removal assertions here are impossible to satisfy without the
/// fix — that is what gives the tests teeth.
///
/// TWO-NODE SEMANTICS (verified against Akka.NET 1.5.62 source + live on the docker
/// rig, 2026-07-21):
///
/// - keep-oldest — downs the side that does NOT contain the oldest member,
/// and its down-if-alone escape only fires when the surviving side has ≥2
/// members (KeepOldest.OldestDecision: otherSide == 1 && thisSide >= 2).
/// With 1-vs-1 the younger survivor therefore takes DownReachable — it downs
/// ITSELF — so only a YOUNGER-node crash is survivable.
/// - auto-down (production default, decision 2026-07-21) — the leader among
/// the reachable members downs the unreachable peer after the stability window, so a
/// crash of EITHER node fails over to the survivor; the accepted trade is dual-active
/// during a real network partition.
///
///
public class SbrFailoverTests
{
private sealed class EchoActor : ReceiveActor
{
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
}
private static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys)
{
var manager = sys.ActorOf(ClusterSingletonManager.Props(
Props.Create(() => new EchoActor()),
PoisonPill.Instance,
ClusterSingletonManagerSettings.Create(sys).WithSingletonName("failover-probe")),
"failover-probe-singleton");
var proxy = sys.ActorOf(ClusterSingletonProxy.Props(
"/user/failover-probe-singleton",
ClusterSingletonProxySettings.Create(sys).WithSingletonName("failover-probe")),
"failover-probe-proxy");
return (manager, proxy);
}
[Fact]
public async Task HardCrashOfYoungerNode_SbrDownsIt_AndOldestKeepsSingleton()
{
// Pinned to keep-oldest: this is the SBR path's (only) survivable direction.
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "keep-oldest");
var (_, proxyA) = StartSingleton(cluster.NodeA); // oldest hosts the singleton
StartSingleton(cluster.NodeB);
// Singleton is reachable from A (the oldest / singleton host).
var echo = await proxyA.Ask("ping", TimeSpan.FromSeconds(20));
Assert.Equal("ping", echo);
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
// 1) SBR must DOWN and REMOVE the crashed younger member (NoDowning => this
// times out; the member stays unreachable forever).
// Budget: failure detection (~2s) + stable-after (3s) + gossip margin.
await TwoNodeClusterFixture.WaitForMemberRemoved(
cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
// 2) The oldest survivor stays up and its singleton keeps answering.
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try
{
var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3));
Assert.Equal("ping-after-crash", echo2);
return;
}
catch (Exception ex) { last = ex; }
}
throw new Xunit.Sdk.XunitException($"Singleton stopped answering on the surviving oldest node after SBR downing: {last}");
}
[Fact]
public async Task AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
{
// Decision 2026-07-21: the direction two-node keep-oldest can NEVER survive
// (proven live on the docker rig — the younger survivor took DownReachable and
// self-downed). Under auto-down the survivor must instead down the crashed
// oldest and TAKE OVER its singleton.
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
var (_, proxyB) = StartSingleton(cluster.NodeB);
// Singleton reachable from B while A is alive (proxy routes to the oldest).
var echo = await proxyB.Ask("ping", TimeSpan.FromSeconds(20));
Assert.Equal("ping", echo);
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
await TwoNodeClusterFixture.CrashNode(cluster.NodeA);
// 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member —
// the exact step keep-oldest refuses (it downs itself instead).
await TwoNodeClusterFixture.WaitForMemberRemoved(
cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));
// 2) B must still be a functioning cluster member (not self-downed) …
var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");
// 3) … and the singleton must migrate to B and answer again.
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try
{
var echo2 = await proxyB.Ask("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
Assert.Equal("ping-after-oldest-crash", echo2);
return;
}
catch (Exception ex) { last = ex; }
}
throw new Xunit.Sdk.XunitException(
$"Singleton never migrated to the younger survivor after the oldest crashed under auto-down: {last}");
}
[Fact]
public async Task AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
{
// The previously-survivable direction must STAY survivable under auto-down.
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: "auto-down");
var (_, proxyA) = StartSingleton(cluster.NodeA);
StartSingleton(cluster.NodeB);
Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(20)));
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
await TwoNodeClusterFixture.WaitForMemberRemoved(
cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try
{
var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3));
Assert.Equal("ping-after-crash", echo2);
return;
}
catch (Exception ex) { last = ex; }
}
throw new Xunit.Sdk.XunitException(
$"Singleton stopped answering on the surviving oldest node under auto-down: {last}");
}
}