using Akka.Actor;
using Xunit;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
///
/// ScadaBridge#106: the in-repo (strategy
/// leader-auto-down) must give a REAL two-node cluster exactly the outcomes Akka's
/// AutoDowning (auto-down) gives in : a hard crash
/// of EITHER node — the oldest included — is downed and removed by the survivor and the
/// singleton ends up answering on the survivor. The two failover cases are clones of
/// SbrFailoverTests.AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton and
/// …AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton, differing only in the strategy.
/// Under a provider that never downs, the member-removal step times out (the crashed member
/// stays Unreachable forever) — that is what gives them teeth.
///
public class LeaderAutoDownFailoverTests
{
private const string Strategy = "leader-auto-down";
///
/// The configured provider is the one Akka actually instantiated and started: the Cluster
/// extension holds a and its downing actor is running
/// under the cluster daemon on both nodes.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task LeaderAutoDown_ProviderIsWhatAkkaLoads_OnBothNodes()
{
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
foreach (var node in new[] { cluster.NodeA, cluster.NodeB })
{
var akkaCluster = Akka.Cluster.Cluster.Get(node);
Assert.IsType(akkaCluster.DowningProvider);
// ClusterCoreDaemon creates the provider's DowningActorProps as "downingProvider".
var identity = await node.ActorSelection("/system/cluster/core/daemon/downingProvider")
.Ask(new Identify("downing"), TimeSpan.FromSeconds(5));
Assert.NotNull(identity.Subject);
}
}
///
/// With leader-auto-down, a hard crash of the OLDEST node lets the younger survivor down it
/// and take over the singleton — the direction two-node keep-oldest cannot survive.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task LeaderAutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
{
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
SbrFailoverTests.StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
var (_, proxyB) = SbrFailoverTests.StartSingleton(cluster.NodeB);
var echo = await proxyB.Ask("ping", TimeSpan.FromSeconds(20));
Assert.Equal("ping", echo);
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
await TwoNodeClusterFixture.CrashNode(cluster.NodeA);
// 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member.
await TwoNodeClusterFixture.WaitForMemberRemoved(
cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));
// 2) B must still be a functioning cluster member (not self-downed) …
var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");
// 3) … and the singleton must migrate to B and answer again.
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try
{
var echo2 = await proxyB.Ask("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
Assert.Equal("ping-after-oldest-crash", echo2);
return;
}
catch (Exception ex) { last = ex; }
}
throw new Xunit.Sdk.XunitException(
$"Singleton never migrated to the younger survivor after the oldest crashed under leader-auto-down: {last}");
}
/// With leader-auto-down, a hard crash of the younger node leaves the singleton on the oldest node.
/// A task that represents the asynchronous operation.
[Fact]
public async Task LeaderAutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
{
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
var (_, proxyA) = SbrFailoverTests.StartSingleton(cluster.NodeA);
SbrFailoverTests.StartSingleton(cluster.NodeB);
Assert.Equal("ping", await proxyA.Ask("ping", TimeSpan.FromSeconds(20)));
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
await TwoNodeClusterFixture.WaitForMemberRemoved(
cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
Assert.False(Akka.Cluster.Cluster.Get(cluster.NodeA).IsTerminated,
"surviving oldest's Cluster extension terminated — it downed itself");
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try
{
var echo2 = await proxyA.Ask("ping-after-crash", TimeSpan.FromSeconds(3));
Assert.Equal("ping-after-crash", echo2);
return;
}
catch (Exception ex) { last = ex; }
}
throw new Xunit.Sdk.XunitException(
$"Singleton stopped answering on the surviving oldest node under leader-auto-down: {last}");
}
}