using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
using ZB.MOM.WW.ScadaBridge.Host;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
using ZB.MOM.WW.ScadaBridge.Host.Health;
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
///
/// Cluster-level proof of the admin "Trigger failover" control: the current active node
/// (the OLDEST Up member — 's rule, which is where
/// ClusterSingletonManager places singletons) leaves GRACEFULLY, so its singletons hand over
/// rather than being killed, and the survivor becomes the active node.
///
/// Graceful Leave, never Down: a Down would skip singleton hand-off and
/// leave the pair to the downing strategy. The peer guard matters just as much — failing over
/// a single-node cluster is not a failover, it is an outage, so the service refuses.
///
public sealed class ManualFailoverTests : IAsyncDisposable
{
private readonly List _systems = new();
/// Starts a one-node cluster (self-first seed list, so it forms alone).
private ActorSystem StartSoloNode()
{
var port = TwoNodeClusterFixture.GetFreeTcpPort();
var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = port };
var clusterOptions = new ClusterOptions
{
SeedNodes = new List { $"akka.tcp://scadabridge@127.0.0.1:{port}" },
AllowSingleNodeCluster = true,
StableAfter = TimeSpan.FromSeconds(3),
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
MinNrOfMembers = 1,
};
var hocon = AkkaHostedService.BuildHocon(
nodeOptions, clusterOptions, new[] { "Central" },
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3));
var system = ActorSystem.Create("scadabridge", ConfigurationFactory.ParseString(hocon));
_systems.Add(system);
return system;
}
[Fact]
public async Task Failover_makes_the_oldest_leave_and_the_survivor_take_over()
{
await using var f = await TwoNodeClusterFixture.StartAsync();
var oldest = Akka.Cluster.Cluster.Get(f.NodeA); // NodeA started first = oldest
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central"),
"precondition: NodeA must be the active (oldest Up) node before the failover");
// Issued from the OTHER node, exactly as the UI does it (the button is served by
// whichever node Traefik routed the admin to).
var target = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central");
Assert.Equal(oldest.SelfAddress, target);
// Graceful exit path: the left node's own ActorSystem terminates…
await f.NodeA.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30));
// …and the survivor becomes a 1-member cluster and the oldest-Up active node.
await TwoNodeClusterFixture.WaitForMemberRemoved(f.NodeB, oldest.SelfAddress, TimeSpan.FromSeconds(30));
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(f.NodeB), "Central"));
}
[Fact]
public async Task Failover_refuses_when_no_peer_exists()
{
var solo = StartSoloNode();
await TwoNodeClusterFixture.WaitForMembersUp(solo, 1, TimeSpan.FromSeconds(30));
var target = AkkaManualFailoverService.FailOverCore(solo, "Central");
Assert.Null(target);
// Positive assert: the refusal left the node running and still active — the guard
// must not have half-issued a Leave.
await Task.Delay(TimeSpan.FromSeconds(2));
Assert.Equal(MemberStatus.Up, Akka.Cluster.Cluster.Get(solo).SelfMember.Status);
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(solo), "Central"));
}
[Fact]
public async Task Failover_dry_run_names_the_target_without_moving_the_cluster()
{
// The service resolves the target first (to audit it BEFORE acting); that probe must
// not itself perturb the cluster.
await using var f = await TwoNodeClusterFixture.StartAsync();
var oldest = Akka.Cluster.Cluster.Get(f.NodeA);
var probed = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central", dryRun: true);
Assert.Equal(oldest.SelfAddress, probed);
await Task.Delay(TimeSpan.FromSeconds(3));
Assert.Equal(2, Akka.Cluster.Cluster.Get(f.NodeB).State.Members.Count(m => m.Status == MemberStatus.Up));
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central"));
}
public async ValueTask DisposeAsync()
{
foreach (var s in _systems)
{
if (s is { WhenTerminated.IsCompleted: false })
{
try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
}
}
}
}