feat(cluster): manual central failover service — graceful Leave of the oldest Up member
Admin-triggered failover of the central pair. IManualFailoverService is declared in CentralUI (plain strings, so that project stays Akka-free); AkkaManualFailoverService implements it in the Host and is registered only in the Central branch. - Leave, never Down: singletons hand over instead of being killed. - Target = oldest Up member with the Central role, mirroring ActiveNodeEvaluator, so the node acted on is exactly the one hosting the singletons (never the leader, whose address-ordered definition diverges from singleton placement after a restart). - Peer guard: returns null when fewer than 2 Up Central members — failing over a lone node is an outage, not a failover. - Audited BEFORE the Leave is issued via ICentralAuditWriter: the acting node can be the one that goes away, and an audit written after could be lost to the shutdown it describes. Best-effort — audit failure never blocks the failover. New audit taxonomy: AuditChannel.Cluster + AuditKind.ManualFailover (operator-initiated topology actions are not script trust-boundary crossings, but are exactly what an audit log exists to attribute). Lock-in tests updated 5->6 channels, 16->17 kinds. alog.md §4 updated per the lock-in tests' contract. Both tables were already stale -- the Channel row omitted SecuredWrite and the Kind table claimed 10 while the code had 16 -- so they are completed here, not merely appended to. ManualFailoverTests: 3 real-cluster tests (oldest leaves + survivor takes over, peer guard refuses with a positive still-running assert, dry-run probe does not perturb).
This commit is contained in:
Joseph Doherty
@@ -12,7 +12,7 @@ public class AuditEnumTests
|
||||
[Fact]
|
||||
public void AuditChannel_HasExactlyExpectedMembers()
|
||||
{
|
||||
var expected = new[] { "ApiOutbound", "DbOutbound", "Notification", "ApiInbound", "SecuredWrite" };
|
||||
var expected = new[] { "ApiOutbound", "DbOutbound", "Notification", "ApiInbound", "SecuredWrite", "Cluster" };
|
||||
var actual = Enum.GetValues(typeof(AuditChannel))
|
||||
.Cast<AuditChannel>()
|
||||
.Select(x => x.ToString())
|
||||
@@ -23,7 +23,7 @@ public class AuditEnumTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AuditKind_HasExactlySixteenExpectedMembers()
|
||||
public void AuditKind_HasExactlySeventeenExpectedMembers()
|
||||
{
|
||||
var expected = new[]
|
||||
{
|
||||
@@ -33,13 +33,14 @@ public class AuditEnumTests
|
||||
"SecuredWriteSubmit", "SecuredWriteApprove", "SecuredWriteReject", "SecuredWriteExecute",
|
||||
"SecuredWriteExpire",
|
||||
"ReconciliationAbandoned",
|
||||
"ManualFailover",
|
||||
};
|
||||
var actual = Enum.GetValues(typeof(AuditKind))
|
||||
.Cast<AuditKind>()
|
||||
.Select(x => x.ToString())
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(16, actual.Length);
|
||||
Assert.Equal(17, actual.Length);
|
||||
Assert.Equal(expected, actual);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Cluster-level proof of the admin "Trigger failover" control: the current active node
|
||||
/// (the OLDEST Up member — <see cref="ActiveNodeEvaluator"/>'s rule, which is where
|
||||
/// ClusterSingletonManager places singletons) leaves GRACEFULLY, so its singletons hand over
|
||||
/// rather than being killed, and the survivor becomes the active node.
|
||||
///
|
||||
/// <para>Graceful <c>Leave</c>, never <c>Down</c>: a Down would skip singleton hand-off and
|
||||
/// leave the pair to the downing strategy. The peer guard matters just as much — failing over
|
||||
/// a single-node cluster is not a failover, it is an outage, so the service refuses.</para>
|
||||
/// </summary>
|
||||
public sealed class ManualFailoverTests : IAsyncDisposable
|
||||
{
|
||||
private readonly List<ActorSystem> _systems = new();
|
||||
|
||||
/// <summary>Starts a one-node cluster (self-first seed list, so it forms alone).</summary>
|
||||
private ActorSystem StartSoloNode()
|
||||
{
|
||||
var port = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = port };
|
||||
var clusterOptions = new ClusterOptions
|
||||
{
|
||||
SeedNodes = new List<string> { $"akka.tcp://scadabridge@127.0.0.1:{port}" },
|
||||
AllowSingleNodeCluster = true,
|
||||
StableAfter = TimeSpan.FromSeconds(3),
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
|
||||
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
|
||||
MinNrOfMembers = 1,
|
||||
};
|
||||
var hocon = AkkaHostedService.BuildHocon(
|
||||
nodeOptions, clusterOptions, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3));
|
||||
var system = ActorSystem.Create("scadabridge", ConfigurationFactory.ParseString(hocon));
|
||||
_systems.Add(system);
|
||||
return system;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_makes_the_oldest_leave_and_the_survivor_take_over()
|
||||
{
|
||||
await using var f = await TwoNodeClusterFixture.StartAsync();
|
||||
var oldest = Akka.Cluster.Cluster.Get(f.NodeA); // NodeA started first = oldest
|
||||
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central"),
|
||||
"precondition: NodeA must be the active (oldest Up) node before the failover");
|
||||
|
||||
// Issued from the OTHER node, exactly as the UI does it (the button is served by
|
||||
// whichever node Traefik routed the admin to).
|
||||
var target = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central");
|
||||
|
||||
Assert.Equal(oldest.SelfAddress, target);
|
||||
|
||||
// Graceful exit path: the left node's own ActorSystem terminates…
|
||||
await f.NodeA.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
// …and the survivor becomes a 1-member cluster and the oldest-Up active node.
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(f.NodeB, oldest.SelfAddress, TimeSpan.FromSeconds(30));
|
||||
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(f.NodeB), "Central"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_refuses_when_no_peer_exists()
|
||||
{
|
||||
var solo = StartSoloNode();
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(solo, 1, TimeSpan.FromSeconds(30));
|
||||
|
||||
var target = AkkaManualFailoverService.FailOverCore(solo, "Central");
|
||||
|
||||
Assert.Null(target);
|
||||
// Positive assert: the refusal left the node running and still active — the guard
|
||||
// must not have half-issued a Leave.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
Assert.Equal(MemberStatus.Up, Akka.Cluster.Cluster.Get(solo).SelfMember.Status);
|
||||
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(solo), "Central"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_dry_run_names_the_target_without_moving_the_cluster()
|
||||
{
|
||||
// The service resolves the target first (to audit it BEFORE acting); that probe must
|
||||
// not itself perturb the cluster.
|
||||
await using var f = await TwoNodeClusterFixture.StartAsync();
|
||||
var oldest = Akka.Cluster.Cluster.Get(f.NodeA);
|
||||
|
||||
var probed = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central", dryRun: true);
|
||||
|
||||
Assert.Equal(oldest.SelfAddress, probed);
|
||||
await Task.Delay(TimeSpan.FromSeconds(3));
|
||||
Assert.Equal(2, Akka.Cluster.Cluster.Get(f.NodeB).State.Members.Count(m => m.Status == MemberStatus.Up));
|
||||
Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central"));
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (var s in _systems)
|
||||
{
|
||||
if (s is { WhenTerminated.IsCompleted: false })
|
||||
{
|
||||
try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user