feat(redundancy): manual failover service — graceful Leave of the driver Primary
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,54 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
|
||||
/// <summary>
|
||||
/// A read-only view of the cluster's driver membership, as the manual-failover control sees it.
|
||||
/// </summary>
|
||||
/// <param name="PrimaryAddress">
|
||||
/// The current driver Primary — the oldest Up <c>driver</c> member, the same node
|
||||
/// <see cref="RedundancyStateActor.SelectDriverPrimary"/> names — or <see langword="null"/> when
|
||||
/// there is no Up driver member at all.
|
||||
/// </param>
|
||||
/// <param name="DriverAddresses">
|
||||
/// Every Up <c>driver</c> member, oldest first. A manual failover needs at least two: with one
|
||||
/// there is nothing to fail over to.
|
||||
/// </param>
|
||||
public sealed record ManualFailoverSnapshot(string? PrimaryAddress, IReadOnlyList<string> DriverAddresses)
|
||||
{
|
||||
/// <summary>Whether a failover can be performed — i.e. a peer exists to take over.</summary>
|
||||
public bool CanFailOver => DriverAddresses.Count >= 2;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Operator-initiated, graceful swap of the driver Primary.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The mechanism is a cluster <b>Leave</b> of the current Primary, never a <c>Down</c>. The
|
||||
/// leaving node hands its singletons over through the normal cluster-leave phases,
|
||||
/// <c>CoordinatedShutdown</c> runs, <c>ActorSystemTerminationWatchdog</c> exits the process,
|
||||
/// the service supervisor restarts it, and it rejoins as the youngest member — so the
|
||||
/// survivor becomes the oldest Up driver member and therefore Primary. ServiceLevel 250
|
||||
/// moves with it and OPC UA clients re-select.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mesh-scope caveat.</b> Until the per-cluster mesh work lands
|
||||
/// (<c>docs/plans/2026-07-21-per-cluster-mesh-design.md</c>) the Primary is elected once per
|
||||
/// Akka mesh, not per application <c>Cluster</c> row. On a fleet running several application
|
||||
/// clusters in one mesh this acts on the whole mesh's Primary. The UI says so.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IManualFailoverService
|
||||
{
|
||||
/// <summary>Reads the current driver membership from live cluster state.</summary>
|
||||
ManualFailoverSnapshot GetSnapshot();
|
||||
|
||||
/// <summary>
|
||||
/// Gracefully removes the current driver Primary from the cluster so its peer takes over.
|
||||
/// </summary>
|
||||
/// <param name="actor">The authenticated identity performing the failover; recorded in the audit event.</param>
|
||||
/// <returns>
|
||||
/// The address of the node asked to leave, or <see langword="null"/> when the failover was
|
||||
/// refused because no driver peer exists (in which case nothing was changed and nothing audited).
|
||||
/// </returns>
|
||||
Task<string?> FailOverDriverPrimaryAsync(string actor);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
|
||||
/// <summary>
|
||||
/// Default <see cref="IManualFailoverService"/> — a graceful cluster <c>Leave</c> of the oldest Up
|
||||
/// <c>driver</c> member. See the interface for the full behaviour and the mesh-scope caveat.
|
||||
/// </summary>
|
||||
public sealed class ManualFailoverService : IManualFailoverService
|
||||
{
|
||||
/// <summary>The audit <see cref="AuditEvent.Action"/> stamped onto every manual failover.</summary>
|
||||
public const string AuditAction = "cluster.manual-failover";
|
||||
|
||||
/// <summary>The audit <see cref="AuditEvent.Category"/> stamped onto every manual failover.</summary>
|
||||
public const string AuditCategory = "Redundancy";
|
||||
|
||||
private readonly Func<ActorSystem> _system;
|
||||
private readonly IAuditWriter _audit;
|
||||
private readonly ILogger<ManualFailoverService> _log;
|
||||
|
||||
/// <summary>
|
||||
/// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance:
|
||||
/// this is registered before Akka's hosted service has built the system, so resolving eagerly
|
||||
/// would race startup (the same lazy-accessor pattern <c>ActorSystemTerminationWatchdog</c> uses).
|
||||
/// </summary>
|
||||
/// <param name="system">Lazy accessor for the node's ActorSystem.</param>
|
||||
/// <param name="audit">The shared audit seam; a failover is recorded before it is performed.</param>
|
||||
/// <param name="log">Logger for the failover decision.</param>
|
||||
public ManualFailoverService(Func<ActorSystem> system, IAuditWriter audit, ILogger<ManualFailoverService> log)
|
||||
{
|
||||
_system = system ?? throw new ArgumentNullException(nameof(system));
|
||||
_audit = audit ?? throw new ArgumentNullException(nameof(audit));
|
||||
_log = log ?? throw new ArgumentNullException(nameof(log));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system());
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<string?> FailOverDriverPrimaryAsync(string actor)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(actor);
|
||||
|
||||
var system = _system();
|
||||
|
||||
// Dry run first so the peer guard is evaluated before anything is audited: a refused
|
||||
// failover changed nothing and must not leave an audit row implying it did.
|
||||
var target = FailOverCore(system, dryRun: true);
|
||||
if (target is null)
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.",
|
||||
actor);
|
||||
return null;
|
||||
}
|
||||
|
||||
await _audit.WriteAsync(new AuditEvent
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
OccurredAtUtc = DateTimeOffset.UtcNow,
|
||||
Actor = actor,
|
||||
Action = AuditAction,
|
||||
Category = AuditCategory,
|
||||
SourceNode = target.ToString(),
|
||||
Outcome = AuditOutcome.Success,
|
||||
DetailsJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
target = target.ToString(),
|
||||
driverMembers = SnapshotCore(system).DriverAddresses,
|
||||
}),
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
_log.LogWarning(
|
||||
"Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. "
|
||||
+ "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest "
|
||||
+ "member; its peer becomes Primary and advertises the primary ServiceLevel.",
|
||||
actor,
|
||||
target);
|
||||
|
||||
FailOverCore(system, dryRun: false);
|
||||
return target.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Selects — and, unless <paramref name="dryRun"/>, gracefully removes — the driver Primary.
|
||||
/// </summary>
|
||||
/// <param name="system">The ActorSystem whose cluster is acted on.</param>
|
||||
/// <param name="dryRun">When <see langword="true"/>, selects the target without leaving.</param>
|
||||
/// <returns>The target address, or <see langword="null"/> when fewer than two Up driver members exist.</returns>
|
||||
public static Address? FailOverCore(ActorSystem system, bool dryRun = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(system);
|
||||
|
||||
var cluster = Akka.Cluster.Cluster.Get(system);
|
||||
|
||||
// Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver,
|
||||
// Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a
|
||||
// failover would remove a node that was not Primary and change nothing. Pinned by
|
||||
// ManualFailoverServiceTests.Parity_with_SelectDriverPrimary.
|
||||
var drivers = OrderedDrivers(cluster.State.Members);
|
||||
|
||||
// A failover needs somewhere to fail over TO. With a single driver member, leaving would be
|
||||
// a shutdown dressed up as a failover.
|
||||
if (drivers.Count < 2) return null;
|
||||
|
||||
var primary = drivers[0];
|
||||
if (!dryRun) cluster.Leave(primary.Address);
|
||||
return primary.Address;
|
||||
}
|
||||
|
||||
private static ManualFailoverSnapshot SnapshotCore(ActorSystem system)
|
||||
{
|
||||
var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members);
|
||||
|
||||
return new ManualFailoverSnapshot(
|
||||
drivers.FirstOrDefault()?.Address.ToString(),
|
||||
drivers.Select(m => m.Address.ToString()).ToList());
|
||||
}
|
||||
|
||||
private static List<Member> OrderedDrivers(IEnumerable<Member> members) => members
|
||||
.Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole))
|
||||
.OrderBy(m => m, Member.AgeOrdering)
|
||||
.ToList();
|
||||
}
|
||||
Reference in New Issue
Block a user