using System.Text.Json;
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Cluster;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
///
/// Default — a graceful cluster Leave of the oldest Up
/// driver member. See the interface for the full behaviour and the mesh-scope caveat.
///
public sealed class ManualFailoverService : IManualFailoverService
{
/// The audit stamped onto every manual failover.
public const string AuditAction = "cluster.manual-failover";
/// The audit stamped onto every manual failover.
public const string AuditCategory = "Redundancy";
private readonly Func _system;
private readonly IAuditWriter _audit;
private readonly ILogger _log;
///
/// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance:
/// this is registered before Akka's hosted service has built the system, so resolving eagerly
/// would race startup (the same lazy-accessor pattern ActorSystemTerminationWatchdog uses).
///
/// Lazy accessor for the node's ActorSystem.
/// The shared audit seam; a failover is recorded before it is performed.
/// Logger for the failover decision.
public ManualFailoverService(Func system, IAuditWriter audit, ILogger log)
{
_system = system ?? throw new ArgumentNullException(nameof(system));
_audit = audit ?? throw new ArgumentNullException(nameof(audit));
_log = log ?? throw new ArgumentNullException(nameof(log));
}
///
public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system());
///
public async Task FailOverDriverPrimaryAsync(string actor)
{
ArgumentException.ThrowIfNullOrWhiteSpace(actor);
var system = _system();
// Dry run first so the peer guard is evaluated before anything is audited: a refused
// failover changed nothing and must not leave an audit row implying it did.
var target = FailOverCore(system, dryRun: true);
if (target is null)
{
_log.LogWarning(
"Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.",
actor);
return null;
}
await _audit.WriteAsync(new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTimeOffset.UtcNow,
Actor = actor,
Action = AuditAction,
Category = AuditCategory,
SourceNode = target.ToString(),
Outcome = AuditOutcome.Success,
DetailsJson = JsonSerializer.Serialize(new
{
target = target.ToString(),
driverMembers = SnapshotCore(system).DriverAddresses,
}),
}).ConfigureAwait(false);
_log.LogWarning(
"Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. "
+ "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest "
+ "member; its peer becomes Primary and advertises the primary ServiceLevel.",
actor,
target);
FailOverCore(system, dryRun: false);
return target.ToString();
}
///
/// Selects — and, unless , gracefully removes — the driver Primary.
///
/// The ActorSystem whose cluster is acted on.
/// When , selects the target without leaving.
/// The target address, or when fewer than two Up driver members exist.
public static Address? FailOverCore(ActorSystem system, bool dryRun = false)
{
ArgumentNullException.ThrowIfNull(system);
var cluster = Akka.Cluster.Cluster.Get(system);
// Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver,
// Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a
// failover would remove a node that was not Primary and change nothing. Pinned by
// ManualFailoverServiceTests.Parity_with_SelectDriverPrimary.
var drivers = OrderedDrivers(cluster.State.Members);
// A failover needs somewhere to fail over TO. With a single driver member, leaving would be
// a shutdown dressed up as a failover.
if (drivers.Count < 2) return null;
var primary = drivers[0];
if (!dryRun) cluster.Leave(primary.Address);
return primary.Address;
}
private static ManualFailoverSnapshot SnapshotCore(ActorSystem system)
{
var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members);
return new ManualFailoverSnapshot(
drivers.FirstOrDefault()?.Address.ToString(),
drivers.Select(m => m.Address.ToString()).ToList());
}
private static List OrderedDrivers(IEnumerable members) => members
// GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
// 'Driver' (rolling-upgrade window) must still be a manual-failover candidate.
.Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, RedundancyStateActor.DriverRole))
.OrderBy(m => m, Member.AgeOrdering)
.ToList();
}