using System.Text.Json; using Akka.Actor; using Akka.Cluster; using Microsoft.Extensions.Logging; using ZB.MOM.WW.Audit; using ZB.MOM.WW.OtOpcUa.Cluster; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; /// /// Default — a graceful cluster Leave of the oldest Up /// driver member. See the interface for the full behaviour and the mesh-scope caveat. /// public sealed class ManualFailoverService : IManualFailoverService { /// The audit stamped onto every manual failover. public const string AuditAction = "cluster.manual-failover"; /// The audit stamped onto every manual failover. public const string AuditCategory = "Redundancy"; private readonly Func _system; private readonly IAuditWriter _audit; private readonly ILogger _log; /// /// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance: /// this is registered before Akka's hosted service has built the system, so resolving eagerly /// would race startup (the same lazy-accessor pattern ActorSystemTerminationWatchdog uses). /// /// Lazy accessor for the node's ActorSystem. /// The shared audit seam; a failover is recorded before it is performed. /// Logger for the failover decision. public ManualFailoverService(Func system, IAuditWriter audit, ILogger log) { _system = system ?? throw new ArgumentNullException(nameof(system)); _audit = audit ?? throw new ArgumentNullException(nameof(audit)); _log = log ?? throw new ArgumentNullException(nameof(log)); } /// public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system()); /// public async Task FailOverDriverPrimaryAsync(string actor) { ArgumentException.ThrowIfNullOrWhiteSpace(actor); var system = _system(); // Dry run first so the peer guard is evaluated before anything is audited: a refused // failover changed nothing and must not leave an audit row implying it did. var target = FailOverCore(system, dryRun: true); if (target is null) { _log.LogWarning( "Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.", actor); return null; } await _audit.WriteAsync(new AuditEvent { EventId = Guid.NewGuid(), OccurredAtUtc = DateTimeOffset.UtcNow, Actor = actor, Action = AuditAction, Category = AuditCategory, SourceNode = target.ToString(), Outcome = AuditOutcome.Success, DetailsJson = JsonSerializer.Serialize(new { target = target.ToString(), driverMembers = SnapshotCore(system).DriverAddresses, }), }).ConfigureAwait(false); _log.LogWarning( "Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. " + "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest " + "member; its peer becomes Primary and advertises the primary ServiceLevel.", actor, target); FailOverCore(system, dryRun: false); return target.ToString(); } /// /// Selects — and, unless , gracefully removes — the driver Primary. /// /// The ActorSystem whose cluster is acted on. /// When , selects the target without leaving. /// The target address, or when fewer than two Up driver members exist. public static Address? FailOverCore(ActorSystem system, bool dryRun = false) { ArgumentNullException.ThrowIfNull(system); var cluster = Akka.Cluster.Cluster.Get(system); // Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver, // Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a // failover would remove a node that was not Primary and change nothing. Pinned by // ManualFailoverServiceTests.Parity_with_SelectDriverPrimary. var drivers = OrderedDrivers(cluster.State.Members); // A failover needs somewhere to fail over TO. With a single driver member, leaving would be // a shutdown dressed up as a failover. if (drivers.Count < 2) return null; var primary = drivers[0]; if (!dryRun) cluster.Leave(primary.Address); return primary.Address; } private static ManualFailoverSnapshot SnapshotCore(ActorSystem system) { var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members); return new ManualFailoverSnapshot( drivers.FirstOrDefault()?.Address.ToString(), drivers.Select(m => m.Address.ToString()).ToList()); } private static List OrderedDrivers(IEnumerable members) => members // GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased // 'Driver' (rolling-upgrade window) must still be a manual-failover candidate. .Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, RedundancyStateActor.DriverRole)) .OrderBy(m => m, Member.AgeOrdering) .ToList(); }