using System.Text.Json; using Akka.Actor; using Akka.Cluster; using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.CentralUI.Services; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Host.Actors; namespace ZB.MOM.WW.ScadaBridge.Host.Health; /// /// Production backed by the running Akka.NET cluster. /// Registered only in the Central-role branch of Program.cs. /// /// Leave, never Down. The target is asked to leave gracefully, so /// ClusterSingletonManager hands its singletons to the survivor before the member is /// removed. A Down would skip that hand-off and leave the pair to the downing strategy /// — the wrong tool for a deliberate, planned role swap. /// /// The target is the oldest Up member, not the leader. That mirrors /// ActiveNodeEvaluator's rule, which is where the singletons actually live; Akka's /// cluster leadership is address-ordered and diverges from singleton placement after a /// restart (review 01 [High]). /// /// Audit before acting. The row is written before the Leave is issued. The node /// serving this call is usually NOT the one leaving, but it can be (an admin routed to the /// active node fails that node over), and an audit written afterwards could be lost to the /// very shutdown it describes. /// public sealed class AkkaManualFailoverService : IManualFailoverService { private readonly AkkaHostedService _akka; private readonly ICentralAuditWriter _audit; private readonly ILogger _logger; /// Initializes a new . /// The Akka hosted service exposing the cluster's actor system. /// Central direct-write audit writer. /// Logger. public AkkaManualFailoverService( AkkaHostedService akka, ICentralAuditWriter audit, ILogger logger) { _akka = akka; _audit = audit; _logger = logger; } /// public async Task FailOverCentralAsync(string actor) { var system = _akka.GetOrCreateActorSystem(); // Resolve first so the audit row can name the target, and so the peer guard rejects // before anything observable happens. var target = FailOverCore(system, role: CentralRole, dryRun: true); if (target is null) { _logger.LogWarning( "Manual failover requested by {Actor} but refused: fewer than 2 Up '{Role}' members, " + "so there is no standby to take over.", actor, CentralRole); return null; } await WriteAuditAsync(actor, target); _logger.LogWarning( "Manual failover triggered by {Actor}: {Target} is leaving the cluster gracefully; " + "its singletons hand over to the standby, it restarts via its supervisor and rejoins " + "as the youngest member.", actor, target); FailOverCore(system, role: CentralRole); return target.ToString(); } /// The Akka role scoping central-cluster membership. private const string CentralRole = "Central"; /// /// Oldest Up member with the role leaves — mirrors ActiveNodeEvaluator's oldest-Up /// rule so the node acted on is exactly the one hosting the singletons. Returns null when /// fewer than 2 Up members carry the role (no peer = failover would be an outage). /// /// The actor system whose cluster is acted on. /// Role scope for membership. /// When true, resolve and return the target without issuing the Leave. /// The address that leaves (or would leave), or null when there is no peer. public static Address? FailOverCore(ActorSystem system, string role, bool dryRun = false) { var cluster = Akka.Cluster.Cluster.Get(system); var withRole = cluster.State.Members .Where(m => m.Status == MemberStatus.Up && m.HasRole(role)) .OrderBy(m => m, Member.AgeOrdering) .ToList(); if (withRole.Count < 2) { return null; } var oldest = withRole[0]; if (!dryRun) { cluster.Leave(oldest.Address); } return oldest.Address; } /// /// Best-effort audit row. Audit failure must never block the failover the admin asked for /// — the same rule the rest of the Audit Log follows (audit is best-effort; the /// user-facing action's own success path is authoritative). /// private async Task WriteAuditAsync(string actor, Address target) { try { var evt = ScadaBridgeAuditEventFactory.Create( channel: AuditChannel.Cluster, kind: AuditKind.ManualFailover, status: AuditStatus.Submitted, actor: actor, target: target.ToString(), extra: JsonSerializer.Serialize(new { target = target.ToString(), role = CentralRole })); await _audit.WriteAsync(evt); } catch (Exception ex) { _logger.LogWarning( ex, "Best-effort manual-failover audit emission failed (actor={Actor}, target={Target}); " + "the failover itself proceeds.", actor, target); } } }