using System.Text.Json;
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
///
/// Production backed by the running Akka.NET cluster.
/// Registered only in the Central-role branch of Program.cs.
///
/// Leave, never Down. The target is asked to leave gracefully, so
/// ClusterSingletonManager hands its singletons to the survivor before the member is
/// removed. A Down would skip that hand-off and leave the pair to the downing strategy
/// — the wrong tool for a deliberate, planned role swap.
///
/// The target is the oldest Up member, not the leader. That mirrors
/// ActiveNodeEvaluator's rule, which is where the singletons actually live; Akka's
/// cluster leadership is address-ordered and diverges from singleton placement after a
/// restart (review 01 [High]).
///
/// Audit before acting. The row is written before the Leave is issued. The node
/// serving this call is usually NOT the one leaving, but it can be (an admin routed to the
/// active node fails that node over), and an audit written afterwards could be lost to the
/// very shutdown it describes.
///
public sealed class AkkaManualFailoverService : IManualFailoverService
{
private readonly AkkaHostedService _akka;
private readonly ICentralAuditWriter _audit;
private readonly CommunicationService _communication;
private readonly ILogger _logger;
/// Initializes a new .
/// The Akka hosted service exposing the cluster's actor system.
/// Central direct-write audit writer.
/// Central→site command/control transport, used for site failover.
/// Logger.
public AkkaManualFailoverService(
AkkaHostedService akka,
ICentralAuditWriter audit,
CommunicationService communication,
ILogger logger)
{
_akka = akka;
_audit = audit;
_communication = communication;
_logger = logger;
}
///
public async Task FailOverCentralAsync(string actor)
{
var system = _akka.GetOrCreateActorSystem();
// Resolve first so the audit row can name the target, and so the peer guard rejects
// before anything observable happens.
var target = FailOverCore(system, role: CentralRole, dryRun: true);
if (target is null)
{
_logger.LogWarning(
"Manual failover requested by {Actor} but refused: fewer than 2 Up '{Role}' members, "
+ "so there is no standby to take over.", actor, CentralRole);
return null;
}
await WriteAuditAsync(actor, target);
_logger.LogWarning(
"Manual failover triggered by {Actor}: {Target} is leaving the cluster gracefully; "
+ "its singletons hand over to the standby, it restarts via its supervisor and rejoins "
+ "as the youngest member.", actor, target);
FailOverCore(system, role: CentralRole);
return target.ToString();
}
///
public async Task FailOverSiteAsync(string siteId, string actor)
{
// Audit BEFORE relaying, for the same reason as central: the row must exist even if
// the outcome is never observed. Unlike central, the acting node is never the one
// leaving — but a relay can still time out, and an un-acked request that DID take
// effect at the site would otherwise be unattributed.
await WriteAuditAsync(actor, target: siteId, sourceSiteId: siteId);
_logger.LogWarning(
"Manual site failover triggered by {Actor} for site {SiteId}; relaying to the site cluster.",
actor, siteId);
try
{
var ack = await _communication.TriggerSiteFailoverAsync(siteId, Guid.NewGuid().ToString());
return new SiteFailoverOutcome(ack.Accepted, ack.TargetAddress, ack.ErrorMessage);
}
catch (Exception ex)
{
// Central never buffers for an unreachable site — the Ask simply times out. Report
// that distinctly from a refusal, which is a definitive answer FROM the site.
_logger.LogWarning(ex,
"Manual site failover for {SiteId} did not get an ack from the site.", siteId);
return new SiteFailoverOutcome(
Accepted: false,
TargetAddress: null,
ErrorMessage: $"Site did not respond: {ex.Message}");
}
}
/// The Akka role scoping central-cluster membership.
private const string CentralRole = "Central";
///
/// Oldest Up member with the role leaves. Delegates to
/// , which lives in Communication so
/// the site-pair failover path (inside SiteCommunicationActor, which cannot reference
/// Host) shares one implementation of the rule.
///
/// The actor system whose cluster is acted on.
/// Role scope for membership.
/// When true, resolve and return the target without issuing the Leave.
/// The address that leaves (or would leave), or null when there is no peer.
public static Address? FailOverCore(ActorSystem system, string role, bool dryRun = false)
=> ClusterFailoverCoordinator.FailOverOldest(system, role, dryRun);
///
/// Best-effort audit row. Audit failure must never block the failover the admin asked for
/// — the same rule the rest of the Audit Log follows (audit is best-effort; the
/// user-facing action's own success path is authoritative).
///
private Task WriteAuditAsync(string actor, Address target)
=> WriteAuditAsync(actor, target.ToString(), sourceSiteId: null);
///
private async Task WriteAuditAsync(string actor, string target, string? sourceSiteId)
{
try
{
var evt = ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.Cluster,
kind: AuditKind.ManualFailover,
status: AuditStatus.Submitted,
actor: actor,
target: target,
sourceSiteId: sourceSiteId,
extra: JsonSerializer.Serialize(new
{
target,
// Central rows name the Central role; site rows name the site, so a query
// can tell which pair an operator moved.
scope = sourceSiteId is null ? CentralRole : $"site-{sourceSiteId}"
}));
await _audit.WriteAsync(evt);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Best-effort manual-failover audit emission failed (actor={Actor}, target={Target}); "
+ "the failover itself proceeds.", actor, target);
}
}
}