ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Host/Health/AkkaManualFailoverService.cs

using System.Text.Json;
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Host.Actors;

namespace ZB.MOM.WW.ScadaBridge.Host.Health;

/// <summary>
/// Production <see cref="IManualFailoverService"/> backed by the running Akka.NET cluster.
/// Registered only in the Central-role branch of <c>Program.cs</c>.
///
/// <para><b>Leave, never Down.</b> The target is asked to leave gracefully, so
/// <c>ClusterSingletonManager</c> hands its singletons to the survivor before the member is
/// removed. A <c>Down</c> would skip that hand-off and leave the pair to the downing strategy
/// — the wrong tool for a deliberate, planned role swap.</para>
///
/// <para><b>The target is the oldest Up member, not the leader.</b> That mirrors
/// <c>ActiveNodeEvaluator</c>'s rule, which is where the singletons actually live; Akka's
/// cluster leadership is address-ordered and diverges from singleton placement after a
/// restart (review 01 [High]).</para>
///
/// <para><b>Audit before acting.</b> The row is written before the Leave is issued. The node
/// serving this call is usually NOT the one leaving, but it can be (an admin routed to the
/// active node fails that node over), and an audit written afterwards could be lost to the
/// very shutdown it describes.</para>
/// </summary>
public sealed class AkkaManualFailoverService : IManualFailoverService
{
    private readonly AkkaHostedService _akka;
    private readonly ICentralAuditWriter _audit;
    private readonly ILogger<AkkaManualFailoverService> _logger;

    /// <summary>Initializes a new <see cref="AkkaManualFailoverService"/>.</summary>
    /// <param name="akka">The Akka hosted service exposing the cluster's actor system.</param>
    /// <param name="audit">Central direct-write audit writer.</param>
    /// <param name="logger">Logger.</param>
    public AkkaManualFailoverService(
        AkkaHostedService akka,
        ICentralAuditWriter audit,
        ILogger<AkkaManualFailoverService> logger)
    {
        _akka = akka;
        _audit = audit;
        _logger = logger;
    }

    /// <inheritdoc />
    public async Task<string?> FailOverCentralAsync(string actor)
    {
        var system = _akka.GetOrCreateActorSystem();

        // Resolve first so the audit row can name the target, and so the peer guard rejects
        // before anything observable happens.
        var target = FailOverCore(system, role: CentralRole, dryRun: true);
        if (target is null)
        {
            _logger.LogWarning(
                "Manual failover requested by {Actor} but refused: fewer than 2 Up '{Role}' members, "
                + "so there is no standby to take over.", actor, CentralRole);
            return null;
        }

        await WriteAuditAsync(actor, target);

        _logger.LogWarning(
            "Manual failover triggered by {Actor}: {Target} is leaving the cluster gracefully; "
            + "its singletons hand over to the standby, it restarts via its supervisor and rejoins "
            + "as the youngest member.", actor, target);

        FailOverCore(system, role: CentralRole);
        return target.ToString();
    }

    /// <summary>The Akka role scoping central-cluster membership.</summary>
    private const string CentralRole = "Central";

    /// <summary>
    /// Oldest Up member with the role leaves — mirrors <c>ActiveNodeEvaluator</c>'s oldest-Up
    /// rule so the node acted on is exactly the one hosting the singletons. Returns null when
    /// fewer than 2 Up members carry the role (no peer = failover would be an outage).
    /// </summary>
    /// <param name="system">The actor system whose cluster is acted on.</param>
    /// <param name="role">Role scope for membership.</param>
    /// <param name="dryRun">When true, resolve and return the target without issuing the Leave.</param>
    /// <returns>The address that leaves (or would leave), or null when there is no peer.</returns>
    public static Address? FailOverCore(ActorSystem system, string role, bool dryRun = false)
    {
        var cluster = Akka.Cluster.Cluster.Get(system);
        var withRole = cluster.State.Members
            .Where(m => m.Status == MemberStatus.Up && m.HasRole(role))
            .OrderBy(m => m, Member.AgeOrdering)
            .ToList();

        if (withRole.Count < 2)
        {
            return null;
        }

        var oldest = withRole[0];
        if (!dryRun)
        {
            cluster.Leave(oldest.Address);
        }

        return oldest.Address;
    }

    /// <summary>
    /// Best-effort audit row. Audit failure must never block the failover the admin asked for
    /// — the same rule the rest of the Audit Log follows (audit is best-effort; the
    /// user-facing action's own success path is authoritative).
    /// </summary>
    private async Task WriteAuditAsync(string actor, Address target)
    {
        try
        {
            var evt = ScadaBridgeAuditEventFactory.Create(
                channel: AuditChannel.Cluster,
                kind: AuditKind.ManualFailover,
                status: AuditStatus.Submitted,
                actor: actor,
                target: target.ToString(),
                extra: JsonSerializer.Serialize(new { target = target.ToString(), role = CentralRole }));

            await _audit.WriteAsync(evt);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(
                ex,
                "Best-effort manual-failover audit emission failed (actor={Actor}, target={Target}); "
                + "the failover itself proceeds.", actor, target);
        }
    }
}