lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs

using System.Text.Json;
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Cluster;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;

/// <summary>
/// Default <see cref="IManualFailoverService"/> — a graceful cluster <c>Leave</c> of the oldest Up
/// <c>driver</c> member. See the interface for the full behaviour and the mesh-scope caveat.
/// </summary>
public sealed class ManualFailoverService : IManualFailoverService
{
    /// <summary>The audit <see cref="AuditEvent.Action"/> stamped onto every manual failover.</summary>
    public const string AuditAction = "cluster.manual-failover";

    /// <summary>The audit <see cref="AuditEvent.Category"/> stamped onto every manual failover.</summary>
    public const string AuditCategory = "Redundancy";

    private readonly Func<ActorSystem> _system;
    private readonly IAuditWriter _audit;
    private readonly ILogger<ManualFailoverService> _log;

    /// <summary>
    /// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance:
    /// this is registered before Akka's hosted service has built the system, so resolving eagerly
    /// would race startup (the same lazy-accessor pattern <c>ActorSystemTerminationWatchdog</c> uses).
    /// </summary>
    /// <param name="system">Lazy accessor for the node's ActorSystem.</param>
    /// <param name="audit">The shared audit seam; a failover is recorded before it is performed.</param>
    /// <param name="log">Logger for the failover decision.</param>
    public ManualFailoverService(Func<ActorSystem> system, IAuditWriter audit, ILogger<ManualFailoverService> log)
    {
        _system = system ?? throw new ArgumentNullException(nameof(system));
        _audit = audit ?? throw new ArgumentNullException(nameof(audit));
        _log = log ?? throw new ArgumentNullException(nameof(log));
    }

    /// <inheritdoc/>
    public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system());

    /// <inheritdoc/>
    public async Task<string?> FailOverDriverPrimaryAsync(string actor)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(actor);

        var system = _system();

        // Dry run first so the peer guard is evaluated before anything is audited: a refused
        // failover changed nothing and must not leave an audit row implying it did.
        var target = FailOverCore(system, dryRun: true);
        if (target is null)
        {
            _log.LogWarning(
                "Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.",
                actor);
            return null;
        }

        await _audit.WriteAsync(new AuditEvent
        {
            EventId = Guid.NewGuid(),
            OccurredAtUtc = DateTimeOffset.UtcNow,
            Actor = actor,
            Action = AuditAction,
            Category = AuditCategory,
            SourceNode = target.ToString(),
            Outcome = AuditOutcome.Success,
            DetailsJson = JsonSerializer.Serialize(new
            {
                target = target.ToString(),
                driverMembers = SnapshotCore(system).DriverAddresses,
            }),
        }).ConfigureAwait(false);

        _log.LogWarning(
            "Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. "
            + "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest "
            + "member; its peer becomes Primary and advertises the primary ServiceLevel.",
            actor,
            target);

        FailOverCore(system, dryRun: false);
        return target.ToString();
    }

    /// <summary>
    /// Selects — and, unless <paramref name="dryRun"/>, gracefully removes — the driver Primary.
    /// </summary>
    /// <param name="system">The ActorSystem whose cluster is acted on.</param>
    /// <param name="dryRun">When <see langword="true"/>, selects the target without leaving.</param>
    /// <returns>The target address, or <see langword="null"/> when fewer than two Up driver members exist.</returns>
    public static Address? FailOverCore(ActorSystem system, bool dryRun = false)
    {
        ArgumentNullException.ThrowIfNull(system);

        var cluster = Akka.Cluster.Cluster.Get(system);

        // Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver,
        // Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a
        // failover would remove a node that was not Primary and change nothing. Pinned by
        // ManualFailoverServiceTests.Parity_with_SelectDriverPrimary.
        var drivers = OrderedDrivers(cluster.State.Members);

        // A failover needs somewhere to fail over TO. With a single driver member, leaving would be
        // a shutdown dressed up as a failover.
        if (drivers.Count < 2) return null;

        var primary = drivers[0];
        if (!dryRun) cluster.Leave(primary.Address);
        return primary.Address;
    }

    private static ManualFailoverSnapshot SnapshotCore(ActorSystem system)
    {
        var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members);

        return new ManualFailoverSnapshot(
            drivers.FirstOrDefault()?.Address.ToString(),
            drivers.Select(m => m.Address.ToString()).ToList());
    }

    private static List<Member> OrderedDrivers(IEnumerable<Member> members) => members
        // GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
        // 'Driver' (rolling-upgrade window) must still be a manual-failover candidate.
        .Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, RedundancyStateActor.DriverRole))
        .OrderBy(m => m, Member.AgeOrdering)
        .ToList();
}