feat(cluster): site-pair manual failover relayed from the central UI (Task 10)

Central and each site are SEPARATE Akka clusters, so central cannot act on a site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract travels the existing ClusterClient command/control channel (mirroring the RetryParkedOperation relay); the site's own SiteCommunicationActor performs the graceful Leave and acks the outcome. - ClusterFailoverCoordinator moved out of Host into Communication/ClusterState, beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation; SiteCommunicationActor cannot reference Host, and the two definitions must not drift or the node asked to leave stops being the singleton host. - Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role -- site singletons are placed on the former, so the base role would move the wrong node. Pinned by a unit test asserting the role string and by a real-cluster test. - Site-side guards: refuses a command addressed to another site (a misroute must never fail over a site the operator did not select), refuses when there is no peer, and reports a fault as an ack rather than throwing into supervision -- a restart there would drop central's Ask into a bare timeout and lose the reason. - Ack is sent before the Leave takes effect so it still reaches central. - UI: the same control now serves both scopes via a SiteId parameter. The site confirmation deliberately does NOT claim the admin's page will disconnect -- it won't, and crying wolf there devalues the central warning that is real. A site refusal and an unreachable site surface distinctly. - Rolling upgrade: a site on an older binary has no handler, so the message dead-letters and the Ask times out, reported as "site did not respond". That is the honest outcome; documented on the contract. Fallout fixed: HealthPageTests now renders the page inside CascadingAuthenticationState with the real policy set and IAuthorizationService, because the cards embed an AuthorizeView. That mirrors production, where the layout supplies the cascading value.
2026-07-22 07:48:56 -04:00
parent d66e0d585f
commit c8e2f4da02
12 changed files with 621 additions and 61 deletions
@@ -13,7 +13,9 @@
        <button class="btn btn-outline-warning btn-sm"
                disabled="@(!HasPeer || _busy)"
                title="@(HasPeer
-                          ? "Gracefully restart the active node; the standby takes over."
+                          ? (IsSite
+                              ? $"Gracefully restart site {SiteId}'s active node; its standby takes over."
+                              : "Gracefully restart the active node; the standby takes over.")
                          : "No standby available — failing over a lone node would be an outage, not a failover.")"
                @onclick="TriggerAsync">
            @(_busy ? "Failing over…" : "Trigger failover")
@@ -34,6 +36,17 @@
    [Parameter]
    public int OnlineCentralNodeCount { get; set; }

+    /// <summary>
+    /// When set, this control fails over the named SITE pair instead of the central pair.
+    /// Sites are separate Akka clusters reached over the command/control relay, so the site
+    /// path cannot disturb the admin's own session — which is why the confirmation text
+    /// differs. <c>null</c> (the default) means the central pair.
+    /// </summary>
+    [Parameter]
+    public string? SiteId { get; set; }
+
+    private bool IsSite => !string.IsNullOrWhiteSpace(SiteId);
+
    [CascadingParameter]
    private Task<AuthenticationState>? AuthState { get; set; }

@@ -45,16 +58,26 @@

    private async Task TriggerAsync()
    {
-        // The admin is almost certainly connected THROUGH the node about to restart (Traefik
-        // routes to the active node), so the dialog has to set that expectation or a working
-        // failover reads as a crash they just caused.
-        var confirmed = await Dialog.ConfirmAsync(
-            "Trigger central failover?",
-            "The active central node will leave the cluster and restart; the standby takes over "
-            + "and becomes active. Cluster singletons hand over gracefully, but in-flight work on "
-            + "the active node is interrupted. This page is served by the active node, so it will "
-            + "briefly disconnect and reconnect against the new active node.",
-            danger: true);
+        // Central: the admin is almost certainly connected THROUGH the node about to restart
+        // (Traefik routes to the active node), so the dialog must set that expectation or a
+        // working failover reads as a crash they caused.
+        // Site: a different cluster entirely — this page is unaffected, and claiming otherwise
+        // would train operators to ignore the warning that does matter.
+        var confirmed = IsSite
+            ? await Dialog.ConfirmAsync(
+                $"Trigger failover for site {SiteId}?",
+                $"The active node of site {SiteId} will leave its cluster and restart; the site's "
+                + "standby takes over and its Deployment Manager singleton hands over gracefully. "
+                + "In-flight work on that site node is interrupted, and the site is briefly "
+                + "unavailable while the handover completes.",
+                danger: true)
+            : await Dialog.ConfirmAsync(
+                "Trigger central failover?",
+                "The active central node will leave the cluster and restart; the standby takes over "
+                + "and becomes active. Cluster singletons hand over gracefully, but in-flight work on "
+                + "the active node is interrupted. This page is served by the active node, so it will "
+                + "briefly disconnect and reconnect against the new active node.",
+                danger: true);

        if (!confirmed)
        {
@@ -68,17 +91,35 @@
        try
        {
            var actor = await ResolveActorAsync();
-            var target = await Failover.FailOverCentralAsync(actor);

-            if (target is null)
+            if (IsSite)
            {
-                // The server-side peer guard refused. Never report a failover that did not happen.
-                _failed = true;
-                _message = "Refused: no standby available to take over.";
+                var outcome = await Failover.FailOverSiteAsync(SiteId!, actor);
+                if (outcome.Accepted)
+                {
+                    _message = $"Failover triggered — {outcome.TargetAddress} is leaving the site cluster.";
+                }
+                else
+                {
+                    // Carry the site's own words through: a peer-guard refusal and an
+                    // unreachable site are different situations for the operator.
+                    _failed = true;
+                    _message = outcome.ErrorMessage ?? "Refused by the site.";
+                }
            }
            else
            {
-                _message = $"Failover triggered — {target} is leaving the cluster.";
+                var target = await Failover.FailOverCentralAsync(actor);
+                if (target is null)
+                {
+                    // The server-side peer guard refused. Never report a failover that did not happen.
+                    _failed = true;
+                    _message = "Refused: no standby available to take over.";
+                }
+                else
+                {
+                    _message = $"Failover triggered — {target} is leaving the cluster.";
+                }
            }
        }
        catch (Exception ex)
@@ -219,12 +219,11 @@
                        }
                    </div>
                    <div class="d-flex align-items-center gap-3">
-                        @if (isCentral)
-                        {
-                            @* Admin-only; the control gates itself, and disables when the
-                               central pair has no online standby to take over. *@
-                            <CentralFailoverControl OnlineCentralNodeCount="@OnlineNodeCount(state)" />
-                        }
+                        @* Admin-only; the control gates itself, and disables when the pair has
+                           no online standby to take over. Central acts on the local cluster;
+                           a site is asked over the command/control relay. *@
+                        <CentralFailoverControl OnlineCentralNodeCount="@OnlineNodeCount(state)"
+                                                SiteId="@(isCentral ? null : siteId)" />
                        <small class="text-muted">
                            Last report: <TimestampDisplay Value="@state.LastReportReceivedAt" Format="HH:mm:ss" NullText="awaiting first report" />
                            | Last heartbeat: <TimestampDisplay Value="@state.LastHeartbeatAt" Format="HH:mm:ss" />
@@ -21,4 +21,28 @@ public interface IManualFailoverService
    /// <returns>The address acted on, or <c>null</c> when there is no peer to fail over to
    /// (failing over a lone node would be an outage, not a failover).</returns>
    Task<string?> FailOverCentralAsync(string actor);
+
+    /// <summary>
+    /// Asks a SITE to gracefully fail over its own two-node pair. Central and each site are
+    /// separate Akka clusters, so this is a request relayed over the command/control channel —
+    /// the site performs the Leave itself and reports the outcome. Unlike central failover,
+    /// this does NOT disturb the caller's own UI session.
+    /// </summary>
+    /// <param name="siteId">The site whose pair should fail over.</param>
+    /// <param name="actor">Authenticated user name, recorded on the audit row.</param>
+    /// <returns>The site's outcome — accepted with a target, or refused with a reason.</returns>
+    Task<SiteFailoverOutcome> FailOverSiteAsync(string siteId, string actor);
 }
+
+/// <summary>
+/// Outcome of a site-pair failover request, in UI terms. Deliberately distinct from the wire
+/// ack so CentralUI stays free of the Akka message contract.
+/// </summary>
+/// <param name="Accepted"><c>true</c> when the site issued the graceful Leave.</param>
+/// <param name="TargetAddress">Address of the leaving site node; <c>null</c> when refused.</param>
+/// <param name="ErrorMessage">
+/// Why the request was refused or failed. A refusal (peer guard, misroute) is a definitive
+/// answer FROM the site; an unreachable site surfaces here as a timeout message. Both are
+/// reported to the operator rather than being flattened into a generic failure.
+/// </param>
+public sealed record SiteFailoverOutcome(bool Accepted, string? TargetAddress, string? ErrorMessage);
@@ -0,0 +1,53 @@
+namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
+
+/// <summary>
+/// Central → site relay command: gracefully fail over the owning site's two-node cluster.
+/// Sent over the command/control channel when an Administrator clicks "Trigger failover" on a
+/// site card in the Central UI Health dashboard.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Central and each site are SEPARATE Akka clusters, so central cannot act on a site's
+/// membership directly — it can only ask. The site's own <c>SiteCommunicationActor</c> performs
+/// the <c>Cluster.Leave</c> against its site-specific <c>site-{SiteId}</c> role, which is the
+/// role its singletons (the Deployment Manager) are scoped to.
+/// </para>
+/// <para>
+/// Either site node may receive this: <c>SiteCommunicationActor</c> is registered per node
+/// (not as a singleton), and ClusterClient contact rotation reaches whichever answers. That is
+/// fine — <c>Cluster.Leave(address)</c> is valid from any member, and the target is resolved
+/// from cluster state rather than from who received the message.
+/// </para>
+/// <para>
+/// <b>Rolling upgrade:</b> a site running a binary older than this contract has no handler for
+/// this message; it becomes an unhandled message / dead letter and central's Ask times out.
+/// The timeout is reported to the operator as "site did not respond", which is the correct
+/// user-facing outcome — an old site genuinely cannot honour the request. Message evolution
+/// stays additive-only.
+/// </para>
+/// </remarks>
+/// <param name="CorrelationId">Correlation id echoed on the ack.</param>
+/// <param name="SiteId">The site whose pair should fail over. Carried explicitly so a
+/// misrouted command is detectable at the site rather than silently acted on.</param>
+public sealed record TriggerSiteFailover(
+    string CorrelationId,
+    string SiteId);
+
+/// <summary>
+/// Site → central ack for a <see cref="TriggerSiteFailover"/> relay command.
+/// </summary>
+/// <param name="CorrelationId">Correlation id of the originating relay command.</param>
+/// <param name="Accepted">
+/// <c>true</c> when the site resolved a target and issued the graceful Leave.
+/// <c>false</c> is a definitive refusal from the site — most often the peer guard (fewer than
+/// two Up members in the site role, so a failover would be an outage) or a site-id mismatch.
+/// A <c>false</c> ack is NOT a transport failure and must be distinguished from an
+/// unreachable-site timeout.
+/// </param>
+/// <param name="TargetAddress">Address of the node that is leaving; <c>null</c> when refused.</param>
+/// <param name="ErrorMessage">Reason for a refusal, or a fault message; <c>null</c> on success.</param>
+public sealed record SiteFailoverAck(
+    string CorrelationId,
+    bool Accepted,
+    string? TargetAddress,
+    string? ErrorMessage);
@@ -37,6 +37,7 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
    /// do not need a real cluster.
    /// </summary>
    private readonly Func<bool> _isActiveCheck;
+    private readonly Func<string, string?> _failOverRole;

    /// <summary>
    /// Reference to the local Deployment Manager singleton proxy.
@@ -76,12 +77,14 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
        string siteId,
        CommunicationOptions options,
        IActorRef deploymentManagerProxy,
-        Func<bool>? isActiveCheck = null)
+        Func<bool>? isActiveCheck = null,
+        Func<string, string?>? failOverRole = null)
    {
        _siteId = siteId;
        _options = options;
        _deploymentManagerProxy = deploymentManagerProxy;
        _isActiveCheck = isActiveCheck ?? DefaultIsActiveCheck;
+        _failOverRole = failOverRole ?? DefaultFailOverRole;

        // Registration
        Receive<RegisterCentralClient>(msg =>
@@ -263,6 +266,14 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
            }
        });

+        // Central→site manual failover relay. Central and the site are separate clusters,
+        // so central can only ask — this node performs the graceful Leave locally, scoped to
+        // the SITE-SPECIFIC role, because that is what site singletons (the Deployment
+        // Manager) are placed on. Either node may receive this (the actor is per-node, not a
+        // singleton, and contact rotation picks whichever answers); the target is resolved
+        // from cluster state, not from who received the message.
+        Receive<TriggerSiteFailover>(HandleTriggerSiteFailover);
+
        // Notification Outbox: forward a buffered notification submitted by the site
        // Store-and-Forward Engine to the central cluster. The original Sender (the
        // S&F forwarder's Ask) is forwarded as the ClusterClient.Send sender so the
@@ -517,6 +528,69 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
    private bool DefaultIsActiveCheck() =>
        ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(Cluster.Get(Context.System));

+    /// <summary>
+    /// Handles a central-initiated site failover. Refuses a command addressed to a different
+    /// site (a misroute must never silently fail over a site the operator did not select) and
+    /// refuses when the site pair has no peer to take over. The ack is sent BEFORE the Leave
+    /// takes effect on the wire, so it still reaches central even when this node is the one
+    /// leaving.
+    /// </summary>
+    private void HandleTriggerSiteFailover(TriggerSiteFailover msg)
+    {
+        if (!string.Equals(msg.SiteId, _siteId, StringComparison.Ordinal))
+        {
+            _log.Warning(
+                "Refusing TriggerSiteFailover addressed to site {Requested}; this node serves {Actual}",
+                msg.SiteId, _siteId);
+            Sender.Tell(new SiteFailoverAck(
+                msg.CorrelationId, Accepted: false, TargetAddress: null,
+                ErrorMessage: $"Command addressed to site '{msg.SiteId}' but this node serves '{_siteId}'."));
+            return;
+        }
+
+        var role = $"site-{_siteId}";
+        try
+        {
+            var target = _failOverRole(role);
+            if (target is null)
+            {
+                _log.Warning(
+                    "Refusing TriggerSiteFailover for {SiteId}: fewer than 2 Up members in role {Role}, "
+                    + "so there is no standby to take over", _siteId, role);
+                Sender.Tell(new SiteFailoverAck(
+                    msg.CorrelationId, Accepted: false, TargetAddress: null,
+                    ErrorMessage: "No standby available — failing over a lone node would be an outage."));
+                return;
+            }
+
+            _log.Warning(
+                "Manual failover requested by central for site {SiteId}: {Target} is leaving the "
+                + "site cluster gracefully; its singletons hand over to the standby.", _siteId, target);
+            Sender.Tell(new SiteFailoverAck(msg.CorrelationId, Accepted: true, target, ErrorMessage: null));
+        }
+        catch (Exception ex)
+        {
+            // A fault here must be reported to the operator, not thrown into supervision —
+            // restarting the communication actor would drop central's Ask into a timeout and
+            // lose the reason.
+            _log.Error(ex, "TriggerSiteFailover for {SiteId} faulted", _siteId);
+            Sender.Tell(new SiteFailoverAck(
+                msg.CorrelationId, Accepted: false, TargetAddress: null, ErrorMessage: ex.Message));
+        }
+    }
+
+    /// <summary>
+    /// Production failover action: gracefully Leave the oldest Up member carrying
+    /// <paramref name="role"/>, via the shared <see cref="ClusterState.ClusterFailoverCoordinator"/>
+    /// so the central and site paths cannot drift. Injected in tests for the same reason
+    /// <see cref="DefaultIsActiveCheck"/> is — a real Leave needs Akka.Cluster in the
+    /// ActorSystem, which the TestKit system does not load.
+    /// </summary>
+    /// <param name="role">Site-specific role scope.</param>
+    /// <returns>Address of the leaving node, or null when there is no peer.</returns>
+    private string? DefaultFailOverRole(string role) =>
+        ClusterState.ClusterFailoverCoordinator.FailOverOldest(Context.System, role)?.ToString();
+
    // ── Internal messages ──

    internal record SendHeartbeat;
@@ -0,0 +1,56 @@
+using Akka.Actor;
+using Akka.Cluster;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
+
+/// <summary>
+/// Deliberate, operator-initiated failover of a two-node cluster: the current active member
+/// leaves GRACEFULLY so <c>ClusterSingletonManager</c> hands its singletons to the survivor.
+///
+/// <para>Lives beside <see cref="ActiveNodeEvaluator"/> — the two share one definition of
+/// "active" (oldest Up member in a role scope), and they must not drift: the node asked to
+/// leave has to be exactly the node hosting the singletons. Placed in Communication rather
+/// than Host because BOTH sides need it — the central pair fails over via the Host's
+/// <c>AkkaManualFailoverService</c>, and a site pair fails over inside
+/// <c>SiteCommunicationActor</c>, which cannot reference Host.</para>
+///
+/// <para><b>Leave, never Down.</b> A Down skips singleton hand-off and hands the outcome to
+/// the downing strategy — the wrong tool for a planned role swap.</para>
+/// </summary>
+public static class ClusterFailoverCoordinator
+{
+    /// <summary>
+    /// Asks the oldest Up member carrying <paramref name="role"/> to leave the cluster.
+    /// Mirrors <see cref="ActiveNodeEvaluator.SelfIsOldestUp"/>'s rule, so the node acted on is
+    /// the singleton host — never Akka's cluster <i>leader</i>, whose address-ordered definition
+    /// diverges from singleton placement once the original first node restarts and rejoins.
+    /// </summary>
+    /// <param name="system">Actor system whose cluster is acted on.</param>
+    /// <param name="role">Role scope. Central uses <c>Central</c>; a site pair uses its
+    /// site-specific <c>site-{SiteId}</c> role, since site singletons are scoped to that role.</param>
+    /// <param name="dryRun">Resolve and return the target WITHOUT issuing the Leave. Used to
+    /// name the target in an audit row before acting, and to answer "would this work?".</param>
+    /// <returns>The address that leaves (or would leave), or <c>null</c> when fewer than two Up
+    /// members carry the role — failing over a lone node is an outage, not a failover.</returns>
+    public static Address? FailOverOldest(ActorSystem system, string role, bool dryRun = false)
+    {
+        var cluster = Akka.Cluster.Cluster.Get(system);
+        var withRole = cluster.State.Members
+            .Where(m => m.Status == MemberStatus.Up && m.HasRole(role))
+            .OrderBy(m => m, Member.AgeOrdering)
+            .ToList();
+
+        if (withRole.Count < 2)
+        {
+            return null;
+        }
+
+        var oldest = withRole[0];
+        if (!dryRun)
+        {
+            cluster.Leave(oldest.Address);
+        }
+
+        return oldest.Address;
+    }
+}
@@ -809,6 +809,34 @@ public class CommunicationService
        return await GetSiteCallAudit().Ask<DiscardSiteCallResponse>(
            request, _options.QueryTimeout, cancellationToken);
    }
+
+    /// <summary>
+    /// Asks a site to gracefully fail over its own two-node pair (Task 10). Central and each
+    /// site are SEPARATE Akka clusters, so central cannot act on site membership directly —
+    /// the site's own <c>SiteCommunicationActor</c> performs the <c>Cluster.Leave</c> against
+    /// its site-specific role and acks the outcome.
+    /// <para>
+    /// A site running a binary older than the <see cref="TriggerSiteFailover"/> contract has no
+    /// handler for it, so the message dead-letters and this Ask times out. That surfaces to the
+    /// operator as "site did not respond", which is the honest outcome — an old site genuinely
+    /// cannot honour the request.
+    /// </para>
+    /// </summary>
+    /// <param name="siteId">Target site.</param>
+    /// <param name="correlationId">Correlation id echoed on the ack.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>The site's ack.</returns>
+    public async Task<SiteFailoverAck> TriggerSiteFailoverAsync(
+        string siteId, string correlationId, CancellationToken cancellationToken = default)
+    {
+        _logger.LogWarning(
+            "Relaying TriggerSiteFailover to site {SiteId}, correlationId={CorrelationId}",
+            siteId, correlationId);
+
+        var envelope = new SiteEnvelope(siteId, new TriggerSiteFailover(correlationId, siteId));
+        return await GetActor().Ask<SiteFailoverAck>(
+            envelope, _options.QueryTimeout, cancellationToken);
+    }
 }

 /// <summary>
@@ -6,6 +6,8 @@ using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.Communication;
+using ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
 using ZB.MOM.WW.ScadaBridge.Host.Actors;

 namespace ZB.MOM.WW.ScadaBridge.Host.Health;
@@ -33,19 +35,23 @@ public sealed class AkkaManualFailoverService : IManualFailoverService
 {
    private readonly AkkaHostedService _akka;
    private readonly ICentralAuditWriter _audit;
+    private readonly CommunicationService _communication;
    private readonly ILogger<AkkaManualFailoverService> _logger;

    /// <summary>Initializes a new <see cref="AkkaManualFailoverService"/>.</summary>
    /// <param name="akka">The Akka hosted service exposing the cluster's actor system.</param>
    /// <param name="audit">Central direct-write audit writer.</param>
+    /// <param name="communication">Central→site command/control transport, used for site failover.</param>
    /// <param name="logger">Logger.</param>
    public AkkaManualFailoverService(
        AkkaHostedService akka,
        ICentralAuditWriter audit,
+        CommunicationService communication,
        ILogger<AkkaManualFailoverService> logger)
    {
        _akka = akka;
        _audit = audit;
+        _communication = communication;
        _logger = logger;
    }

@@ -76,46 +82,63 @@ public sealed class AkkaManualFailoverService : IManualFailoverService
        return target.ToString();
    }

+    /// <inheritdoc />
+    public async Task<SiteFailoverOutcome> FailOverSiteAsync(string siteId, string actor)
+    {
+        // Audit BEFORE relaying, for the same reason as central: the row must exist even if
+        // the outcome is never observed. Unlike central, the acting node is never the one
+        // leaving — but a relay can still time out, and an un-acked request that DID take
+        // effect at the site would otherwise be unattributed.
+        await WriteAuditAsync(actor, target: siteId, sourceSiteId: siteId);
+
+        _logger.LogWarning(
+            "Manual site failover triggered by {Actor} for site {SiteId}; relaying to the site cluster.",
+            actor, siteId);
+
+        try
+        {
+            var ack = await _communication.TriggerSiteFailoverAsync(siteId, Guid.NewGuid().ToString());
+            return new SiteFailoverOutcome(ack.Accepted, ack.TargetAddress, ack.ErrorMessage);
+        }
+        catch (Exception ex)
+        {
+            // Central never buffers for an unreachable site — the Ask simply times out. Report
+            // that distinctly from a refusal, which is a definitive answer FROM the site.
+            _logger.LogWarning(ex,
+                "Manual site failover for {SiteId} did not get an ack from the site.", siteId);
+            return new SiteFailoverOutcome(
+                Accepted: false,
+                TargetAddress: null,
+                ErrorMessage: $"Site did not respond: {ex.Message}");
+        }
+    }
+
    /// <summary>The Akka role scoping central-cluster membership.</summary>
    private const string CentralRole = "Central";

    /// <summary>
-    /// Oldest Up member with the role leaves — mirrors <c>ActiveNodeEvaluator</c>'s oldest-Up
-    /// rule so the node acted on is exactly the one hosting the singletons. Returns null when
-    /// fewer than 2 Up members carry the role (no peer = failover would be an outage).
+    /// Oldest Up member with the role leaves. Delegates to
+    /// <see cref="ClusterFailoverCoordinator.FailOverOldest"/>, which lives in Communication so
+    /// the site-pair failover path (inside <c>SiteCommunicationActor</c>, which cannot reference
+    /// Host) shares one implementation of the rule.
    /// </summary>
    /// <param name="system">The actor system whose cluster is acted on.</param>
    /// <param name="role">Role scope for membership.</param>
    /// <param name="dryRun">When true, resolve and return the target without issuing the Leave.</param>
    /// <returns>The address that leaves (or would leave), or null when there is no peer.</returns>
    public static Address? FailOverCore(ActorSystem system, string role, bool dryRun = false)
-    {
-        var cluster = Akka.Cluster.Cluster.Get(system);
-        var withRole = cluster.State.Members
-            .Where(m => m.Status == MemberStatus.Up && m.HasRole(role))
-            .OrderBy(m => m, Member.AgeOrdering)
-            .ToList();
-
-        if (withRole.Count < 2)
-        {
-            return null;
-        }
-
-        var oldest = withRole[0];
-        if (!dryRun)
-        {
-            cluster.Leave(oldest.Address);
-        }
-
-        return oldest.Address;
-    }
+        => ClusterFailoverCoordinator.FailOverOldest(system, role, dryRun);

    /// <summary>
    /// Best-effort audit row. Audit failure must never block the failover the admin asked for
    /// — the same rule the rest of the Audit Log follows (audit is best-effort; the
    /// user-facing action's own success path is authoritative).
    /// </summary>
-    private async Task WriteAuditAsync(string actor, Address target)
+    private Task WriteAuditAsync(string actor, Address target)
+        => WriteAuditAsync(actor, target.ToString(), sourceSiteId: null);
+
+    /// <inheritdoc cref="WriteAuditAsync(string, Address)"/>
+    private async Task WriteAuditAsync(string actor, string target, string? sourceSiteId)
    {
        try
        {
@@ -124,8 +147,15 @@ public sealed class AkkaManualFailoverService : IManualFailoverService
                kind: AuditKind.ManualFailover,
                status: AuditStatus.Submitted,
                actor: actor,
-                target: target.ToString(),
-                extra: JsonSerializer.Serialize(new { target = target.ToString(), role = CentralRole }));
+                target: target,
+                sourceSiteId: sourceSiteId,
+                extra: JsonSerializer.Serialize(new
+                {
+                    target,
+                    // Central rows name the Central role; site rows name the site, so a query
+                    // can tell which pair an operator moved.
+                    scope = sourceSiteId is null ? CentralRole : $"site-{sourceSiteId}"
+                }));

            await _audit.WriteAsync(evt);
        }
@@ -153,4 +153,89 @@ public class HealthFailoverButtonTests : BunitContext

        Assert.Contains("no standby", cut.Markup, StringComparison.OrdinalIgnoreCase);
    }
+
+    // ---- Site-pair failover (Task 10) ------------------------------------------
+    // Same control, SiteId set. The differences that matter: it calls the site path,
+    // and its confirmation must NOT claim the admin's own page will drop (it won't —
+    // the site is a different cluster).
+
+    private IRenderedComponent<CentralFailoverControl> RenderForSite(int onlineNodes, string siteId)
+    {
+        var host = Render<CascadingAuthenticationState>(parameters => parameters
+            .Add(p => p.ChildContent, (RenderFragment)(builder =>
+            {
+                builder.OpenComponent<CentralFailoverControl>(0);
+                builder.AddAttribute(1, nameof(CentralFailoverControl.OnlineCentralNodeCount), onlineNodes);
+                builder.AddAttribute(2, nameof(CentralFailoverControl.SiteId), siteId);
+                builder.CloseComponent();
+            })));
+
+        return host.FindComponent<CentralFailoverControl>();
+    }
+
+    [Fact]
+    public async Task Site_card_confirming_triggers_the_site_failover_path()
+    {
+        Arrange("Administrator");
+        _dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
+        _failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
+            .Returns(Task.FromResult(new SiteFailoverOutcome(true, "akka.tcp://scadabridge@site-a-a:8082", null)));
+
+        var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
+        await cut.Find("button").ClickAsync(new());
+
+        await _failover.Received(1).FailOverSiteAsync("SiteA", "tester");
+        await _failover.DidNotReceive().FailOverCentralAsync(Arg.Any<string>());
+    }
+
+    [Fact]
+    public async Task Site_card_confirmation_does_not_claim_this_page_disconnects()
+    {
+        // Central failover drops the admin's circuit; a SITE failover does not, and saying
+        // otherwise would train operators to distrust the warning that does matter.
+        Arrange("Administrator");
+        _dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
+        _failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
+            .Returns(Task.FromResult(new SiteFailoverOutcome(true, "addr", null)));
+
+        var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
+        await cut.Find("button").ClickAsync(new());
+
+        await _dialog.Received(1).ConfirmAsync(
+            Arg.Any<string>(),
+            Arg.Is<string>(m => !m.Contains("reconnect", StringComparison.OrdinalIgnoreCase)),
+            Arg.Any<bool>());
+    }
+
+    [Fact]
+    public async Task Site_card_surfaces_a_refusal_from_the_site()
+    {
+        Arrange("Administrator");
+        _dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
+        _failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
+            .Returns(Task.FromResult(new SiteFailoverOutcome(
+                false, null, "No standby available — failing over a lone node would be an outage.")));
+
+        var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
+        await cut.Find("button").ClickAsync(new());
+
+        Assert.Contains("No standby available", cut.Markup);
+    }
+
+    [Fact]
+    public async Task Site_card_surfaces_an_unreachable_site_distinctly_from_a_refusal()
+    {
+        // A timeout is not a "no" from the site — the operator must be able to tell the
+        // difference, because the failover may or may not have taken effect.
+        Arrange("Administrator");
+        _dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
+        _failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
+            .Returns(Task.FromResult(new SiteFailoverOutcome(
+                false, null, "Site did not respond: Ask timed out")));
+
+        var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
+        await cut.Find("button").ClickAsync(new());
+
+        Assert.Contains("did not respond", cut.Markup);
+    }
 }
@@ -2,11 +2,14 @@ using System.Security.Claims;
 using ZB.MOM.WW.ScadaBridge.Security;
 using Akka.Actor;
 using Bunit;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Authorization;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared;
 using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
@@ -105,12 +108,43 @@ public class HealthPageTests : BunitContext
        var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
        Services.AddSingleton<AuthenticationStateProvider>(new TestAuthStateProvider(user));
        Services.AddAuthorizationCore();
+        // The failover control's AuthorizeView evaluates the named RequireAdmin policy, so the
+        // real policy set has to be registered or the page throws on any card that renders it.
+        AuthorizationPolicies.AddScadaBridgeAuthorization(Services);
+        // BunitContext pre-registers a placeholder IAuthorizationService that throws when a
+        // policy is evaluated; force the real one (same note as NavMenuTests).
+        Services.AddSingleton<IAuthorizationService, DefaultAuthorizationService>();
+
+        // The cluster cards embed CentralFailoverControl (Task 10), which injects these two.
+        // The page's DI graph genuinely requires them in production, so the test supplies
+        // them rather than the component making its dependencies optional. Behaviour of the
+        // control itself is covered by HealthFailoverButtonTests.
+        Services.AddSingleton(Substitute.For<IManualFailoverService>());
+        Services.AddSingleton(Substitute.For<IDialogService>());
+    }
+
+    /// <summary>
+    /// Renders the dashboard the way the app does — inside a
+    /// <see cref="CascadingAuthenticationState"/>, which the real layout supplies. The
+    /// cluster cards contain an AuthorizeView (the Task 10 failover control), and
+    /// AuthorizeView throws without that cascading value.
+    /// </summary>
+    private IRenderedComponent<HealthPage> RenderHealthPage()
+    {
+        var host = Render<CascadingAuthenticationState>(parameters => parameters
+            .Add(p => p.ChildContent, (RenderFragment)(builder =>
+            {
+                builder.OpenComponent<HealthPage>(0);
+                builder.CloseComponent();
+            })));
+
+        return host.FindComponent<HealthPage>();
    }

    [Fact]
    public void Renders_OutboxKpiTiles_WithValues()
    {
-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        // KPI data arrives via an async actor Ask after first render.
        cut.WaitForAssertion(() =>
@@ -129,7 +163,7 @@ public class HealthPageTests : BunitContext
    [Fact]
    public void RendersLinkToTheNotificationKpisPage()
    {
-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();
        var link = cut.Find("a[href='/notifications/kpis']");
        Assert.Contains("View details", link.TextContent);
    }
@@ -148,7 +182,7 @@ public class HealthPageTests : BunitContext
                AsOfUtc: DateTime.UtcNow)));
        Services.AddSingleton(auditService);

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -166,7 +200,7 @@ public class HealthPageTests : BunitContext
    [Fact]
    public void Renders_SiteCallKpiTiles_WithValues()
    {
-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        // KPI data arrives via an async actor Ask after first render.
        cut.WaitForAssertion(() =>
@@ -186,7 +220,7 @@ public class HealthPageTests : BunitContext
    [Fact]
    public void RendersLinkToTheSiteCallsReportPage()
    {
-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();
        var link = cut.Find("a[href='/site-calls/report']");
        Assert.Contains("View details", link.TextContent);
    }
@@ -197,7 +231,7 @@ public class HealthPageTests : BunitContext
        _siteCallKpiReply = new SiteCallKpiResponse(
            "k", false, "site call repository unavailable", 0, 0, 0, 0, null, 0);

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -216,7 +250,7 @@ public class HealthPageTests : BunitContext
        _kpiReply = new NotificationKpiResponse(
            "k", false, "outbox repository unavailable", 0, 0, 0, 0, null);

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -234,7 +268,7 @@ public class HealthPageTests : BunitContext
        // default-site load produces charts.
        SeedSites("site-a");

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -268,7 +302,7 @@ public class HealthPageTests : BunitContext
                throw new InvalidOperationException("kpi history unavailable"));
        Services.AddSingleton(faulting);

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -294,7 +328,7 @@ public class HealthPageTests : BunitContext
            LastReportReceivedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
        });

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -316,7 +350,7 @@ public class HealthPageTests : BunitContext
            LastStatusChangeAt = changedAt,
        });

-        var cut = Render<HealthPage>();
+        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
@@ -372,4 +372,89 @@ public class SiteCommunicationActorTests : TestKit
        Assert.Equal(isActive, heartbeat.IsActive);
        Assert.Equal("site1", heartbeat.SiteId);
    }
+
+    // ---- Central→site failover relay (Task 10) ----------------------------------
+    // The failover action is injected for the same reason isActiveCheck is: performing a
+    // real Cluster.Leave would require Akka.Cluster in the TestKit ActorSystem. These
+    // tests pin the ROUTING and GUARD logic; the actual Leave against a real two-node
+    // site cluster is covered by SiteFailoverRelayTests in the integration suite.
+
+    [Fact]
+    public void TriggerSiteFailover_IssuesTheLeave_AndAcksWithTheTarget()
+    {
+        var dmProbe = CreateTestProbe();
+        string? roleAskedFor = null;
+        Func<string, string?> failOver = role =>
+        {
+            roleAskedFor = role;
+            return "akka.tcp://scadabridge@site1-a:8082";
+        };
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-1", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.True(ack.Accepted);
+        Assert.Equal("corr-1", ack.CorrelationId);
+        Assert.Equal("akka.tcp://scadabridge@site1-a:8082", ack.TargetAddress);
+        Assert.Null(ack.ErrorMessage);
+        // Site singletons are scoped to the site-specific role, not the base "Site" role;
+        // failing over the wrong scope would move the wrong node.
+        Assert.Equal("site-site1", roleAskedFor);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_RefusesWhenThereIsNoPeer()
+    {
+        var dmProbe = CreateTestProbe();
+        Func<string, string?> failOver = _ => null;
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-2", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Null(ack.TargetAddress);
+        Assert.NotNull(ack.ErrorMessage);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_RefusesACommandAddressedToAnotherSite()
+    {
+        // A misrouted command must be refused, not silently acted on — acting would fail
+        // over a site the operator never selected.
+        var dmProbe = CreateTestProbe();
+        var invoked = false;
+        Func<string, string?> failOver = _ =>
+        {
+            invoked = true;
+            return "addr";
+        };
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-3", "site2"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Contains("site2", ack.ErrorMessage);
+        Assert.False(invoked);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_FaultInTheLeave_IsReportedNotThrown()
+    {
+        var dmProbe = CreateTestProbe();
+        Func<string, string?> failOver = _ => throw new InvalidOperationException("cluster unavailable");
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-4", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Contains("cluster unavailable", ack.ErrorMessage);
+    }
 }
@@ -0,0 +1,51 @@
+using Akka.Actor;
+using Akka.Cluster;
+using ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
+
+namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
+
+/// <summary>
+/// Real-cluster proof for the central→site failover relay (Task 10). The unit tests in
+/// <c>SiteCommunicationActorTests</c> pin the routing and guard logic with the failover action
+/// stubbed out; this pins the part they cannot — that the shared
+/// <see cref="ClusterFailoverCoordinator"/> actually moves a SITE pair when scoped to the
+/// site-specific role.
+///
+/// <para>The site-specific role scope is the load-bearing detail: site singletons (the
+/// Deployment Manager) are placed on <c>site-{SiteId}</c>, not on the base <c>Site</c> role, so
+/// failing over the wrong scope would move the wrong node.</para>
+/// </summary>
+public sealed class SiteFailoverRelayTests
+{
+    [Fact]
+    public async Task Failing_over_a_site_pair_moves_the_oldest_and_the_survivor_takes_over()
+    {
+        // A site pair, both nodes carrying the site-specific role.
+        await using var f = await TwoNodeClusterFixture.StartAsync(role: "site-SiteA");
+        var oldest = Akka.Cluster.Cluster.Get(f.NodeA);
+        Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "site-SiteA"),
+            "precondition: NodeA must be the active (oldest Up) site node");
+
+        // Issued from the other node, as the relay does when contact rotation lands there.
+        var target = ClusterFailoverCoordinator.FailOverOldest(f.NodeB, "site-SiteA");
+
+        Assert.Equal(oldest.SelfAddress, target);
+        await f.NodeA.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30));
+        await TwoNodeClusterFixture.WaitForMemberRemoved(f.NodeB, oldest.SelfAddress, TimeSpan.FromSeconds(30));
+        Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(f.NodeB), "site-SiteA"));
+    }
+
+    [Fact]
+    public async Task A_site_role_scope_that_matches_no_member_is_refused()
+    {
+        // Guards the role-scoping mistake directly: asking for a site that isn't this pair
+        // must find no members and refuse, rather than falling back to some other node.
+        await using var f = await TwoNodeClusterFixture.StartAsync(role: "site-SiteA");
+
+        var target = ClusterFailoverCoordinator.FailOverOldest(f.NodeB, "site-SiteB");
+
+        Assert.Null(target);
+        // Positive control: the pair is untouched and still fully formed.
+        await TwoNodeClusterFixture.WaitForMembersUp(f.NodeB, 2, TimeSpan.FromSeconds(10));
+    }
+}