feat(cluster): site-pair manual failover relayed from the central UI (Task 10)
Central and each site are SEPARATE Akka clusters, so central cannot act on a
site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract
travels the existing ClusterClient command/control channel (mirroring the
RetryParkedOperation relay); the site's own SiteCommunicationActor performs the
graceful Leave and acks the outcome.
- ClusterFailoverCoordinator moved out of Host into Communication/ClusterState,
beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation;
SiteCommunicationActor cannot reference Host, and the two definitions must not
drift or the node asked to leave stops being the singleton host.
- Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role --
site singletons are placed on the former, so the base role would move the wrong
node. Pinned by a unit test asserting the role string and by a real-cluster test.
- Site-side guards: refuses a command addressed to another site (a misroute must
never fail over a site the operator did not select), refuses when there is no
peer, and reports a fault as an ack rather than throwing into supervision --
a restart there would drop central's Ask into a bare timeout and lose the reason.
- Ack is sent before the Leave takes effect so it still reaches central.
- UI: the same control now serves both scopes via a SiteId parameter. The site
confirmation deliberately does NOT claim the admin's page will disconnect --
it won't, and crying wolf there devalues the central warning that is real. A
site refusal and an unreachable site surface distinctly.
- Rolling upgrade: a site on an older binary has no handler, so the message
dead-letters and the Ask times out, reported as "site did not respond". That
is the honest outcome; documented on the contract.
Fallout fixed: HealthPageTests now renders the page inside
CascadingAuthenticationState with the real policy set and IAuthorizationService,
because the cards embed an AuthorizeView. That mirrors production, where the
layout supplies the cascading value.
This commit is contained in:
@@ -37,6 +37,7 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
/// do not need a real cluster.
|
||||
/// </summary>
|
||||
private readonly Func<bool> _isActiveCheck;
|
||||
private readonly Func<string, string?> _failOverRole;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the local Deployment Manager singleton proxy.
|
||||
@@ -76,12 +77,14 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
string siteId,
|
||||
CommunicationOptions options,
|
||||
IActorRef deploymentManagerProxy,
|
||||
Func<bool>? isActiveCheck = null)
|
||||
Func<bool>? isActiveCheck = null,
|
||||
Func<string, string?>? failOverRole = null)
|
||||
{
|
||||
_siteId = siteId;
|
||||
_options = options;
|
||||
_deploymentManagerProxy = deploymentManagerProxy;
|
||||
_isActiveCheck = isActiveCheck ?? DefaultIsActiveCheck;
|
||||
_failOverRole = failOverRole ?? DefaultFailOverRole;
|
||||
|
||||
// Registration
|
||||
Receive<RegisterCentralClient>(msg =>
|
||||
@@ -263,6 +266,14 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
});
|
||||
|
||||
// Central→site manual failover relay. Central and the site are separate clusters,
|
||||
// so central can only ask — this node performs the graceful Leave locally, scoped to
|
||||
// the SITE-SPECIFIC role, because that is what site singletons (the Deployment
|
||||
// Manager) are placed on. Either node may receive this (the actor is per-node, not a
|
||||
// singleton, and contact rotation picks whichever answers); the target is resolved
|
||||
// from cluster state, not from who received the message.
|
||||
Receive<TriggerSiteFailover>(HandleTriggerSiteFailover);
|
||||
|
||||
// Notification Outbox: forward a buffered notification submitted by the site
|
||||
// Store-and-Forward Engine to the central cluster. The original Sender (the
|
||||
// S&F forwarder's Ask) is forwarded as the ClusterClient.Send sender so the
|
||||
@@ -517,6 +528,69 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
private bool DefaultIsActiveCheck() =>
|
||||
ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(Cluster.Get(Context.System));
|
||||
|
||||
/// <summary>
|
||||
/// Handles a central-initiated site failover. Refuses a command addressed to a different
|
||||
/// site (a misroute must never silently fail over a site the operator did not select) and
|
||||
/// refuses when the site pair has no peer to take over. The ack is sent BEFORE the Leave
|
||||
/// takes effect on the wire, so it still reaches central even when this node is the one
|
||||
/// leaving.
|
||||
/// </summary>
|
||||
private void HandleTriggerSiteFailover(TriggerSiteFailover msg)
|
||||
{
|
||||
if (!string.Equals(msg.SiteId, _siteId, StringComparison.Ordinal))
|
||||
{
|
||||
_log.Warning(
|
||||
"Refusing TriggerSiteFailover addressed to site {Requested}; this node serves {Actual}",
|
||||
msg.SiteId, _siteId);
|
||||
Sender.Tell(new SiteFailoverAck(
|
||||
msg.CorrelationId, Accepted: false, TargetAddress: null,
|
||||
ErrorMessage: $"Command addressed to site '{msg.SiteId}' but this node serves '{_siteId}'."));
|
||||
return;
|
||||
}
|
||||
|
||||
var role = $"site-{_siteId}";
|
||||
try
|
||||
{
|
||||
var target = _failOverRole(role);
|
||||
if (target is null)
|
||||
{
|
||||
_log.Warning(
|
||||
"Refusing TriggerSiteFailover for {SiteId}: fewer than 2 Up members in role {Role}, "
|
||||
+ "so there is no standby to take over", _siteId, role);
|
||||
Sender.Tell(new SiteFailoverAck(
|
||||
msg.CorrelationId, Accepted: false, TargetAddress: null,
|
||||
ErrorMessage: "No standby available — failing over a lone node would be an outage."));
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Warning(
|
||||
"Manual failover requested by central for site {SiteId}: {Target} is leaving the "
|
||||
+ "site cluster gracefully; its singletons hand over to the standby.", _siteId, target);
|
||||
Sender.Tell(new SiteFailoverAck(msg.CorrelationId, Accepted: true, target, ErrorMessage: null));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A fault here must be reported to the operator, not thrown into supervision —
|
||||
// restarting the communication actor would drop central's Ask into a timeout and
|
||||
// lose the reason.
|
||||
_log.Error(ex, "TriggerSiteFailover for {SiteId} faulted", _siteId);
|
||||
Sender.Tell(new SiteFailoverAck(
|
||||
msg.CorrelationId, Accepted: false, TargetAddress: null, ErrorMessage: ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Production failover action: gracefully Leave the oldest Up member carrying
|
||||
/// <paramref name="role"/>, via the shared <see cref="ClusterState.ClusterFailoverCoordinator"/>
|
||||
/// so the central and site paths cannot drift. Injected in tests for the same reason
|
||||
/// <see cref="DefaultIsActiveCheck"/> is — a real Leave needs Akka.Cluster in the
|
||||
/// ActorSystem, which the TestKit system does not load.
|
||||
/// </summary>
|
||||
/// <param name="role">Site-specific role scope.</param>
|
||||
/// <returns>Address of the leaving node, or null when there is no peer.</returns>
|
||||
private string? DefaultFailOverRole(string role) =>
|
||||
ClusterState.ClusterFailoverCoordinator.FailOverOldest(Context.System, role)?.ToString();
|
||||
|
||||
// ── Internal messages ──
|
||||
|
||||
internal record SendHeartbeat;
|
||||
|
||||
Reference in New Issue
Block a user