feat(cluster): site-pair manual failover relayed from the central UI (Task 10)

Central and each site are SEPARATE Akka clusters, so central cannot act on a site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract travels the existing ClusterClient command/control channel (mirroring the RetryParkedOperation relay); the site's own SiteCommunicationActor performs the graceful Leave and acks the outcome. - ClusterFailoverCoordinator moved out of Host into Communication/ClusterState, beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation; SiteCommunicationActor cannot reference Host, and the two definitions must not drift or the node asked to leave stops being the singleton host. - Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role -- site singletons are placed on the former, so the base role would move the wrong node. Pinned by a unit test asserting the role string and by a real-cluster test. - Site-side guards: refuses a command addressed to another site (a misroute must never fail over a site the operator did not select), refuses when there is no peer, and reports a fault as an ack rather than throwing into supervision -- a restart there would drop central's Ask into a bare timeout and lose the reason. - Ack is sent before the Leave takes effect so it still reaches central. - UI: the same control now serves both scopes via a SiteId parameter. The site confirmation deliberately does NOT claim the admin's page will disconnect -- it won't, and crying wolf there devalues the central warning that is real. A site refusal and an unreachable site surface distinctly. - Rolling upgrade: a site on an older binary has no handler, so the message dead-letters and the Ask times out, reported as "site did not respond". That is the honest outcome; documented on the contract. Fallout fixed: HealthPageTests now renders the page inside CascadingAuthenticationState with the real policy set and IAuthorizationService, because the cards embed an AuthorizeView. That mirrors production, where the layout supplies the cascading value.
2026-07-22 07:48:56 -04:00
parent d66e0d585f
commit c8e2f4da02
12 changed files with 621 additions and 61 deletions
@@ -372,4 +372,89 @@ public class SiteCommunicationActorTests : TestKit
        Assert.Equal(isActive, heartbeat.IsActive);
        Assert.Equal("site1", heartbeat.SiteId);
    }
+
+    // ---- Central→site failover relay (Task 10) ----------------------------------
+    // The failover action is injected for the same reason isActiveCheck is: performing a
+    // real Cluster.Leave would require Akka.Cluster in the TestKit ActorSystem. These
+    // tests pin the ROUTING and GUARD logic; the actual Leave against a real two-node
+    // site cluster is covered by SiteFailoverRelayTests in the integration suite.
+
+    [Fact]
+    public void TriggerSiteFailover_IssuesTheLeave_AndAcksWithTheTarget()
+    {
+        var dmProbe = CreateTestProbe();
+        string? roleAskedFor = null;
+        Func<string, string?> failOver = role =>
+        {
+            roleAskedFor = role;
+            return "akka.tcp://scadabridge@site1-a:8082";
+        };
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-1", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.True(ack.Accepted);
+        Assert.Equal("corr-1", ack.CorrelationId);
+        Assert.Equal("akka.tcp://scadabridge@site1-a:8082", ack.TargetAddress);
+        Assert.Null(ack.ErrorMessage);
+        // Site singletons are scoped to the site-specific role, not the base "Site" role;
+        // failing over the wrong scope would move the wrong node.
+        Assert.Equal("site-site1", roleAskedFor);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_RefusesWhenThereIsNoPeer()
+    {
+        var dmProbe = CreateTestProbe();
+        Func<string, string?> failOver = _ => null;
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-2", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Null(ack.TargetAddress);
+        Assert.NotNull(ack.ErrorMessage);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_RefusesACommandAddressedToAnotherSite()
+    {
+        // A misrouted command must be refused, not silently acted on — acting would fail
+        // over a site the operator never selected.
+        var dmProbe = CreateTestProbe();
+        var invoked = false;
+        Func<string, string?> failOver = _ =>
+        {
+            invoked = true;
+            return "addr";
+        };
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-3", "site2"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Contains("site2", ack.ErrorMessage);
+        Assert.False(invoked);
+    }
+
+    [Fact]
+    public void TriggerSiteFailover_FaultInTheLeave_IsReportedNotThrown()
+    {
+        var dmProbe = CreateTestProbe();
+        Func<string, string?> failOver = _ => throw new InvalidOperationException("cluster unavailable");
+        var siteActor = Sys.ActorOf(Props.Create(() =>
+            new SiteCommunicationActor("site1", _options, dmProbe.Ref, null, failOver)));
+
+        siteActor.Tell(new TriggerSiteFailover("corr-4", "site1"));
+
+        var ack = ExpectMsg<SiteFailoverAck>();
+        Assert.False(ack.Accepted);
+        Assert.Contains("cluster unavailable", ack.ErrorMessage);
+    }
 }