Files
Joseph Doherty c8e2f4da02 feat(cluster): site-pair manual failover relayed from the central UI (Task 10)
Central and each site are SEPARATE Akka clusters, so central cannot act on a
site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract
travels the existing ClusterClient command/control channel (mirroring the
RetryParkedOperation relay); the site's own SiteCommunicationActor performs the
graceful Leave and acks the outcome.

- ClusterFailoverCoordinator moved out of Host into Communication/ClusterState,
  beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation;
  SiteCommunicationActor cannot reference Host, and the two definitions must not
  drift or the node asked to leave stops being the singleton host.
- Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role --
  site singletons are placed on the former, so the base role would move the wrong
  node. Pinned by a unit test asserting the role string and by a real-cluster test.
- Site-side guards: refuses a command addressed to another site (a misroute must
  never fail over a site the operator did not select), refuses when there is no
  peer, and reports a fault as an ack rather than throwing into supervision --
  a restart there would drop central's Ask into a bare timeout and lose the reason.
- Ack is sent before the Leave takes effect so it still reaches central.
- UI: the same control now serves both scopes via a SiteId parameter. The site
  confirmation deliberately does NOT claim the admin's page will disconnect --
  it won't, and crying wolf there devalues the central warning that is real. A
  site refusal and an unreachable site surface distinctly.
- Rolling upgrade: a site on an older binary has no handler, so the message
  dead-letters and the Ask times out, reported as "site did not respond". That
  is the honest outcome; documented on the contract.

Fallout fixed: HealthPageTests now renders the page inside
CascadingAuthenticationState with the real policy set and IAuthorizationService,
because the cards embed an AuthorizeView. That mirrors production, where the
layout supplies the cascading value.
2026-07-22 07:48:56 -04:00

242 lines
10 KiB
C#

using System.Security.Claims;
using Bunit;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.Extensions.DependencyInjection;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health;
using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared;
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
using ZB.MOM.WW.ScadaBridge.Security;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.Monitoring;
/// <summary>
/// bUnit tests for the admin "Trigger failover" control on the Health dashboard
/// (decision 2026-07-22). The control is destructive and privileged, so the three
/// things that must hold are: only an Administrator sees it, it is refused when
/// there is no standby to take over, and it never fires without an explicit
/// confirmation.
/// </summary>
public class HealthFailoverButtonTests : BunitContext
{
private readonly IManualFailoverService _failover = Substitute.For<IManualFailoverService>();
private readonly IDialogService _dialog = Substitute.For<IDialogService>();
private void Arrange(string role)
{
var claims = new[]
{
new Claim(JwtTokenService.UsernameClaimType, "tester"),
new Claim(JwtTokenService.RoleClaimType, role)
};
var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
Services.AddSingleton<AuthenticationStateProvider>(new TestAuthStateProvider(user));
Services.AddAuthorizationCore();
AuthorizationPolicies.AddScadaBridgeAuthorization(Services);
// BunitContext pre-registers a placeholder IAuthorizationService that throws when
// AuthorizeView evaluates a policy. Force the real service so the admin gate is
// genuinely exercised rather than trivially satisfied (same note as NavMenuTests).
Services.AddSingleton<IAuthorizationService, DefaultAuthorizationService>();
Services.AddSingleton(_failover);
Services.AddSingleton(_dialog);
}
/// <summary>
/// AuthorizeView requires a cascading Task&lt;AuthenticationState&gt;, which the app
/// supplies from its layout; a bare component render has to provide it explicitly.
/// </summary>
private IRenderedComponent<CentralFailoverControl> Render(int onlineNodes)
{
var host = Render<CascadingAuthenticationState>(parameters => parameters
.Add(p => p.ChildContent, (RenderFragment)(builder =>
{
builder.OpenComponent<CentralFailoverControl>(0);
builder.AddAttribute(1, nameof(CentralFailoverControl.OnlineCentralNodeCount), onlineNodes);
builder.CloseComponent();
})));
return host.FindComponent<CentralFailoverControl>();
}
[Fact]
public void Button_is_hidden_for_a_non_admin()
{
Arrange("Viewer");
var cut = Render(onlineNodes: 2);
Assert.Empty(cut.FindAll("button"));
}
[Fact]
public void Button_is_rendered_and_enabled_for_an_admin_with_a_standby()
{
Arrange("Administrator");
var cut = Render(onlineNodes: 2);
var button = cut.Find("button");
Assert.False(button.HasAttribute("disabled"));
Assert.Contains("failover", button.TextContent, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Button_is_disabled_with_an_explanatory_title_when_there_is_no_standby()
{
Arrange("Administrator");
var cut = Render(onlineNodes: 1);
var button = cut.Find("button");
Assert.True(button.HasAttribute("disabled"));
// The tooltip must say WHY, not just that it is unavailable.
Assert.Contains("outage", button.GetAttribute("title"), StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task Confirming_triggers_the_failover_exactly_once()
{
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverCentralAsync(Arg.Any<string>())
.Returns(Task.FromResult<string?>("akka.tcp://scadabridge@central-a:8081"));
var cut = Render(onlineNodes: 2);
await cut.Find("button").ClickAsync(new());
await _failover.Received(1).FailOverCentralAsync("tester");
}
[Fact]
public async Task Cancelling_the_confirmation_does_not_trigger_a_failover()
{
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(false);
var cut = Render(onlineNodes: 2);
await cut.Find("button").ClickAsync(new());
await _failover.DidNotReceive().FailOverCentralAsync(Arg.Any<string>());
}
[Fact]
public async Task Confirmation_warns_that_this_page_will_disconnect()
{
// Traefik routes the UI to the ACTIVE node — the very node being failed over — so
// the admin's own circuit drops. If the dialog does not say so, a working failover
// looks like a crash they caused.
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
var cut = Render(onlineNodes: 2);
await cut.Find("button").ClickAsync(new());
await _dialog.Received(1).ConfirmAsync(
Arg.Any<string>(),
Arg.Is<string>(m => m.Contains("reconnect", StringComparison.OrdinalIgnoreCase)),
Arg.Any<bool>());
}
[Fact]
public async Task A_refused_failover_surfaces_the_refusal_instead_of_claiming_success()
{
// The service returns null when the peer guard refuses. The UI must not report a
// failover that never happened.
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverCentralAsync(Arg.Any<string>()).Returns(Task.FromResult<string?>(null));
var cut = Render(onlineNodes: 2);
await cut.Find("button").ClickAsync(new());
Assert.Contains("no standby", cut.Markup, StringComparison.OrdinalIgnoreCase);
}
// ---- Site-pair failover (Task 10) ------------------------------------------
// Same control, SiteId set. The differences that matter: it calls the site path,
// and its confirmation must NOT claim the admin's own page will drop (it won't —
// the site is a different cluster).
private IRenderedComponent<CentralFailoverControl> RenderForSite(int onlineNodes, string siteId)
{
var host = Render<CascadingAuthenticationState>(parameters => parameters
.Add(p => p.ChildContent, (RenderFragment)(builder =>
{
builder.OpenComponent<CentralFailoverControl>(0);
builder.AddAttribute(1, nameof(CentralFailoverControl.OnlineCentralNodeCount), onlineNodes);
builder.AddAttribute(2, nameof(CentralFailoverControl.SiteId), siteId);
builder.CloseComponent();
})));
return host.FindComponent<CentralFailoverControl>();
}
[Fact]
public async Task Site_card_confirming_triggers_the_site_failover_path()
{
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
.Returns(Task.FromResult(new SiteFailoverOutcome(true, "akka.tcp://scadabridge@site-a-a:8082", null)));
var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
await cut.Find("button").ClickAsync(new());
await _failover.Received(1).FailOverSiteAsync("SiteA", "tester");
await _failover.DidNotReceive().FailOverCentralAsync(Arg.Any<string>());
}
[Fact]
public async Task Site_card_confirmation_does_not_claim_this_page_disconnects()
{
// Central failover drops the admin's circuit; a SITE failover does not, and saying
// otherwise would train operators to distrust the warning that does matter.
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
.Returns(Task.FromResult(new SiteFailoverOutcome(true, "addr", null)));
var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
await cut.Find("button").ClickAsync(new());
await _dialog.Received(1).ConfirmAsync(
Arg.Any<string>(),
Arg.Is<string>(m => !m.Contains("reconnect", StringComparison.OrdinalIgnoreCase)),
Arg.Any<bool>());
}
[Fact]
public async Task Site_card_surfaces_a_refusal_from_the_site()
{
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
.Returns(Task.FromResult(new SiteFailoverOutcome(
false, null, "No standby available — failing over a lone node would be an outage.")));
var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
await cut.Find("button").ClickAsync(new());
Assert.Contains("No standby available", cut.Markup);
}
[Fact]
public async Task Site_card_surfaces_an_unreachable_site_distinctly_from_a_refusal()
{
// A timeout is not a "no" from the site — the operator must be able to tell the
// difference, because the failover may or may not have taken effect.
Arrange("Administrator");
_dialog.ConfirmAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>()).Returns(true);
_failover.FailOverSiteAsync(Arg.Any<string>(), Arg.Any<string>())
.Returns(Task.FromResult(new SiteFailoverOutcome(
false, null, "Site did not respond: Ask timed out")));
var cut = RenderForSite(onlineNodes: 2, siteId: "SiteA");
await cut.Find("button").ClickAsync(new());
Assert.Contains("did not respond", cut.Markup);
}
}