diff --git a/docker/README.md b/docker/README.md index ddc7cc4f..46bcc649 100644 --- a/docker/README.md +++ b/docker/README.md @@ -327,6 +327,14 @@ open http://localhost:9002 docker start scadabridge-central-a ``` +**Manual failover from the UI (admin-only).** Instead of stopping a container, an Administrator can trigger a planned role swap from the **Trigger failover** button on the central-cluster card at `/monitoring/health` (via Traefik, `http://localhost:9000`). The active (oldest Up) node leaves the cluster **gracefully**, so singletons hand over rather than being killed; the node then restarts under `restart: unless-stopped` and rejoins as the standby. + +- The button is disabled when the pair has no online standby — the same guard is re-enforced server-side, since failing over a lone node is an outage, not a failover. +- Triggering it **disconnects the page you clicked it on**: Traefik routes the UI to the active node, which is the node being restarted. The page reconnects against the new active node. +- Each invocation writes one `Cluster` / `ManualFailover` row to `dbo.AuditLog` naming the admin and the target address, written before the Leave is issued. + +To verify on the rig: press the button, watch `central-a` restart and `central-b`'s badge flip to Primary, then confirm the audit row landed. + ### Site Failover ```bash diff --git a/docs/requirements/Component-ClusterInfrastructure.md b/docs/requirements/Component-ClusterInfrastructure.md index 6342e290..29c7e725 100644 --- a/docs/requirements/Component-ClusterInfrastructure.md +++ b/docs/requirements/Component-ClusterInfrastructure.md @@ -137,6 +137,21 @@ Behavior, covered by `SelfFirstSeedBootstrapTests` (real in-process clusters bui The docker failover drill (`docker/failover-drill.sh`) proves both directions: `standby` mode kills the younger node (active untouched, zero routing blips); `active` mode kills the active/oldest node and asserts the survivor **takes over while the victim is still down**. +### Manual Failover (admin-triggered) + +An Administrator can swap the central pair's roles deliberately — for planned maintenance on the active node, or to move singletons off a node behaving badly without waiting for a crash. The control is the **Trigger failover** button on the central-cluster card of the Health dashboard (`/monitoring/health`). + +Semantics: + +- **Graceful `Leave`, never `Down`.** The active node leaves the cluster so `ClusterSingletonManager` hands its singletons to the standby before the member is removed. A `Down` would skip the hand-off. +- **Target = the oldest Up member with the `Central` role**, the same rule `ActiveNodeEvaluator` uses, so the node acted on is exactly the one hosting the singletons — never Akka's cluster *leader*, whose address-ordered definition diverges from singleton placement after a restart. +- **Peer guard.** Refused when fewer than two Up `Central` members exist: failing over a lone node is an outage, not a failover. Enforced server-side in `AkkaManualFailoverService`; the button is also disabled client-side when the pair has no online standby. +- **Admin-only** (`AuthorizationPolicies.RequireAdmin`). The Health page itself is all-roles, so the gate lives on the control. +- **Audited before acting.** One `AuditChannel.Cluster` / `AuditKind.ManualFailover` row is written through `ICentralAuditWriter` *before* the Leave is issued, naming the actor and the target address — the acting node can be the one that goes away, and an audit written afterwards could be lost to the shutdown it describes. Audit failure never blocks the failover. +- **Your own page will disconnect.** Traefik routes the UI to the active node, so triggering a failover drops the admin's Blazor circuit; it reconnects against the new active node. The confirmation dialog says so. + +After the Leave the node's `ActorSystem` terminates, the `WhenTerminated` watchdog exits the process, the service supervisor restarts it, and it rejoins as the youngest member (the new standby). Recovery is the normal restart contract above — no interaction with seed ordering, since the peer is alive on this path. + ## Single-Node Operation `akka.cluster.min-nr-of-members` must be set to **1**. After failover, only one node is running. If set to 2, the surviving node waits for a second member before allowing the Cluster Singleton (Site Runtime Deployment Manager) to start — blocking all data collection and script execution indefinitely. diff --git a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Health/CentralFailoverControl.razor b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Health/CentralFailoverControl.razor new file mode 100644 index 00000000..fbe96dd0 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Health/CentralFailoverControl.razor @@ -0,0 +1,108 @@ +@using Microsoft.AspNetCore.Components.Authorization +@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared +@using ZB.MOM.WW.ScadaBridge.CentralUI.Services +@using ZB.MOM.WW.ScadaBridge.Security +@inject IManualFailoverService Failover +@inject IDialogService Dialog + +@* Admin-only manual failover of the central pair (decision 2026-07-22). Rendered on the + Health dashboard's central-cluster card. The page itself is all-roles, so the gate lives + here rather than on the page's [Authorize] attribute. *@ + + + + @if (_message is not null) + { + @_message + } + + + +@code { + /// + /// Number of central nodes currently reporting online. Fewer than two means there is no + /// standby to take over, so the control is disabled — the guard is enforced again + /// server-side in the failover service, which is the authoritative check. + /// + [Parameter] + public int OnlineCentralNodeCount { get; set; } + + [CascadingParameter] + private Task? AuthState { get; set; } + + private bool HasPeer => OnlineCentralNodeCount >= 2; + + private bool _busy; + private bool _failed; + private string? _message; + + private async Task TriggerAsync() + { + // The admin is almost certainly connected THROUGH the node about to restart (Traefik + // routes to the active node), so the dialog has to set that expectation or a working + // failover reads as a crash they just caused. + var confirmed = await Dialog.ConfirmAsync( + "Trigger central failover?", + "The active central node will leave the cluster and restart; the standby takes over " + + "and becomes active. Cluster singletons hand over gracefully, but in-flight work on " + + "the active node is interrupted. This page is served by the active node, so it will " + + "briefly disconnect and reconnect against the new active node.", + danger: true); + + if (!confirmed) + { + return; + } + + _busy = true; + _failed = false; + _message = null; + + try + { + var actor = await ResolveActorAsync(); + var target = await Failover.FailOverCentralAsync(actor); + + if (target is null) + { + // The server-side peer guard refused. Never report a failover that did not happen. + _failed = true; + _message = "Refused: no standby available to take over."; + } + else + { + _message = $"Failover triggered — {target} is leaving the cluster."; + } + } + catch (Exception ex) + { + _failed = true; + _message = $"Failover failed: {ex.Message}"; + } + finally + { + _busy = false; + } + } + + /// Authenticated user name recorded on the audit row. + private async Task ResolveActorAsync() + { + if (AuthState is null) + { + return "unknown"; + } + + var state = await AuthState; + return state.User.Identity?.Name + ?? state.User.FindFirst(JwtTokenService.UsernameClaimType)?.Value + ?? "unknown"; + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Monitoring/Health.razor b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Monitoring/Health.razor index a26d8442..2ae273cd 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Monitoring/Health.razor +++ b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Monitoring/Health.razor @@ -218,11 +218,19 @@ offline since @changedAt.ToString("u") } - - Last report: - | Last heartbeat: - | Seq: @state.LastSequenceNumber - +
+ @if (isCentral) + { + @* Admin-only; the control gates itself, and disables when the + central pair has no online standby to take over. *@ + + } + + Last report: + | Last heartbeat: + | Seq: @state.LastSequenceNumber + +
@if (state.LatestReport != null) @@ -442,6 +450,15 @@ private string StaleTimeoutDisplay => FormatDuration(HealthOptions.Value.MetricsStaleTimeout); + // Online central nodes, from the same ClusterNodes list the Nodes column renders. + // Fewer than two means no standby, which disables the manual-failover control. This is + // a display-side guard only — AkkaManualFailoverService re-checks against live cluster + // membership, which is the authoritative answer. + private static int OnlineNodeCount(SiteHealthState state) => + state.LatestReport?.ClusterNodes is { Count: > 0 } nodes + ? nodes.Count(n => n.IsOnline) + : (state.IsOnline ? 1 : 0); + private static string FormatDuration(TimeSpan span) => span.TotalMinutes >= 1 && span == TimeSpan.FromMinutes(Math.Round(span.TotalMinutes)) ? $"{span.TotalMinutes:0} minute{(span.TotalMinutes == 1 ? "" : "s")}" diff --git a/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/HealthFailoverButtonTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/HealthFailoverButtonTests.cs new file mode 100644 index 00000000..2544e78c --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/HealthFailoverButtonTests.cs @@ -0,0 +1,156 @@ +using System.Security.Claims; +using Bunit; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health; +using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared; +using ZB.MOM.WW.ScadaBridge.CentralUI.Services; +using ZB.MOM.WW.ScadaBridge.Security; + +namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.Monitoring; + +/// +/// bUnit tests for the admin "Trigger failover" control on the Health dashboard +/// (decision 2026-07-22). The control is destructive and privileged, so the three +/// things that must hold are: only an Administrator sees it, it is refused when +/// there is no standby to take over, and it never fires without an explicit +/// confirmation. +/// +public class HealthFailoverButtonTests : BunitContext +{ + private readonly IManualFailoverService _failover = Substitute.For(); + private readonly IDialogService _dialog = Substitute.For(); + + private void Arrange(string role) + { + var claims = new[] + { + new Claim(JwtTokenService.UsernameClaimType, "tester"), + new Claim(JwtTokenService.RoleClaimType, role) + }; + var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth")); + Services.AddSingleton(new TestAuthStateProvider(user)); + Services.AddAuthorizationCore(); + AuthorizationPolicies.AddScadaBridgeAuthorization(Services); + // BunitContext pre-registers a placeholder IAuthorizationService that throws when + // AuthorizeView evaluates a policy. Force the real service so the admin gate is + // genuinely exercised rather than trivially satisfied (same note as NavMenuTests). + Services.AddSingleton(); + Services.AddSingleton(_failover); + Services.AddSingleton(_dialog); + } + + /// + /// AuthorizeView requires a cascading Task<AuthenticationState>, which the app + /// supplies from its layout; a bare component render has to provide it explicitly. + /// + private IRenderedComponent Render(int onlineNodes) + { + var host = Render(parameters => parameters + .Add(p => p.ChildContent, (RenderFragment)(builder => + { + builder.OpenComponent(0); + builder.AddAttribute(1, nameof(CentralFailoverControl.OnlineCentralNodeCount), onlineNodes); + builder.CloseComponent(); + }))); + + return host.FindComponent(); + } + + [Fact] + public void Button_is_hidden_for_a_non_admin() + { + Arrange("Viewer"); + + var cut = Render(onlineNodes: 2); + + Assert.Empty(cut.FindAll("button")); + } + + [Fact] + public void Button_is_rendered_and_enabled_for_an_admin_with_a_standby() + { + Arrange("Administrator"); + + var cut = Render(onlineNodes: 2); + + var button = cut.Find("button"); + Assert.False(button.HasAttribute("disabled")); + Assert.Contains("failover", button.TextContent, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Button_is_disabled_with_an_explanatory_title_when_there_is_no_standby() + { + Arrange("Administrator"); + + var cut = Render(onlineNodes: 1); + + var button = cut.Find("button"); + Assert.True(button.HasAttribute("disabled")); + // The tooltip must say WHY, not just that it is unavailable. + Assert.Contains("outage", button.GetAttribute("title"), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Confirming_triggers_the_failover_exactly_once() + { + Arrange("Administrator"); + _dialog.ConfirmAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _failover.FailOverCentralAsync(Arg.Any()) + .Returns(Task.FromResult("akka.tcp://scadabridge@central-a:8081")); + + var cut = Render(onlineNodes: 2); + await cut.Find("button").ClickAsync(new()); + + await _failover.Received(1).FailOverCentralAsync("tester"); + } + + [Fact] + public async Task Cancelling_the_confirmation_does_not_trigger_a_failover() + { + Arrange("Administrator"); + _dialog.ConfirmAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns(false); + + var cut = Render(onlineNodes: 2); + await cut.Find("button").ClickAsync(new()); + + await _failover.DidNotReceive().FailOverCentralAsync(Arg.Any()); + } + + [Fact] + public async Task Confirmation_warns_that_this_page_will_disconnect() + { + // Traefik routes the UI to the ACTIVE node — the very node being failed over — so + // the admin's own circuit drops. If the dialog does not say so, a working failover + // looks like a crash they caused. + Arrange("Administrator"); + _dialog.ConfirmAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + + var cut = Render(onlineNodes: 2); + await cut.Find("button").ClickAsync(new()); + + await _dialog.Received(1).ConfirmAsync( + Arg.Any(), + Arg.Is(m => m.Contains("reconnect", StringComparison.OrdinalIgnoreCase)), + Arg.Any()); + } + + [Fact] + public async Task A_refused_failover_surfaces_the_refusal_instead_of_claiming_success() + { + // The service returns null when the peer guard refuses. The UI must not report a + // failover that never happened. + Arrange("Administrator"); + _dialog.ConfirmAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _failover.FailOverCentralAsync(Arg.Any()).Returns(Task.FromResult(null)); + + var cut = Render(onlineNodes: 2); + await cut.Find("button").ClickAsync(new()); + + Assert.Contains("no standby", cut.Markup, StringComparison.OrdinalIgnoreCase); + } +}