From d8a85c3d890412e2a212e5e4e95ba7b426b5a2aa Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 22 Jul 2026 06:44:44 -0400 Subject: [PATCH] feat(adminui): manual failover control on the cluster redundancy page Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- docs/Redundancy.md | 32 ++- .../Pages/Clusters/ClusterRedundancy.razor | 93 +++++++++ .../Redundancy/ManualFailoverPageModel.cs | 134 +++++++++++++ .../ManualFailoverPageModelTests.cs | 185 ++++++++++++++++++ 4 files changed, 442 insertions(+), 2 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Redundancy/ManualFailoverPageModel.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Redundancy/ManualFailoverPageModelTests.cs diff --git a/docs/Redundancy.md b/docs/Redundancy.md index c0fd7345..fcd6ca7f 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -258,8 +258,36 @@ bootstrap and reading `akka.cluster.downing-provider-class` back off the running the typed option or the akka.conf text is not sufficient, because several HOCON fragments compete and the losing one still looks correct in the file. -There is no operator-driven role swap during a partition; failover is what the cluster and supervisor do -automatically. +Failover during a partition is what the cluster and supervisor do automatically. For a *planned* swap — +patching, draining a node — see [Manual failover](#manual-failover) below. + +## Manual failover + +The cluster redundancy page (`/clusters/{id}/redundancy`) carries a **Live redundancy** panel — the current +driver Primary and every Up `driver` member, read from live cluster state — and an admin-gated **Trigger +failover** button. + +The mechanism is a graceful cluster **`Leave`** of the current Primary, never a `Down`: +`IManualFailoverService.FailOverDriverPrimaryAsync` selects the **oldest Up `driver` member** — the identical +query to `RedundancyStateActor.SelectDriverPrimary`, pinned together by +`ManualFailoverServiceTests.Parity_with_SelectDriverPrimary` so the node acted on is exactly the node the +election names. The leaving node hands its singletons over through the cluster-leave phases, +`CoordinatedShutdown` runs, `ActorSystemTerminationWatchdog` exits the process, the supervisor restarts it, +and it rejoins as the **youngest** member — so the survivor is now the oldest, becomes Primary, and +advertises `ServiceLevel` 250. OPC UA clients re-select. + +| Rule | Behaviour | +|---|---| +| Authorization | `AdminUiPolicies.FleetAdmin` (**Administrator only**) — gated in markup *and* re-checked server-side before the call. Deliberately not `ConfigEditor`, which the neighbouring cluster pages use: this restarts a production node rather than editing config, and ConfigEditor also admits Designer. | +| Peer guard | Disabled, with the reason as its tooltip, when fewer than two Up driver members exist — with one node a "failover" is a shutdown. Re-evaluated inside the service against live state, so a stale page cannot bypass it. | +| Confirmation | A dialog naming the node, and what follows (restart, ServiceLevel 250 moves, clients re-select). | +| Audit | Written through the shared `ZB.MOM.WW.Audit.IAuditWriter` seam **before** the Leave — action `cluster.manual-failover`, actor, target. A *refused* failover changes nothing and writes nothing. | + +> **Mesh-scope caveat.** Until the per-cluster mesh work lands +> (`docs/plans/2026-07-21-per-cluster-mesh-design.md`), the Primary is elected once per Akka mesh rather than +> per application `Cluster` row — so on a fleet running several clusters in one mesh this button acts on the +> whole mesh's Primary, which may belong to a different cluster than the page you are on. The panel says so. +> Phase 6 of the mesh program removes both the caveat and this notice. > **Live gate outstanding.** The auto-down change is verified by unit tests against the effective > configuration, and by the sister project's live drill on an equivalent Akka version and topology. It has diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/ClusterRedundancy.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/ClusterRedundancy.razor index cd9af9aa..36f3201f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/ClusterRedundancy.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/ClusterRedundancy.razor @@ -1,10 +1,16 @@ @page "/clusters/{ClusterId}/redundancy" @attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)] @rendermode RenderMode.InteractiveServer +@using Microsoft.AspNetCore.Authorization @using Microsoft.EntityFrameworkCore +@using ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy @using ZB.MOM.WW.OtOpcUa.Configuration @using ZB.MOM.WW.OtOpcUa.Configuration.Entities +@using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy @inject IDbContextFactory DbFactory +@inject IManualFailoverService FailoverService +@inject AuthenticationStateProvider AuthState +@inject IAuthorizationService AuthorizationService @if (!_loaded) { @@ -45,6 +51,67 @@ else +
+
Live redundancy
+
+
+ Driver Primary + @(_failover.Snapshot?.PrimaryAddress ?? "—") +
+
+ Up driver members + + @(_failover.Snapshot is { DriverAddresses.Count: > 0 } s + ? string.Join(", ", s.DriverAddresses) + : "—") + +
+ +

+ Read live from cluster state on this node. Mesh-wide scope: the Primary is + elected once per Akka mesh, not per cluster row — in the current single-mesh topology this + acts on the whole mesh's Primary, which may be a node of another + Cluster. See docs/Redundancy.md. +

+ + + +
+ + @if (_failover.DisabledReason is { } reason) + { + @reason + } +
+ + @if (_failover.ConfirmOpen) + { +
+ Fail over the driver Primary? +
    +
  • @(_failover.Snapshot?.PrimaryAddress) leaves the cluster and its process restarts.
    • +
  • Its peer becomes Primary and advertises ServiceLevel 250.
    • +
  • Connected OPC UA clients re-select the new Primary.
    • +
+ + +
+ } + + + + @if (_failover.StatusMessage is { } msg) + { +
@msg
+ } +
+
+
Node service-level configuration
@if (_nodes is null || _nodes.Count == 0) @@ -91,8 +158,16 @@ else private ServerCluster? _cluster; private List? _nodes; + // Everything consequential about the failover control (peer guard, confirm flow, outcome text) + // lives in this pure model rather than in the markup — the repo has no bUnit, so logic left in a + // .razor is verified only by driving the page. Covered by ManualFailoverPageModelTests. + private ManualFailoverPageModel _failover = default!; + protected override async Task OnInitializedAsync() { + _failover = new ManualFailoverPageModel(FailoverService); + _failover.Refresh(); + await using var db = await DbFactory.CreateDbContextAsync(); _cluster = await db.ServerClusters.AsNoTracking() .FirstOrDefaultAsync(c => c.ClusterId == ClusterId); @@ -105,4 +180,22 @@ else } _loaded = true; } + + /// + /// Defense-in-depth: the button is FleetAdmin-gated in markup, but this handler runs on the + /// server circuit — re-check the policy before bouncing a production node (the same pattern the + /// certificate-store actions use). + /// + private async Task ConfirmFailoverAsync() + { + var authState = await AuthState.GetAuthenticationStateAsync(); + if (!(await AuthorizationService.AuthorizeAsync( + authState.User, null, ManualFailoverPageModel.RequiredPolicy)).Succeeded) + { + _failover.CancelFailover(); + return; + } + + await _failover.ConfirmFailoverAsync(authState.User.Identity?.Name ?? "system"); + } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Redundancy/ManualFailoverPageModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Redundancy/ManualFailoverPageModel.cs new file mode 100644 index 00000000..20b69c4d --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Redundancy/ManualFailoverPageModel.cs @@ -0,0 +1,134 @@ +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.Security.Auth; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy; + +/// +/// Presentation state for the Trigger-failover control on the cluster redundancy page. +/// +/// +/// +/// A pure model behind a thin razor shell — the same split the driver-typed tag editors use, +/// and for the same reason: this repo has no bUnit (see +/// PageAuthorizationGuardTests), so anything left inside the .razor is verified +/// only by driving the page live. The peer guard, the confirm flow, and the +/// refused-vs-succeeded outcome are consequential enough to be unit-testable, so they live +/// here. +/// +/// +/// What is deliberately NOT here: the markup authorization gate. It is expressed in the razor +/// as <AuthorizeView Policy="@ManualFailoverPageModel.RequiredPolicy"> and +/// re-checked server-side before the call, so is the single +/// source of truth for both. +/// +/// +public sealed class ManualFailoverPageModel +{ + /// + /// The policy gating the control, in markup and in the server-side re-check. + /// + /// + /// (Administrator only) rather than the + /// the neighbouring cluster-authoring pages use: + /// this does not edit configuration, it restarts a production node. ConfigEditor also admits the + /// Designer role, which must not be able to bounce the Primary. + /// + public const string RequiredPolicy = AdminUiPolicies.FleetAdmin; + + private readonly IManualFailoverService _service; + + /// Creates the model over the failover service. + /// The manual-failover seam; faked in tests. + public ManualFailoverPageModel(IManualFailoverService service) + => _service = service ?? throw new ArgumentNullException(nameof(service)); + + /// The last read cluster snapshot, or if it could not be read. + public ManualFailoverSnapshot? Snapshot { get; private set; } + + /// Whether the confirmation dialog is open. + public bool ConfirmOpen { get; private set; } + + /// Status text from the last completed action, if any. + public string? StatusMessage { get; private set; } + + /// Whether reports a failure. + public bool StatusIsError { get; private set; } + + /// Whether the button is enabled — a peer must exist to take over. + public bool CanFailOver => Snapshot?.CanFailOver == true; + + /// + /// Why the button is disabled, or when it is enabled. Rendered as the + /// button's tooltip so a disabled control explains itself. + /// + public string? DisabledReason => Snapshot is null + ? "Cluster state is unavailable on this node." + : Snapshot.CanFailOver + ? null + : $"Failover needs a driver peer to take over; this mesh has {Snapshot.DriverAddresses.Count} Up driver member(s)."; + + /// Re-reads live cluster state. Never throws — an unreadable cluster disables the button. + public void Refresh() + { + try + { + Snapshot = _service.GetSnapshot(); + } + catch (Exception ex) + { + // A node whose ActorSystem is not yet up (or not a cluster provider) must render the page, + // not 500. The button stays disabled with DisabledReason explaining why. + Snapshot = null; + StatusIsError = true; + StatusMessage = $"Could not read cluster state: {ex.Message}"; + } + } + + /// Opens the confirmation dialog. No-op when the peer guard is closed. + public void RequestFailover() + { + if (!CanFailOver) return; + StatusMessage = null; + StatusIsError = false; + ConfirmOpen = true; + } + + /// Closes the confirmation dialog without acting. + public void CancelFailover() => ConfirmOpen = false; + + /// + /// Performs the failover the dialog is confirming. + /// + /// The authenticated user name, recorded in the audit event. + public async Task ConfirmFailoverAsync(string actor) + { + if (!ConfirmOpen) return; + ConfirmOpen = false; + + try + { + var target = await _service.FailOverDriverPrimaryAsync(actor).ConfigureAwait(false); + if (target is null) + { + // The service re-evaluates the peer guard against live state, which may have changed + // since Refresh(). A refusal is not an error — but it must not read as a success. + StatusIsError = true; + StatusMessage = "Failover refused: no driver peer is available to take over."; + } + else + { + StatusIsError = false; + StatusMessage = + $"Failover requested. {target} is leaving the cluster; its peer takes over as Primary " + + "and the node restarts as the youngest member."; + } + } + catch (Exception ex) + { + StatusIsError = true; + StatusMessage = $"Failover failed: {ex.Message}"; + } + + Refresh(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Redundancy/ManualFailoverPageModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Redundancy/ManualFailoverPageModelTests.cs new file mode 100644 index 00000000..f942d1f7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Redundancy/ManualFailoverPageModelTests.cs @@ -0,0 +1,185 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.Security.Auth; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Redundancy; + +/// +/// Covers the Trigger-failover control's behaviour. The repo has no bUnit (see +/// PageAuthorizationGuardTests), so the consequential parts — the peer guard, the confirm +/// flow, and the refused-vs-succeeded outcome — live in a pure model that the razor is a shell over, +/// and are tested here rather than left to live verification alone. +/// +public sealed class ManualFailoverPageModelTests +{ + private sealed class FakeFailoverService : IManualFailoverService + { + public ManualFailoverSnapshot Next { get; set; } = new("akka.tcp://otopcua@a:4053", new[] + { + "akka.tcp://otopcua@a:4053", "akka.tcp://otopcua@b:4053", + }); + + public int FailOverCalls { get; private set; } + public List Actors { get; } = new(); + public string? Result { get; set; } = "akka.tcp://otopcua@a:4053"; + public Exception? Throw { get; set; } + public Exception? ThrowOnSnapshot { get; set; } + + public ManualFailoverSnapshot GetSnapshot() + => ThrowOnSnapshot is not null ? throw ThrowOnSnapshot : Next; + + public Task FailOverDriverPrimaryAsync(string actor) + { + FailOverCalls++; + Actors.Add(actor); + if (Throw is not null) throw Throw; + return Task.FromResult(Result); + } + } + + private static (ManualFailoverPageModel Model, FakeFailoverService Service) Build() + { + var svc = new FakeFailoverService(); + var model = new ManualFailoverPageModel(svc); + model.Refresh(); + return (model, svc); + } + + /// + /// The control is Administrator-only. Not — which the + /// neighbouring cluster-authoring pages use, and which also admits Designer: this restarts a + /// production node rather than editing configuration. + /// + [Fact] + public void Control_is_gated_on_the_fleet_admin_policy() + { + ManualFailoverPageModel.RequiredPolicy.ShouldBe(AdminUiPolicies.FleetAdmin); + ManualFailoverPageModel.RequiredPolicy.ShouldNotBe(AdminUiPolicies.ConfigEditor); + } + + /// With a peer present the button is live and explains nothing away. + [Fact] + public void Button_is_enabled_when_a_driver_peer_exists() + { + var (model, _) = Build(); + + model.CanFailOver.ShouldBeTrue(); + model.DisabledReason.ShouldBeNull(); + model.Snapshot!.PrimaryAddress.ShouldBe("akka.tcp://otopcua@a:4053"); + } + + /// + /// THE peer guard. On a lone driver node a "failover" is a shutdown: the button must be disabled + /// and say why, and requesting it must not even open the dialog. + /// + [Fact] + public void Button_is_disabled_with_a_reason_when_there_is_no_peer() + { + var (model, svc) = Build(); + svc.Next = new ManualFailoverSnapshot("akka.tcp://otopcua@a:4053", new[] { "akka.tcp://otopcua@a:4053" }); + model.Refresh(); + + model.CanFailOver.ShouldBeFalse(); + model.DisabledReason.ShouldNotBeNull().ShouldContain("1 Up driver member"); + + model.RequestFailover(); + model.ConfirmOpen.ShouldBeFalse("a guarded-off control must not open its confirmation dialog"); + } + + /// The confirm flow calls the service exactly once, with the authenticated user. + [Fact] + public async Task Confirm_flow_calls_the_service_exactly_once() + { + var (model, svc) = Build(); + + model.RequestFailover(); + model.ConfirmOpen.ShouldBeTrue(); + await model.ConfirmFailoverAsync("alice"); + + svc.FailOverCalls.ShouldBe(1); + svc.Actors.ShouldBe(new[] { "alice" }); + model.ConfirmOpen.ShouldBeFalse(); + model.StatusIsError.ShouldBeFalse(); + model.StatusMessage.ShouldNotBeNull().ShouldContain("akka.tcp://otopcua@a:4053"); + } + + /// + /// Confirming without an open dialog does nothing — a stray double-submit on the circuit must not + /// bounce a second node. + /// + [Fact] + public async Task Confirming_twice_only_fails_over_once() + { + var (model, svc) = Build(); + + model.RequestFailover(); + await model.ConfirmFailoverAsync("alice"); + await model.ConfirmFailoverAsync("alice"); + + svc.FailOverCalls.ShouldBe(1); + } + + /// Cancelling closes the dialog and calls nothing. + [Fact] + public async Task Cancel_does_not_call_the_service() + { + var (model, svc) = Build(); + + model.RequestFailover(); + model.CancelFailover(); + await model.ConfirmFailoverAsync("alice"); + + model.ConfirmOpen.ShouldBeFalse(); + svc.FailOverCalls.ShouldBe(0); + } + + /// + /// The service re-evaluates the peer guard against live state, which can have changed since the + /// page rendered. A refusal must surface as a failure — reporting it as a success would tell an + /// operator the Primary moved when it did not. + /// + [Fact] + public async Task Refusal_is_reported_as_an_error_not_a_success() + { + var (model, svc) = Build(); + svc.Result = null; + + model.RequestFailover(); + await model.ConfirmFailoverAsync("alice"); + + model.StatusIsError.ShouldBeTrue(); + model.StatusMessage.ShouldNotBeNull().ShouldContain("refused"); + } + + /// A throwing service surfaces as an error, not an unhandled circuit exception. + [Fact] + public async Task Service_failure_is_surfaced_not_thrown() + { + var (model, svc) = Build(); + svc.Throw = new InvalidOperationException("cluster gone"); + + model.RequestFailover(); + await model.ConfirmFailoverAsync("alice"); + + model.StatusIsError.ShouldBeTrue(); + model.StatusMessage.ShouldNotBeNull().ShouldContain("cluster gone"); + } + + /// + /// An unreadable cluster renders a disabled control, not a 500 — the page is also the place an + /// operator looks when the node is unhealthy. + /// + [Fact] + public void Unreadable_cluster_state_disables_the_control_instead_of_throwing() + { + var svc = new FakeFailoverService { ThrowOnSnapshot = new InvalidOperationException("no cluster") }; + var model = new ManualFailoverPageModel(svc); + + Should.NotThrow(model.Refresh); + model.Snapshot.ShouldBeNull(); + model.CanFailOver.ShouldBeFalse(); + model.DisabledReason.ShouldNotBeNull().ShouldContain("unavailable"); + } +}