feat(adminui): manual failover control on the cluster redundancy page

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

docs/Redundancy.md

@@ -258,8 +258,36 @@ bootstrap and reading `akka.cluster.downing-provider-class` back off the running
 the typed option or the akka.conf text is not sufficient, because several HOCON fragments compete and the
 losing one still looks correct in the file.
 
-There is no operator-driven role swap during a partition; failover is what the cluster and supervisor do
+Failover during a partition is what the cluster and supervisor do automatically. For a *planned* swap —
-automatically.
+patching, draining a node — see [Manual failover](#manual-failover) below.
+
+## Manual failover
+
+The cluster redundancy page (`/clusters/{id}/redundancy`) carries a **Live redundancy** panel — the current
+driver Primary and every Up `driver` member, read from live cluster state — and an admin-gated **Trigger
+failover** button.
+
+The mechanism is a graceful cluster **`Leave`** of the current Primary, never a `Down`:
+`IManualFailoverService.FailOverDriverPrimaryAsync` selects the **oldest Up `driver` member** — the identical
+query to `RedundancyStateActor.SelectDriverPrimary`, pinned together by
+`ManualFailoverServiceTests.Parity_with_SelectDriverPrimary` so the node acted on is exactly the node the
+election names. The leaving node hands its singletons over through the cluster-leave phases,
+`CoordinatedShutdown` runs, `ActorSystemTerminationWatchdog` exits the process, the supervisor restarts it,
+and it rejoins as the **youngest** member — so the survivor is now the oldest, becomes Primary, and
+advertises `ServiceLevel` 250. OPC UA clients re-select.
+
+| Rule | Behaviour |
+|---|---|
+| Authorization | `AdminUiPolicies.FleetAdmin` (**Administrator only**) — gated in markup *and* re-checked server-side before the call. Deliberately not `ConfigEditor`, which the neighbouring cluster pages use: this restarts a production node rather than editing config, and ConfigEditor also admits Designer. |
+| Peer guard | Disabled, with the reason as its tooltip, when fewer than two Up driver members exist — with one node a "failover" is a shutdown. Re-evaluated inside the service against live state, so a stale page cannot bypass it. |
+| Confirmation | A dialog naming the node, and what follows (restart, ServiceLevel 250 moves, clients re-select). |
+| Audit | Written through the shared `ZB.MOM.WW.Audit.IAuditWriter` seam **before** the Leave — action `cluster.manual-failover`, actor, target. A *refused* failover changes nothing and writes nothing. |
+
+> **Mesh-scope caveat.** Until the per-cluster mesh work lands
+> (`docs/plans/2026-07-21-per-cluster-mesh-design.md`), the Primary is elected once per Akka mesh rather than
+> per application `Cluster` row — so on a fleet running several clusters in one mesh this button acts on the
+> whole mesh's Primary, which may belong to a different cluster than the page you are on. The panel says so.
+> Phase 6 of the mesh program removes both the caveat and this notice.
 
 > **Live gate outstanding.** The auto-down change is verified by unit tests against the effective
 > configuration, and by the sister project's live drill on an equivalent Akka version and topology. It has

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/ClusterRedundancy.razor

@@ -1,10 +1,16 @@
 @page "/clusters/{ClusterId}/redundancy"
 @attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
 @rendermode RenderMode.InteractiveServer
+@using Microsoft.AspNetCore.Authorization
 @using Microsoft.EntityFrameworkCore
+@using ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy
 @using ZB.MOM.WW.OtOpcUa.Configuration
 @using ZB.MOM.WW.OtOpcUa.Configuration.Entities
+@using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy
 @inject IDbContextFactory<OtOpcUaConfigDbContext> DbFactory
+@inject IManualFailoverService FailoverService
+@inject AuthenticationStateProvider AuthState
+@inject IAuthorizationService AuthorizationService
 
 @if (!_loaded)
 {
@@ -45,6 +51,67 @@ else
         </div>
     </section>
 
+    <section class="panel rise mt-3" style="animation-delay:.11s">
+        <div class="panel-head">Live redundancy</div>
+        <div style="padding:1rem">
+            <div class="kv">
+                <span class="k">Driver Primary</span>
+                <span class="v mono">@(_failover.Snapshot?.PrimaryAddress ?? "—")</span>
+            </div>
+            <div class="kv">
+                <span class="k">Up driver members</span>
+                <span class="v mono">
+                    @(_failover.Snapshot is { DriverAddresses.Count: > 0 } s
+                        ? string.Join(", ", s.DriverAddresses)
+                        : "—")
+                </span>
+            </div>
+
+            <p class="text-muted small mt-2 mb-0">
+                Read live from cluster state on this node. <strong>Mesh-wide scope:</strong> the Primary is
+                elected once per Akka mesh, not per cluster row — in the current single-mesh topology this
+                acts on the whole mesh's Primary, which may be a node of another
+                <span class="mono">Cluster</span>. See <span class="mono">docs/Redundancy.md</span>.
+            </p>
+
+            <AuthorizeView Policy="@ManualFailoverPageModel.RequiredPolicy">
+                <Authorized>
+                    <div class="mt-3">
+                        <button class="btn btn-sm btn-outline-danger"
+                                disabled="@(!_failover.CanFailOver)"
+                                title="@(_failover.DisabledReason ?? "Gracefully move the driver Primary to its peer")"
+                                @onclick="() => _failover.RequestFailover()">
+                            Trigger failover
+                        </button>
+                        @if (_failover.DisabledReason is { } reason)
+                        {
+                            <span class="text-muted small ms-2">@reason</span>
+                        }
+                    </div>
+
+                    @if (_failover.ConfirmOpen)
+                    {
+                        <div class="panel notice mt-3">
+                            <strong>Fail over the driver Primary?</strong>
+                            <ul class="mb-2 mt-2">
+                                <li><span class="mono">@(_failover.Snapshot?.PrimaryAddress)</span> leaves the cluster and its process restarts.</li>
+                                <li>Its peer becomes Primary and advertises <span class="mono">ServiceLevel</span> 250.</li>
+                                <li>Connected OPC UA clients re-select the new Primary.</li>
+                            </ul>
+                            <button class="btn btn-sm btn-danger" @onclick="ConfirmFailoverAsync">Confirm failover</button>
+                            <button class="btn btn-sm btn-outline-secondary ms-2" @onclick="() => _failover.CancelFailover()">Cancel</button>
+                        </div>
+                    }
+                </Authorized>
+            </AuthorizeView>
+
+            @if (_failover.StatusMessage is { } msg)
+            {
+                <div class="mt-3 @(_failover.StatusIsError ? "text-danger" : "text-success")">@msg</div>
+            }
+        </div>
+    </section>
+
     <section class="panel rise mt-3" style="animation-delay:.14s">
         <div class="panel-head">Node service-level configuration</div>
         @if (_nodes is null || _nodes.Count == 0)
@@ -91,8 +158,16 @@ else
     private ServerCluster? _cluster;
     private List<ClusterNode>? _nodes;
 
+    // Everything consequential about the failover control (peer guard, confirm flow, outcome text)
+    // lives in this pure model rather than in the markup — the repo has no bUnit, so logic left in a
+    // .razor is verified only by driving the page. Covered by ManualFailoverPageModelTests.
+    private ManualFailoverPageModel _failover = default!;
+
     protected override async Task OnInitializedAsync()
     {
+        _failover = new ManualFailoverPageModel(FailoverService);
+        _failover.Refresh();
+
         await using var db = await DbFactory.CreateDbContextAsync();
         _cluster = await db.ServerClusters.AsNoTracking()
             .FirstOrDefaultAsync(c => c.ClusterId == ClusterId);
@@ -105,4 +180,22 @@ else
         }
         _loaded = true;
     }
+
+    /// <summary>
+    /// Defense-in-depth: the button is FleetAdmin-gated in markup, but this handler runs on the
+    /// server circuit — re-check the policy before bouncing a production node (the same pattern the
+    /// certificate-store actions use).
+    /// </summary>
+    private async Task ConfirmFailoverAsync()
+    {
+        var authState = await AuthState.GetAuthenticationStateAsync();
+        if (!(await AuthorizationService.AuthorizeAsync(
+                authState.User, null, ManualFailoverPageModel.RequiredPolicy)).Succeeded)
+        {
+            _failover.CancelFailover();
+            return;
+        }
+
+        await _failover.ConfirmFailoverAsync(authState.User.Identity?.Name ?? "system");
+    }
 }

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Redundancy/ManualFailoverPageModel.cs

@@ -0,0 +1,134 @@
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Security.Auth;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy;
+
+/// <summary>
+/// Presentation state for the Trigger-failover control on the cluster redundancy page.
+/// </summary>
+/// <remarks>
+///     <para>
+///         A pure model behind a thin razor shell — the same split the driver-typed tag editors use,
+///         and for the same reason: this repo has no bUnit (see
+///         <c>PageAuthorizationGuardTests</c>), so anything left inside the <c>.razor</c> is verified
+///         only by driving the page live. The peer guard, the confirm flow, and the
+///         refused-vs-succeeded outcome are consequential enough to be unit-testable, so they live
+///         here.
+///     </para>
+///     <para>
+///         What is deliberately NOT here: the markup authorization gate. It is expressed in the razor
+///         as <c>&lt;AuthorizeView Policy="@ManualFailoverPageModel.RequiredPolicy"&gt;</c> and
+///         re-checked server-side before the call, so <see cref="RequiredPolicy"/> is the single
+///         source of truth for both.
+///     </para>
+/// </remarks>
+public sealed class ManualFailoverPageModel
+{
+    /// <summary>
+    /// The policy gating the control, in markup and in the server-side re-check.
+    /// </summary>
+    /// <remarks>
+    /// <see cref="AdminUiPolicies.FleetAdmin"/> (Administrator only) rather than the
+    /// <see cref="AdminUiPolicies.ConfigEditor"/> the neighbouring cluster-authoring pages use:
+    /// this does not edit configuration, it restarts a production node. ConfigEditor also admits the
+    /// Designer role, which must not be able to bounce the Primary.
+    /// </remarks>
+    public const string RequiredPolicy = AdminUiPolicies.FleetAdmin;
+
+    private readonly IManualFailoverService _service;
+
+    /// <summary>Creates the model over the failover service.</summary>
+    /// <param name="service">The manual-failover seam; faked in tests.</param>
+    public ManualFailoverPageModel(IManualFailoverService service)
+        => _service = service ?? throw new ArgumentNullException(nameof(service));
+
+    /// <summary>The last read cluster snapshot, or <see langword="null"/> if it could not be read.</summary>
+    public ManualFailoverSnapshot? Snapshot { get; private set; }
+
+    /// <summary>Whether the confirmation dialog is open.</summary>
+    public bool ConfirmOpen { get; private set; }
+
+    /// <summary>Status text from the last completed action, if any.</summary>
+    public string? StatusMessage { get; private set; }
+
+    /// <summary>Whether <see cref="StatusMessage"/> reports a failure.</summary>
+    public bool StatusIsError { get; private set; }
+
+    /// <summary>Whether the button is enabled — a peer must exist to take over.</summary>
+    public bool CanFailOver => Snapshot?.CanFailOver == true;
+
+    /// <summary>
+    /// Why the button is disabled, or <see langword="null"/> when it is enabled. Rendered as the
+    /// button's tooltip so a disabled control explains itself.
+    /// </summary>
+    public string? DisabledReason => Snapshot is null
+        ? "Cluster state is unavailable on this node."
+        : Snapshot.CanFailOver
+            ? null
+            : $"Failover needs a driver peer to take over; this mesh has {Snapshot.DriverAddresses.Count} Up driver member(s).";
+
+    /// <summary>Re-reads live cluster state. Never throws — an unreadable cluster disables the button.</summary>
+    public void Refresh()
+    {
+        try
+        {
+            Snapshot = _service.GetSnapshot();
+        }
+        catch (Exception ex)
+        {
+            // A node whose ActorSystem is not yet up (or not a cluster provider) must render the page,
+            // not 500. The button stays disabled with DisabledReason explaining why.
+            Snapshot = null;
+            StatusIsError = true;
+            StatusMessage = $"Could not read cluster state: {ex.Message}";
+        }
+    }
+
+    /// <summary>Opens the confirmation dialog. No-op when the peer guard is closed.</summary>
+    public void RequestFailover()
+    {
+        if (!CanFailOver) return;
+        StatusMessage = null;
+        StatusIsError = false;
+        ConfirmOpen = true;
+    }
+
+    /// <summary>Closes the confirmation dialog without acting.</summary>
+    public void CancelFailover() => ConfirmOpen = false;
+
+    /// <summary>
+    /// Performs the failover the dialog is confirming.
+    /// </summary>
+    /// <param name="actor">The authenticated user name, recorded in the audit event.</param>
+    public async Task ConfirmFailoverAsync(string actor)
+    {
+        if (!ConfirmOpen) return;
+        ConfirmOpen = false;
+
+        try
+        {
+            var target = await _service.FailOverDriverPrimaryAsync(actor).ConfigureAwait(false);
+            if (target is null)
+            {
+                // The service re-evaluates the peer guard against live state, which may have changed
+                // since Refresh(). A refusal is not an error — but it must not read as a success.
+                StatusIsError = true;
+                StatusMessage = "Failover refused: no driver peer is available to take over.";
+            }
+            else
+            {
+                StatusIsError = false;
+                StatusMessage =
+                    $"Failover requested. {target} is leaving the cluster; its peer takes over as Primary "
+                    + "and the node restarts as the youngest member.";
+            }
+        }
+        catch (Exception ex)
+        {
+            StatusIsError = true;
+            StatusMessage = $"Failover failed: {ex.Message}";
+        }
+
+        Refresh();
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Redundancy/ManualFailoverPageModelTests.cs

@@ -0,0 +1,185 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Redundancy;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Security.Auth;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Redundancy;
+
+/// <summary>
+/// Covers the Trigger-failover control's behaviour. The repo has no bUnit (see
+/// <c>PageAuthorizationGuardTests</c>), so the consequential parts — the peer guard, the confirm
+/// flow, and the refused-vs-succeeded outcome — live in a pure model that the razor is a shell over,
+/// and are tested here rather than left to live verification alone.
+/// </summary>
+public sealed class ManualFailoverPageModelTests
+{
+    private sealed class FakeFailoverService : IManualFailoverService
+    {
+        public ManualFailoverSnapshot Next { get; set; } = new("akka.tcp://otopcua@a:4053", new[]
+        {
+            "akka.tcp://otopcua@a:4053", "akka.tcp://otopcua@b:4053",
+        });
+
+        public int FailOverCalls { get; private set; }
+        public List<string> Actors { get; } = new();
+        public string? Result { get; set; } = "akka.tcp://otopcua@a:4053";
+        public Exception? Throw { get; set; }
+        public Exception? ThrowOnSnapshot { get; set; }
+
+        public ManualFailoverSnapshot GetSnapshot()
+            => ThrowOnSnapshot is not null ? throw ThrowOnSnapshot : Next;
+
+        public Task<string?> FailOverDriverPrimaryAsync(string actor)
+        {
+            FailOverCalls++;
+            Actors.Add(actor);
+            if (Throw is not null) throw Throw;
+            return Task.FromResult(Result);
+        }
+    }
+
+    private static (ManualFailoverPageModel Model, FakeFailoverService Service) Build()
+    {
+        var svc = new FakeFailoverService();
+        var model = new ManualFailoverPageModel(svc);
+        model.Refresh();
+        return (model, svc);
+    }
+
+    /// <summary>
+    /// The control is Administrator-only. Not <see cref="AdminUiPolicies.ConfigEditor"/> — which the
+    /// neighbouring cluster-authoring pages use, and which also admits Designer: this restarts a
+    /// production node rather than editing configuration.
+    /// </summary>
+    [Fact]
+    public void Control_is_gated_on_the_fleet_admin_policy()
+    {
+        ManualFailoverPageModel.RequiredPolicy.ShouldBe(AdminUiPolicies.FleetAdmin);
+        ManualFailoverPageModel.RequiredPolicy.ShouldNotBe(AdminUiPolicies.ConfigEditor);
+    }
+
+    /// <summary>With a peer present the button is live and explains nothing away.</summary>
+    [Fact]
+    public void Button_is_enabled_when_a_driver_peer_exists()
+    {
+        var (model, _) = Build();
+
+        model.CanFailOver.ShouldBeTrue();
+        model.DisabledReason.ShouldBeNull();
+        model.Snapshot!.PrimaryAddress.ShouldBe("akka.tcp://otopcua@a:4053");
+    }
+
+    /// <summary>
+    /// THE peer guard. On a lone driver node a "failover" is a shutdown: the button must be disabled
+    /// and say why, and requesting it must not even open the dialog.
+    /// </summary>
+    [Fact]
+    public void Button_is_disabled_with_a_reason_when_there_is_no_peer()
+    {
+        var (model, svc) = Build();
+        svc.Next = new ManualFailoverSnapshot("akka.tcp://otopcua@a:4053", new[] { "akka.tcp://otopcua@a:4053" });
+        model.Refresh();
+
+        model.CanFailOver.ShouldBeFalse();
+        model.DisabledReason.ShouldNotBeNull().ShouldContain("1 Up driver member");
+
+        model.RequestFailover();
+        model.ConfirmOpen.ShouldBeFalse("a guarded-off control must not open its confirmation dialog");
+    }
+
+    /// <summary>The confirm flow calls the service exactly once, with the authenticated user.</summary>
+    [Fact]
+    public async Task Confirm_flow_calls_the_service_exactly_once()
+    {
+        var (model, svc) = Build();
+
+        model.RequestFailover();
+        model.ConfirmOpen.ShouldBeTrue();
+        await model.ConfirmFailoverAsync("alice");
+
+        svc.FailOverCalls.ShouldBe(1);
+        svc.Actors.ShouldBe(new[] { "alice" });
+        model.ConfirmOpen.ShouldBeFalse();
+        model.StatusIsError.ShouldBeFalse();
+        model.StatusMessage.ShouldNotBeNull().ShouldContain("akka.tcp://otopcua@a:4053");
+    }
+
+    /// <summary>
+    /// Confirming without an open dialog does nothing — a stray double-submit on the circuit must not
+    /// bounce a second node.
+    /// </summary>
+    [Fact]
+    public async Task Confirming_twice_only_fails_over_once()
+    {
+        var (model, svc) = Build();
+
+        model.RequestFailover();
+        await model.ConfirmFailoverAsync("alice");
+        await model.ConfirmFailoverAsync("alice");
+
+        svc.FailOverCalls.ShouldBe(1);
+    }
+
+    /// <summary>Cancelling closes the dialog and calls nothing.</summary>
+    [Fact]
+    public async Task Cancel_does_not_call_the_service()
+    {
+        var (model, svc) = Build();
+
+        model.RequestFailover();
+        model.CancelFailover();
+        await model.ConfirmFailoverAsync("alice");
+
+        model.ConfirmOpen.ShouldBeFalse();
+        svc.FailOverCalls.ShouldBe(0);
+    }
+
+    /// <summary>
+    /// The service re-evaluates the peer guard against live state, which can have changed since the
+    /// page rendered. A refusal must surface as a failure — reporting it as a success would tell an
+    /// operator the Primary moved when it did not.
+    /// </summary>
+    [Fact]
+    public async Task Refusal_is_reported_as_an_error_not_a_success()
+    {
+        var (model, svc) = Build();
+        svc.Result = null;
+
+        model.RequestFailover();
+        await model.ConfirmFailoverAsync("alice");
+
+        model.StatusIsError.ShouldBeTrue();
+        model.StatusMessage.ShouldNotBeNull().ShouldContain("refused");
+    }
+
+    /// <summary>A throwing service surfaces as an error, not an unhandled circuit exception.</summary>
+    [Fact]
+    public async Task Service_failure_is_surfaced_not_thrown()
+    {
+        var (model, svc) = Build();
+        svc.Throw = new InvalidOperationException("cluster gone");
+
+        model.RequestFailover();
+        await model.ConfirmFailoverAsync("alice");
+
+        model.StatusIsError.ShouldBeTrue();
+        model.StatusMessage.ShouldNotBeNull().ShouldContain("cluster gone");
+    }
+
+    /// <summary>
+    /// An unreadable cluster renders a disabled control, not a 500 — the page is also the place an
+    /// operator looks when the node is unhealthy.
+    /// </summary>
+    [Fact]
+    public void Unreadable_cluster_state_disables_the_control_instead_of_throwing()
+    {
+        var svc = new FakeFailoverService { ThrowOnSnapshot = new InvalidOperationException("no cluster") };
+        var model = new ManualFailoverPageModel(svc);
+
+        Should.NotThrow(model.Refresh);
+        model.Snapshot.ShouldBeNull();
+        model.CanFailOver.ShouldBeFalse();
+        model.DisabledReason.ShouldNotBeNull().ShouldContain("unavailable");
+    }
+}