feat(redundancy): manual failover service — graceful Leave of the driver Primary

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/IManualFailoverService.cs

@@ -0,0 +1,54 @@
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+
+/// <summary>
+/// A read-only view of the cluster's driver membership, as the manual-failover control sees it.
+/// </summary>
+/// <param name="PrimaryAddress">
+/// The current driver Primary — the oldest Up <c>driver</c> member, the same node
+/// <see cref="RedundancyStateActor.SelectDriverPrimary"/> names — or <see langword="null"/> when
+/// there is no Up driver member at all.
+/// </param>
+/// <param name="DriverAddresses">
+/// Every Up <c>driver</c> member, oldest first. A manual failover needs at least two: with one
+/// there is nothing to fail over to.
+/// </param>
+public sealed record ManualFailoverSnapshot(string? PrimaryAddress, IReadOnlyList<string> DriverAddresses)
+{
+    /// <summary>Whether a failover can be performed — i.e. a peer exists to take over.</summary>
+    public bool CanFailOver => DriverAddresses.Count >= 2;
+}
+
+/// <summary>
+/// Operator-initiated, graceful swap of the driver Primary.
+/// </summary>
+/// <remarks>
+///     <para>
+///         The mechanism is a cluster <b>Leave</b> of the current Primary, never a <c>Down</c>. The
+///         leaving node hands its singletons over through the normal cluster-leave phases,
+///         <c>CoordinatedShutdown</c> runs, <c>ActorSystemTerminationWatchdog</c> exits the process,
+///         the service supervisor restarts it, and it rejoins as the youngest member — so the
+///         survivor becomes the oldest Up driver member and therefore Primary. ServiceLevel 250
+///         moves with it and OPC UA clients re-select.
+///     </para>
+///     <para>
+///         <b>Mesh-scope caveat.</b> Until the per-cluster mesh work lands
+///         (<c>docs/plans/2026-07-21-per-cluster-mesh-design.md</c>) the Primary is elected once per
+///         Akka mesh, not per application <c>Cluster</c> row. On a fleet running several application
+///         clusters in one mesh this acts on the whole mesh's Primary. The UI says so.
+///     </para>
+/// </remarks>
+public interface IManualFailoverService
+{
+    /// <summary>Reads the current driver membership from live cluster state.</summary>
+    ManualFailoverSnapshot GetSnapshot();
+
+    /// <summary>
+    /// Gracefully removes the current driver Primary from the cluster so its peer takes over.
+    /// </summary>
+    /// <param name="actor">The authenticated identity performing the failover; recorded in the audit event.</param>
+    /// <returns>
+    /// The address of the node asked to leave, or <see langword="null"/> when the failover was
+    /// refused because no driver peer exists (in which case nothing was changed and nothing audited).
+    /// </returns>
+    Task<string?> FailOverDriverPrimaryAsync(string actor);
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs

@@ -0,0 +1,128 @@
+using System.Text.Json;
+using Akka.Actor;
+using Akka.Cluster;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.Audit;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+
+/// <summary>
+/// Default <see cref="IManualFailoverService"/> — a graceful cluster <c>Leave</c> of the oldest Up
+/// <c>driver</c> member. See the interface for the full behaviour and the mesh-scope caveat.
+/// </summary>
+public sealed class ManualFailoverService : IManualFailoverService
+{
+    /// <summary>The audit <see cref="AuditEvent.Action"/> stamped onto every manual failover.</summary>
+    public const string AuditAction = "cluster.manual-failover";
+
+    /// <summary>The audit <see cref="AuditEvent.Category"/> stamped onto every manual failover.</summary>
+    public const string AuditCategory = "Redundancy";
+
+    private readonly Func<ActorSystem> _system;
+    private readonly IAuditWriter _audit;
+    private readonly ILogger<ManualFailoverService> _log;
+
+    /// <summary>
+    /// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance:
+    /// this is registered before Akka's hosted service has built the system, so resolving eagerly
+    /// would race startup (the same lazy-accessor pattern <c>ActorSystemTerminationWatchdog</c> uses).
+    /// </summary>
+    /// <param name="system">Lazy accessor for the node's ActorSystem.</param>
+    /// <param name="audit">The shared audit seam; a failover is recorded before it is performed.</param>
+    /// <param name="log">Logger for the failover decision.</param>
+    public ManualFailoverService(Func<ActorSystem> system, IAuditWriter audit, ILogger<ManualFailoverService> log)
+    {
+        _system = system ?? throw new ArgumentNullException(nameof(system));
+        _audit = audit ?? throw new ArgumentNullException(nameof(audit));
+        _log = log ?? throw new ArgumentNullException(nameof(log));
+    }
+
+    /// <inheritdoc/>
+    public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system());
+
+    /// <inheritdoc/>
+    public async Task<string?> FailOverDriverPrimaryAsync(string actor)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(actor);
+
+        var system = _system();
+
+        // Dry run first so the peer guard is evaluated before anything is audited: a refused
+        // failover changed nothing and must not leave an audit row implying it did.
+        var target = FailOverCore(system, dryRun: true);
+        if (target is null)
+        {
+            _log.LogWarning(
+                "Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.",
+                actor);
+            return null;
+        }
+
+        await _audit.WriteAsync(new AuditEvent
+        {
+            EventId = Guid.NewGuid(),
+            OccurredAtUtc = DateTimeOffset.UtcNow,
+            Actor = actor,
+            Action = AuditAction,
+            Category = AuditCategory,
+            SourceNode = target.ToString(),
+            Outcome = AuditOutcome.Success,
+            DetailsJson = JsonSerializer.Serialize(new
+            {
+                target = target.ToString(),
+                driverMembers = SnapshotCore(system).DriverAddresses,
+            }),
+        }).ConfigureAwait(false);
+
+        _log.LogWarning(
+            "Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. "
+            + "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest "
+            + "member; its peer becomes Primary and advertises the primary ServiceLevel.",
+            actor,
+            target);
+
+        FailOverCore(system, dryRun: false);
+        return target.ToString();
+    }
+
+    /// <summary>
+    /// Selects — and, unless <paramref name="dryRun"/>, gracefully removes — the driver Primary.
+    /// </summary>
+    /// <param name="system">The ActorSystem whose cluster is acted on.</param>
+    /// <param name="dryRun">When <see langword="true"/>, selects the target without leaving.</param>
+    /// <returns>The target address, or <see langword="null"/> when fewer than two Up driver members exist.</returns>
+    public static Address? FailOverCore(ActorSystem system, bool dryRun = false)
+    {
+        ArgumentNullException.ThrowIfNull(system);
+
+        var cluster = Akka.Cluster.Cluster.Get(system);
+
+        // Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver,
+        // Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a
+        // failover would remove a node that was not Primary and change nothing. Pinned by
+        // ManualFailoverServiceTests.Parity_with_SelectDriverPrimary.
+        var drivers = OrderedDrivers(cluster.State.Members);
+
+        // A failover needs somewhere to fail over TO. With a single driver member, leaving would be
+        // a shutdown dressed up as a failover.
+        if (drivers.Count < 2) return null;
+
+        var primary = drivers[0];
+        if (!dryRun) cluster.Leave(primary.Address);
+        return primary.Address;
+    }
+
+    private static ManualFailoverSnapshot SnapshotCore(ActorSystem system)
+    {
+        var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members);
+
+        return new ManualFailoverSnapshot(
+            drivers.FirstOrDefault()?.Address.ToString(),
+            drivers.Select(m => m.Address.ToString()).ToList());
+    }
+
+    private static List<Member> OrderedDrivers(IEnumerable<Member> members) => members
+        .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole))
+        .OrderBy(m => m, Member.AgeOrdering)
+        .ToList();
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.ControlPlane;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting;
@@ -372,6 +373,13 @@ if (hasAdmin)
     builder.Services.AddCascadingAuthenticationState();
     builder.Services.AddSignalR();
     builder.Services.AddOtOpcUaAdminClients();
+    // Manual failover backs the Trigger-failover control on the cluster redundancy page. Admin-only:
+    // it is driven from the UI, and the node it acts on is chosen from live cluster state, so any
+    // admin node can drive it regardless of whether that node is the Primary. The ActorSystem comes
+    // through the Func<> accessor below (never resolved at construction — this runs before Akka's
+    // hosted service has built the system).
+    builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
+    builder.Services.AddSingleton<IManualFailoverService, ManualFailoverService>();
 }
 
 // Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI.

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ManualFailoverServiceTests.cs

@@ -0,0 +1,226 @@
+using Akka.Actor;
+using Akka.Cluster;
+using Akka.Configuration;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Audit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+/// Pins <b>which</b> node a manual failover removes, on a real two-node cluster built so that the
+/// oldest member is not the lowest-addressed one.
+/// </summary>
+/// <remarks>
+///     <para>
+///         The fixture mirrors <see cref="RedundancyPrimaryElectionTests"/> deliberately, and for the
+///         same reason: age order and address order coincide on a freshly-formed cluster and diverge
+///         after any restart. A failover that targeted the <i>role leader</i> (lowest address) rather
+///         than the oldest member would, after a restart, remove the node that is <b>not</b> hosting
+///         the singletons — leaving the Primary exactly where it was while taking an innocent node
+///         down. Node A therefore binds the <b>higher</b> port and joins first.
+///     </para>
+///     <para>
+///         The parity test is the durable one: it pins the service's target to the member
+///         <see cref="RedundancyStateActor"/> actually names Primary, so the two five-line queries
+///         cannot drift apart silently.
+///     </para>
+/// </remarks>
+public sealed class ManualFailoverServiceTests : IAsyncLifetime
+{
+    // Distinct from RedundancyPrimaryElectionTests' 19530/19531 — xunit may run the two classes in
+    // parallel, and both need deterministic (non-zero) ports.
+    private const int OldestPort = 19_541;   // joins FIRST  -> oldest, but the HIGHER address
+    private const int YoungestPort = 19_540; // joins SECOND -> role leader, the LOWER address
+
+    private ActorSystem? _oldest;
+    private ActorSystem? _youngest;
+
+    private static Config NodeConfig(int port, string systemName, int seedPort) =>
+        ConfigurationFactory.ParseString($$"""
+        akka {
+            loglevel = "WARNING"
+            actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster"
+            remote.dot-netty.tcp {
+                hostname = "127.0.0.1"
+                port = {{port}}
+            }
+            cluster {
+                seed-nodes = ["akka.tcp://{{systemName}}@127.0.0.1:{{seedPort}}"]
+                roles = ["admin", "driver"]
+                min-nr-of-members = 1
+                run-coordinated-shutdown-when-down = off
+                downing-provider-class = ""
+            }
+        }
+        """);
+
+    public async Task InitializeAsync()
+    {
+        // Order matters: the seed forms the cluster and is therefore the oldest member.
+        _oldest = ActorSystem.Create("manual-failover", NodeConfig(OldestPort, "manual-failover", OldestPort));
+        await WaitForUpAsync(_oldest, expectedMembers: 1);
+
+        _youngest = ActorSystem.Create("manual-failover", NodeConfig(YoungestPort, "manual-failover", OldestPort));
+        await WaitForUpAsync(_oldest, expectedMembers: 2);
+        await WaitForUpAsync(_youngest, expectedMembers: 2);
+    }
+
+    public async Task DisposeAsync()
+    {
+        if (_youngest is not null) await _youngest.Terminate();
+        if (_oldest is not null) await _oldest.Terminate();
+    }
+
+    private static async Task WaitForUpAsync(ActorSystem system, int expectedMembers)
+    {
+        var cluster = Akka.Cluster.Cluster.Get(system);
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+        while (DateTime.UtcNow < deadline)
+        {
+            if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers) return;
+            await Task.Delay(100);
+        }
+
+        throw new TimeoutException(
+            $"cluster on port {cluster.SelfAddress.Port} never saw {expectedMembers} Up members "
+            + $"(saw {cluster.State.Members.Count(m => m.Status == MemberStatus.Up)})");
+    }
+
+    private static (ManualFailoverService Service, RecordingAuditWriter Audit) BuildService(ActorSystem system)
+    {
+        var audit = new RecordingAuditWriter();
+        return (new ManualFailoverService(() => system, audit, NullLogger<ManualFailoverService>.Instance), audit);
+    }
+
+    /// <summary>
+    /// Fixture sanity check: if age order and address order did not actually diverge, the assertion
+    /// below would pass for the wrong reason.
+    /// </summary>
+    [Fact]
+    public void Fixture_actually_produces_divergent_age_and_address_ordering()
+    {
+        Akka.Cluster.Cluster.Get(_oldest!).State.RoleLeader("driver")!.Port.ShouldBe(
+            YoungestPort,
+            "the fixture is only meaningful if the lowest-addressed node is the YOUNGER one");
+    }
+
+    /// <summary>
+    /// The failover removes the oldest driver member — the node hosting the singletons — not the
+    /// role leader, and the survivor is then elected Primary.
+    /// </summary>
+    [Fact]
+    public async Task FailOver_targets_the_oldest_driver_member_not_the_role_leader()
+    {
+        // Driven from the survivor, as production does: the admin node asks the Primary to leave.
+        var (service, audit) = BuildService(_youngest!);
+
+        var target = await service.FailOverDriverPrimaryAsync("tester");
+
+        target.ShouldNotBeNull();
+        target.ShouldContain(
+            $":{OldestPort}",
+            Case.Sensitive,
+            "the failover must remove the OLDEST driver member (which holds the singletons), not the role leader");
+
+        // The leaving node runs CoordinatedShutdown and terminates — that is what makes the process
+        // exit and be restarted as the youngest member.
+        await _oldest!.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30));
+
+        // ... and the survivor is now what RedundancyStateActor names Primary.
+        var snapshot = await CaptureSnapshotAsync(_youngest!);
+        var primaries = snapshot.Where(n => n.Role == RedundancyRole.Primary).ToList();
+        primaries.Count.ShouldBe(1, "exactly one node may be Primary after the swap");
+        primaries[0].NodeId.Value.ShouldBe($"127.0.0.1:{YoungestPort}");
+
+        audit.Events.Count.ShouldBe(1, "a manual failover must be audited exactly once");
+        audit.Events[0].Action.ShouldBe(ManualFailoverService.AuditAction);
+        audit.Events[0].Actor.ShouldBe("tester");
+        audit.Events[0].SourceNode.ShouldNotBeNull().ShouldContain($":{OldestPort}");
+    }
+
+    /// <summary>
+    /// With no peer to take over, a failover is a disguised shutdown. It must refuse — and must not
+    /// audit, because nothing happened.
+    /// </summary>
+    [Fact]
+    public async Task FailOver_refuses_when_no_driver_peer_exists()
+    {
+        const int LonePort = 19_542;
+        var lone = ActorSystem.Create("manual-failover-lone", NodeConfig(LonePort, "manual-failover-lone", LonePort));
+        try
+        {
+            await WaitForUpAsync(lone, expectedMembers: 1);
+            var (service, audit) = BuildService(lone);
+
+            var result = await service.FailOverDriverPrimaryAsync("tester");
+
+            result.ShouldBeNull("a single driver member has nowhere to fail over to");
+            audit.Events.ShouldBeEmpty("a refused failover changed nothing and must not be audited");
+
+            // Positive assert: the node is still a live Up member — the refusal did not half-leave it.
+            Akka.Cluster.Cluster.Get(lone).State.Members
+                .Count(m => m.Status == MemberStatus.Up)
+                .ShouldBe(1);
+            lone.WhenTerminated.IsCompleted.ShouldBeFalse();
+        }
+        finally
+        {
+            await lone.Terminate();
+        }
+    }
+
+    /// <summary>
+    /// The service's target and <see cref="RedundancyStateActor.SelectDriverPrimary"/> are two copies
+    /// of the same query. This pins them together, so a change to one that is not made to the other
+    /// fails here rather than in production, where it would mean failing over the wrong node.
+    /// </summary>
+    [Fact]
+    public async Task Parity_with_SelectDriverPrimary()
+    {
+        var elected = RedundancyStateActor.SelectDriverPrimary(
+            Akka.Cluster.Cluster.Get(_oldest!).State.Members);
+
+        var target = ManualFailoverService.FailOverCore(_oldest!, dryRun: true);
+        var snapshotPrimary = BuildService(_oldest!).Service.GetSnapshot().PrimaryAddress;
+
+        elected.ShouldNotBeNull();
+        target.ShouldBe(elected, "the node acted on must be exactly the node the election names Primary");
+        snapshotPrimary.ShouldBe(elected.ToString(), "the UI panel must name the same node the button acts on");
+
+        // The dry run really was dry.
+        await Task.Delay(500);
+        Akka.Cluster.Cluster.Get(_oldest!).State.Members
+            .Count(m => m.Status == MemberStatus.Up)
+            .ShouldBe(2, "a dry run must not remove anyone");
+    }
+
+    private static async Task<IReadOnlyList<NodeRedundancyState>> CaptureSnapshotAsync(ActorSystem system)
+    {
+        var tcs = new TaskCompletionSource<RedundancyStateChanged>(
+            TaskCreationOptions.RunContinuationsAsynchronously);
+
+        system.ActorOf(
+            RedundancyStateActor.Props(broadcast: msg =>
+            {
+                if (msg is RedundancyStateChanged changed) tcs.TrySetResult(changed);
+            }),
+            $"redundancy-{Guid.NewGuid():N}");
+
+        return (await tcs.Task.WaitAsync(TimeSpan.FromSeconds(15))).Nodes;
+    }
+
+    private sealed class RecordingAuditWriter : IAuditWriter
+    {
+        public List<AuditEvent> Events { get; } = new();
+
+        public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default)
+        {
+            Events.Add(auditEvent);
+            return Task.CompletedTask;
+        }
+    }
+}