diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/IManualFailoverService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/IManualFailoverService.cs new file mode 100644 index 00000000..f7369427 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/IManualFailoverService.cs @@ -0,0 +1,54 @@ +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; + +/// +/// A read-only view of the cluster's driver membership, as the manual-failover control sees it. +/// +/// +/// The current driver Primary — the oldest Up driver member, the same node +/// names — or when +/// there is no Up driver member at all. +/// +/// +/// Every Up driver member, oldest first. A manual failover needs at least two: with one +/// there is nothing to fail over to. +/// +public sealed record ManualFailoverSnapshot(string? PrimaryAddress, IReadOnlyList DriverAddresses) +{ + /// Whether a failover can be performed — i.e. a peer exists to take over. + public bool CanFailOver => DriverAddresses.Count >= 2; +} + +/// +/// Operator-initiated, graceful swap of the driver Primary. +/// +/// +/// +/// The mechanism is a cluster Leave of the current Primary, never a Down. The +/// leaving node hands its singletons over through the normal cluster-leave phases, +/// CoordinatedShutdown runs, ActorSystemTerminationWatchdog exits the process, +/// the service supervisor restarts it, and it rejoins as the youngest member — so the +/// survivor becomes the oldest Up driver member and therefore Primary. ServiceLevel 250 +/// moves with it and OPC UA clients re-select. +/// +/// +/// Mesh-scope caveat. Until the per-cluster mesh work lands +/// (docs/plans/2026-07-21-per-cluster-mesh-design.md) the Primary is elected once per +/// Akka mesh, not per application Cluster row. On a fleet running several application +/// clusters in one mesh this acts on the whole mesh's Primary. The UI says so. +/// +/// +public interface IManualFailoverService +{ + /// Reads the current driver membership from live cluster state. + ManualFailoverSnapshot GetSnapshot(); + + /// + /// Gracefully removes the current driver Primary from the cluster so its peer takes over. + /// + /// The authenticated identity performing the failover; recorded in the audit event. + /// + /// The address of the node asked to leave, or when the failover was + /// refused because no driver peer exists (in which case nothing was changed and nothing audited). + /// + Task FailOverDriverPrimaryAsync(string actor); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs new file mode 100644 index 00000000..ddedd053 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs @@ -0,0 +1,128 @@ +using System.Text.Json; +using Akka.Actor; +using Akka.Cluster; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.Audit; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; + +/// +/// Default — a graceful cluster Leave of the oldest Up +/// driver member. See the interface for the full behaviour and the mesh-scope caveat. +/// +public sealed class ManualFailoverService : IManualFailoverService +{ + /// The audit stamped onto every manual failover. + public const string AuditAction = "cluster.manual-failover"; + + /// The audit stamped onto every manual failover. + public const string AuditCategory = "Redundancy"; + + private readonly Func _system; + private readonly IAuditWriter _audit; + private readonly ILogger _log; + + /// + /// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance: + /// this is registered before Akka's hosted service has built the system, so resolving eagerly + /// would race startup (the same lazy-accessor pattern ActorSystemTerminationWatchdog uses). + /// + /// Lazy accessor for the node's ActorSystem. + /// The shared audit seam; a failover is recorded before it is performed. + /// Logger for the failover decision. + public ManualFailoverService(Func system, IAuditWriter audit, ILogger log) + { + _system = system ?? throw new ArgumentNullException(nameof(system)); + _audit = audit ?? throw new ArgumentNullException(nameof(audit)); + _log = log ?? throw new ArgumentNullException(nameof(log)); + } + + /// + public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system()); + + /// + public async Task FailOverDriverPrimaryAsync(string actor) + { + ArgumentException.ThrowIfNullOrWhiteSpace(actor); + + var system = _system(); + + // Dry run first so the peer guard is evaluated before anything is audited: a refused + // failover changed nothing and must not leave an audit row implying it did. + var target = FailOverCore(system, dryRun: true); + if (target is null) + { + _log.LogWarning( + "Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.", + actor); + return null; + } + + await _audit.WriteAsync(new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = actor, + Action = AuditAction, + Category = AuditCategory, + SourceNode = target.ToString(), + Outcome = AuditOutcome.Success, + DetailsJson = JsonSerializer.Serialize(new + { + target = target.ToString(), + driverMembers = SnapshotCore(system).DriverAddresses, + }), + }).ConfigureAwait(false); + + _log.LogWarning( + "Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. " + + "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest " + + "member; its peer becomes Primary and advertises the primary ServiceLevel.", + actor, + target); + + FailOverCore(system, dryRun: false); + return target.ToString(); + } + + /// + /// Selects — and, unless , gracefully removes — the driver Primary. + /// + /// The ActorSystem whose cluster is acted on. + /// When , selects the target without leaving. + /// The target address, or when fewer than two Up driver members exist. + public static Address? FailOverCore(ActorSystem system, bool dryRun = false) + { + ArgumentNullException.ThrowIfNull(system); + + var cluster = Akka.Cluster.Cluster.Get(system); + + // Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver, + // Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a + // failover would remove a node that was not Primary and change nothing. Pinned by + // ManualFailoverServiceTests.Parity_with_SelectDriverPrimary. + var drivers = OrderedDrivers(cluster.State.Members); + + // A failover needs somewhere to fail over TO. With a single driver member, leaving would be + // a shutdown dressed up as a failover. + if (drivers.Count < 2) return null; + + var primary = drivers[0]; + if (!dryRun) cluster.Leave(primary.Address); + return primary.Address; + } + + private static ManualFailoverSnapshot SnapshotCore(ActorSystem system) + { + var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members); + + return new ManualFailoverSnapshot( + drivers.FirstOrDefault()?.Address.ToString(), + drivers.Select(m => m.Address.ToString()).ToList()); + } + + private static List OrderedDrivers(IEnumerable members) => members + .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole)) + .OrderBy(m => m, Member.AgeOrdering) + .ToList(); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 0da86839..ef193f04 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.ControlPlane; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -372,6 +373,13 @@ if (hasAdmin) builder.Services.AddCascadingAuthenticationState(); builder.Services.AddSignalR(); builder.Services.AddOtOpcUaAdminClients(); + // Manual failover backs the Trigger-failover control on the cluster redundancy page. Admin-only: + // it is driven from the UI, and the node it acts on is chosen from live cluster state, so any + // admin node can drive it regardless of whether that node is the Primary. The ActorSystem comes + // through the Func<> accessor below (never resolved at construction — this runs before Akka's + // hosted service has built the system). + builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + builder.Services.AddSingleton(); } // Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ManualFailoverServiceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ManualFailoverServiceTests.cs new file mode 100644 index 00000000..c2f08852 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ManualFailoverServiceTests.cs @@ -0,0 +1,226 @@ +using Akka.Actor; +using Akka.Cluster; +using Akka.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Pins which node a manual failover removes, on a real two-node cluster built so that the +/// oldest member is not the lowest-addressed one. +/// +/// +/// +/// The fixture mirrors deliberately, and for the +/// same reason: age order and address order coincide on a freshly-formed cluster and diverge +/// after any restart. A failover that targeted the role leader (lowest address) rather +/// than the oldest member would, after a restart, remove the node that is not hosting +/// the singletons — leaving the Primary exactly where it was while taking an innocent node +/// down. Node A therefore binds the higher port and joins first. +/// +/// +/// The parity test is the durable one: it pins the service's target to the member +/// actually names Primary, so the two five-line queries +/// cannot drift apart silently. +/// +/// +public sealed class ManualFailoverServiceTests : IAsyncLifetime +{ + // Distinct from RedundancyPrimaryElectionTests' 19530/19531 — xunit may run the two classes in + // parallel, and both need deterministic (non-zero) ports. + private const int OldestPort = 19_541; // joins FIRST -> oldest, but the HIGHER address + private const int YoungestPort = 19_540; // joins SECOND -> role leader, the LOWER address + + private ActorSystem? _oldest; + private ActorSystem? _youngest; + + private static Config NodeConfig(int port, string systemName, int seedPort) => + ConfigurationFactory.ParseString($$""" + akka { + loglevel = "WARNING" + actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster" + remote.dot-netty.tcp { + hostname = "127.0.0.1" + port = {{port}} + } + cluster { + seed-nodes = ["akka.tcp://{{systemName}}@127.0.0.1:{{seedPort}}"] + roles = ["admin", "driver"] + min-nr-of-members = 1 + run-coordinated-shutdown-when-down = off + downing-provider-class = "" + } + } + """); + + public async Task InitializeAsync() + { + // Order matters: the seed forms the cluster and is therefore the oldest member. + _oldest = ActorSystem.Create("manual-failover", NodeConfig(OldestPort, "manual-failover", OldestPort)); + await WaitForUpAsync(_oldest, expectedMembers: 1); + + _youngest = ActorSystem.Create("manual-failover", NodeConfig(YoungestPort, "manual-failover", OldestPort)); + await WaitForUpAsync(_oldest, expectedMembers: 2); + await WaitForUpAsync(_youngest, expectedMembers: 2); + } + + public async Task DisposeAsync() + { + if (_youngest is not null) await _youngest.Terminate(); + if (_oldest is not null) await _oldest.Terminate(); + } + + private static async Task WaitForUpAsync(ActorSystem system, int expectedMembers) + { + var cluster = Akka.Cluster.Cluster.Get(system); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers) return; + await Task.Delay(100); + } + + throw new TimeoutException( + $"cluster on port {cluster.SelfAddress.Port} never saw {expectedMembers} Up members " + + $"(saw {cluster.State.Members.Count(m => m.Status == MemberStatus.Up)})"); + } + + private static (ManualFailoverService Service, RecordingAuditWriter Audit) BuildService(ActorSystem system) + { + var audit = new RecordingAuditWriter(); + return (new ManualFailoverService(() => system, audit, NullLogger.Instance), audit); + } + + /// + /// Fixture sanity check: if age order and address order did not actually diverge, the assertion + /// below would pass for the wrong reason. + /// + [Fact] + public void Fixture_actually_produces_divergent_age_and_address_ordering() + { + Akka.Cluster.Cluster.Get(_oldest!).State.RoleLeader("driver")!.Port.ShouldBe( + YoungestPort, + "the fixture is only meaningful if the lowest-addressed node is the YOUNGER one"); + } + + /// + /// The failover removes the oldest driver member — the node hosting the singletons — not the + /// role leader, and the survivor is then elected Primary. + /// + [Fact] + public async Task FailOver_targets_the_oldest_driver_member_not_the_role_leader() + { + // Driven from the survivor, as production does: the admin node asks the Primary to leave. + var (service, audit) = BuildService(_youngest!); + + var target = await service.FailOverDriverPrimaryAsync("tester"); + + target.ShouldNotBeNull(); + target.ShouldContain( + $":{OldestPort}", + Case.Sensitive, + "the failover must remove the OLDEST driver member (which holds the singletons), not the role leader"); + + // The leaving node runs CoordinatedShutdown and terminates — that is what makes the process + // exit and be restarted as the youngest member. + await _oldest!.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30)); + + // ... and the survivor is now what RedundancyStateActor names Primary. + var snapshot = await CaptureSnapshotAsync(_youngest!); + var primaries = snapshot.Where(n => n.Role == RedundancyRole.Primary).ToList(); + primaries.Count.ShouldBe(1, "exactly one node may be Primary after the swap"); + primaries[0].NodeId.Value.ShouldBe($"127.0.0.1:{YoungestPort}"); + + audit.Events.Count.ShouldBe(1, "a manual failover must be audited exactly once"); + audit.Events[0].Action.ShouldBe(ManualFailoverService.AuditAction); + audit.Events[0].Actor.ShouldBe("tester"); + audit.Events[0].SourceNode.ShouldNotBeNull().ShouldContain($":{OldestPort}"); + } + + /// + /// With no peer to take over, a failover is a disguised shutdown. It must refuse — and must not + /// audit, because nothing happened. + /// + [Fact] + public async Task FailOver_refuses_when_no_driver_peer_exists() + { + const int LonePort = 19_542; + var lone = ActorSystem.Create("manual-failover-lone", NodeConfig(LonePort, "manual-failover-lone", LonePort)); + try + { + await WaitForUpAsync(lone, expectedMembers: 1); + var (service, audit) = BuildService(lone); + + var result = await service.FailOverDriverPrimaryAsync("tester"); + + result.ShouldBeNull("a single driver member has nowhere to fail over to"); + audit.Events.ShouldBeEmpty("a refused failover changed nothing and must not be audited"); + + // Positive assert: the node is still a live Up member — the refusal did not half-leave it. + Akka.Cluster.Cluster.Get(lone).State.Members + .Count(m => m.Status == MemberStatus.Up) + .ShouldBe(1); + lone.WhenTerminated.IsCompleted.ShouldBeFalse(); + } + finally + { + await lone.Terminate(); + } + } + + /// + /// The service's target and are two copies + /// of the same query. This pins them together, so a change to one that is not made to the other + /// fails here rather than in production, where it would mean failing over the wrong node. + /// + [Fact] + public async Task Parity_with_SelectDriverPrimary() + { + var elected = RedundancyStateActor.SelectDriverPrimary( + Akka.Cluster.Cluster.Get(_oldest!).State.Members); + + var target = ManualFailoverService.FailOverCore(_oldest!, dryRun: true); + var snapshotPrimary = BuildService(_oldest!).Service.GetSnapshot().PrimaryAddress; + + elected.ShouldNotBeNull(); + target.ShouldBe(elected, "the node acted on must be exactly the node the election names Primary"); + snapshotPrimary.ShouldBe(elected.ToString(), "the UI panel must name the same node the button acts on"); + + // The dry run really was dry. + await Task.Delay(500); + Akka.Cluster.Cluster.Get(_oldest!).State.Members + .Count(m => m.Status == MemberStatus.Up) + .ShouldBe(2, "a dry run must not remove anyone"); + } + + private static async Task> CaptureSnapshotAsync(ActorSystem system) + { + var tcs = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + system.ActorOf( + RedundancyStateActor.Props(broadcast: msg => + { + if (msg is RedundancyStateChanged changed) tcs.TrySetResult(changed); + }), + $"redundancy-{Guid.NewGuid():N}"); + + return (await tcs.Task.WaitAsync(TimeSpan.FromSeconds(15))).Nodes; + } + + private sealed class RecordingAuditWriter : IAuditWriter + { + public List Events { get; } = new(); + + public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default) + { + Events.Add(auditEvent); + return Task.CompletedTask; + } + } +}