dohertj2/lmxopcua
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(redundancy): manual failover service — graceful Leave of the driver Primary

Browse Source 
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-22 06:40:58 -04:00
parent 983310edbb
commit 69b697bc3e
4 changed files with 416 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/IManualFailoverService.cs
+54
View File
@@ -0,0 +1,54 @@
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
/// <summary>
/// A read-only view of the cluster's driver membership, as the manual-failover control sees it.
/// </summary>
/// <param name="PrimaryAddress">
/// The current driver Primary — the oldest Up <c>driver</c> member, the same node
/// <see cref="RedundancyStateActor.SelectDriverPrimary"/> names — or <see langword="null"/> when
/// there is no Up driver member at all.
/// </param>
/// <param name="DriverAddresses">
/// Every Up <c>driver</c> member, oldest first. A manual failover needs at least two: with one
/// there is nothing to fail over to.
/// </param>
public sealed record ManualFailoverSnapshot(string? PrimaryAddress, IReadOnlyList<string> DriverAddresses)
{
/// <summary>Whether a failover can be performed — i.e. a peer exists to take over.</summary>
public bool CanFailOver => DriverAddresses.Count >= 2;
}
/// <summary>
/// Operator-initiated, graceful swap of the driver Primary.
/// </summary>
/// <remarks>
/// <para>
/// The mechanism is a cluster <b>Leave</b> of the current Primary, never a <c>Down</c>. The
/// leaving node hands its singletons over through the normal cluster-leave phases,
/// <c>CoordinatedShutdown</c> runs, <c>ActorSystemTerminationWatchdog</c> exits the process,
/// the service supervisor restarts it, and it rejoins as the youngest member — so the
/// survivor becomes the oldest Up driver member and therefore Primary. ServiceLevel 250
/// moves with it and OPC UA clients re-select.
/// </para>
/// <para>
/// <b>Mesh-scope caveat.</b> Until the per-cluster mesh work lands
/// (<c>docs/plans/2026-07-21-per-cluster-mesh-design.md</c>) the Primary is elected once per
/// Akka mesh, not per application <c>Cluster</c> row. On a fleet running several application
/// clusters in one mesh this acts on the whole mesh's Primary. The UI says so.
/// </para>
/// </remarks>
public interface IManualFailoverService
{
/// <summary>Reads the current driver membership from live cluster state.</summary>
ManualFailoverSnapshot GetSnapshot();
/// <summary>
/// Gracefully removes the current driver Primary from the cluster so its peer takes over.
/// </summary>
/// <param name="actor">The authenticated identity performing the failover; recorded in the audit event.</param>
/// <returns>
/// The address of the node asked to leave, or <see langword="null"/> when the failover was
/// refused because no driver peer exists (in which case nothing was changed and nothing audited).
/// </returns>
Task<string?> FailOverDriverPrimaryAsync(string actor);
}
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs
+128
View File
@@ -0,0 +1,128 @@
using System.Text.Json;
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.Audit;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
/// <summary>
/// Default <see cref="IManualFailoverService"/> — a graceful cluster <c>Leave</c> of the oldest Up
/// <c>driver</c> member. See the interface for the full behaviour and the mesh-scope caveat.
/// </summary>
public sealed class ManualFailoverService : IManualFailoverService
{
/// <summary>The audit <see cref="AuditEvent.Action"/> stamped onto every manual failover.</summary>
public const string AuditAction = "cluster.manual-failover";
/// <summary>The audit <see cref="AuditEvent.Category"/> stamped onto every manual failover.</summary>
public const string AuditCategory = "Redundancy";
private readonly Func<ActorSystem> _system;
private readonly IAuditWriter _audit;
private readonly ILogger<ManualFailoverService> _log;
/// <summary>
/// Creates the service. The ActorSystem is taken as a factory, never as a constructed instance:
/// this is registered before Akka's hosted service has built the system, so resolving eagerly
/// would race startup (the same lazy-accessor pattern <c>ActorSystemTerminationWatchdog</c> uses).
/// </summary>
/// <param name="system">Lazy accessor for the node's ActorSystem.</param>
/// <param name="audit">The shared audit seam; a failover is recorded before it is performed.</param>
/// <param name="log">Logger for the failover decision.</param>
public ManualFailoverService(Func<ActorSystem> system, IAuditWriter audit, ILogger<ManualFailoverService> log)
{
_system = system ?? throw new ArgumentNullException(nameof(system));
_audit = audit ?? throw new ArgumentNullException(nameof(audit));
_log = log ?? throw new ArgumentNullException(nameof(log));
}
/// <inheritdoc/>
public ManualFailoverSnapshot GetSnapshot() => SnapshotCore(_system());
/// <inheritdoc/>
public async Task<string?> FailOverDriverPrimaryAsync(string actor)
{
ArgumentException.ThrowIfNullOrWhiteSpace(actor);
var system = _system();
// Dry run first so the peer guard is evaluated before anything is audited: a refused
// failover changed nothing and must not leave an audit row implying it did.
var target = FailOverCore(system, dryRun: true);
if (target is null)
{
_log.LogWarning(
"Manual failover refused for {Actor}: fewer than two Up driver members — there is no peer to take over.",
actor);
return null;
}
await _audit.WriteAsync(new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTimeOffset.UtcNow,
Actor = actor,
Action = AuditAction,
Category = AuditCategory,
SourceNode = target.ToString(),
Outcome = AuditOutcome.Success,
DetailsJson = JsonSerializer.Serialize(new
{
target = target.ToString(),
driverMembers = SnapshotCore(system).DriverAddresses,
}),
}).ConfigureAwait(false);
_log.LogWarning(
"Manual failover requested by {Actor}: asking the driver Primary {Target} to leave the cluster. "
+ "It hands over its singletons, shuts down, and is restarted by its supervisor as the youngest "
+ "member; its peer becomes Primary and advertises the primary ServiceLevel.",
actor,
target);
FailOverCore(system, dryRun: false);
return target.ToString();
}
/// <summary>
/// Selects — and, unless <paramref name="dryRun"/>, gracefully removes — the driver Primary.
/// </summary>
/// <param name="system">The ActorSystem whose cluster is acted on.</param>
/// <param name="dryRun">When <see langword="true"/>, selects the target without leaving.</param>
/// <returns>The target address, or <see langword="null"/> when fewer than two Up driver members exist.</returns>
public static Address? FailOverCore(ActorSystem system, bool dryRun = false)
{
ArgumentNullException.ThrowIfNull(system);
var cluster = Akka.Cluster.Cluster.Get(system);
// Intentionally identical to RedundancyStateActor.SelectDriverPrimary (oldest Up driver,
// Member.AgeOrdering) — the node acted on must be exactly the one the election names, or a
// failover would remove a node that was not Primary and change nothing. Pinned by
// ManualFailoverServiceTests.Parity_with_SelectDriverPrimary.
var drivers = OrderedDrivers(cluster.State.Members);
// A failover needs somewhere to fail over TO. With a single driver member, leaving would be
// a shutdown dressed up as a failover.
if (drivers.Count < 2) return null;
var primary = drivers[0];
if (!dryRun) cluster.Leave(primary.Address);
return primary.Address;
}
private static ManualFailoverSnapshot SnapshotCore(ActorSystem system)
{
var drivers = OrderedDrivers(Akka.Cluster.Cluster.Get(system).State.Members);
return new ManualFailoverSnapshot(
drivers.FirstOrDefault()?.Address.ToString(),
drivers.Select(m => m.Address.ToString()).ToList());
}
private static List<Member> OrderedDrivers(IEnumerable<Member> members) => members
.Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole))
.OrderBy(m => m, Member.AgeOrdering)
.ToList();
}
src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs
+8
View File
@@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.ControlPlane;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
@@ -372,6 +373,13 @@ if (hasAdmin)
builder.Services.AddCascadingAuthenticationState();
builder.Services.AddSignalR();
builder.Services.AddOtOpcUaAdminClients();
// Manual failover backs the Trigger-failover control on the cluster redundancy page. Admin-only:
// it is driven from the UI, and the node it acts on is chosen from live cluster state, so any
// admin node can drive it regardless of whether that node is the Primary. The ActorSystem comes
// through the Func<> accessor below (never resolved at construction — this runs before Akka's
// hosted service has built the system).
builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
builder.Services.AddSingleton<IManualFailoverService, ManualFailoverService>();
}
// Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI.
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ManualFailoverServiceTests.cs
+226
View File
@@ -0,0 +1,226 @@
using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// Pins <b>which</b> node a manual failover removes, on a real two-node cluster built so that the
/// oldest member is not the lowest-addressed one.
/// </summary>
/// <remarks>
/// <para>
/// The fixture mirrors <see cref="RedundancyPrimaryElectionTests"/> deliberately, and for the
/// same reason: age order and address order coincide on a freshly-formed cluster and diverge
/// after any restart. A failover that targeted the <i>role leader</i> (lowest address) rather
/// than the oldest member would, after a restart, remove the node that is <b>not</b> hosting
/// the singletons — leaving the Primary exactly where it was while taking an innocent node
/// down. Node A therefore binds the <b>higher</b> port and joins first.
/// </para>
/// <para>
/// The parity test is the durable one: it pins the service's target to the member
/// <see cref="RedundancyStateActor"/> actually names Primary, so the two five-line queries
/// cannot drift apart silently.
/// </para>
/// </remarks>
public sealed class ManualFailoverServiceTests : IAsyncLifetime
{
// Distinct from RedundancyPrimaryElectionTests' 19530/19531 — xunit may run the two classes in
// parallel, and both need deterministic (non-zero) ports.
private const int OldestPort = 19_541; // joins FIRST -> oldest, but the HIGHER address
private const int YoungestPort = 19_540; // joins SECOND -> role leader, the LOWER address
private ActorSystem? _oldest;
private ActorSystem? _youngest;
private static Config NodeConfig(int port, string systemName, int seedPort) =>
ConfigurationFactory.ParseString($$"""
akka {
loglevel = "WARNING"
actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster"
remote.dot-netty.tcp {
hostname = "127.0.0.1"
port = {{port}}
}
cluster {
seed-nodes = ["akka.tcp://{{systemName}}@127.0.0.1:{{seedPort}}"]
roles = ["admin", "driver"]
min-nr-of-members = 1
run-coordinated-shutdown-when-down = off
downing-provider-class = ""
}
}
""");
public async Task InitializeAsync()
{
// Order matters: the seed forms the cluster and is therefore the oldest member.
_oldest = ActorSystem.Create("manual-failover", NodeConfig(OldestPort, "manual-failover", OldestPort));
await WaitForUpAsync(_oldest, expectedMembers: 1);
_youngest = ActorSystem.Create("manual-failover", NodeConfig(YoungestPort, "manual-failover", OldestPort));
await WaitForUpAsync(_oldest, expectedMembers: 2);
await WaitForUpAsync(_youngest, expectedMembers: 2);
}
public async Task DisposeAsync()
{
if (_youngest is not null) await _youngest.Terminate();
if (_oldest is not null) await _oldest.Terminate();
}
private static async Task WaitForUpAsync(ActorSystem system, int expectedMembers)
{
var cluster = Akka.Cluster.Cluster.Get(system);
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (DateTime.UtcNow < deadline)
{
if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers) return;
await Task.Delay(100);
}
throw new TimeoutException(
$"cluster on port {cluster.SelfAddress.Port} never saw {expectedMembers} Up members "
+ $"(saw {cluster.State.Members.Count(m => m.Status == MemberStatus.Up)})");
}
private static (ManualFailoverService Service, RecordingAuditWriter Audit) BuildService(ActorSystem system)
{
var audit = new RecordingAuditWriter();
return (new ManualFailoverService(() => system, audit, NullLogger<ManualFailoverService>.Instance), audit);
}
/// <summary>
/// Fixture sanity check: if age order and address order did not actually diverge, the assertion
/// below would pass for the wrong reason.
/// </summary>
[Fact]
public void Fixture_actually_produces_divergent_age_and_address_ordering()
{
Akka.Cluster.Cluster.Get(_oldest!).State.RoleLeader("driver")!.Port.ShouldBe(
YoungestPort,
"the fixture is only meaningful if the lowest-addressed node is the YOUNGER one");
}
/// <summary>
/// The failover removes the oldest driver member — the node hosting the singletons — not the
/// role leader, and the survivor is then elected Primary.
/// </summary>
[Fact]
public async Task FailOver_targets_the_oldest_driver_member_not_the_role_leader()
{
// Driven from the survivor, as production does: the admin node asks the Primary to leave.
var (service, audit) = BuildService(_youngest!);
var target = await service.FailOverDriverPrimaryAsync("tester");
target.ShouldNotBeNull();
target.ShouldContain(
$":{OldestPort}",
Case.Sensitive,
"the failover must remove the OLDEST driver member (which holds the singletons), not the role leader");
// The leaving node runs CoordinatedShutdown and terminates — that is what makes the process
// exit and be restarted as the youngest member.
await _oldest!.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30));
// ... and the survivor is now what RedundancyStateActor names Primary.
var snapshot = await CaptureSnapshotAsync(_youngest!);
var primaries = snapshot.Where(n => n.Role == RedundancyRole.Primary).ToList();
primaries.Count.ShouldBe(1, "exactly one node may be Primary after the swap");
primaries[0].NodeId.Value.ShouldBe($"127.0.0.1:{YoungestPort}");
audit.Events.Count.ShouldBe(1, "a manual failover must be audited exactly once");
audit.Events[0].Action.ShouldBe(ManualFailoverService.AuditAction);
audit.Events[0].Actor.ShouldBe("tester");
audit.Events[0].SourceNode.ShouldNotBeNull().ShouldContain($":{OldestPort}");
}
/// <summary>
/// With no peer to take over, a failover is a disguised shutdown. It must refuse — and must not
/// audit, because nothing happened.
/// </summary>
[Fact]
public async Task FailOver_refuses_when_no_driver_peer_exists()
{
const int LonePort = 19_542;
var lone = ActorSystem.Create("manual-failover-lone", NodeConfig(LonePort, "manual-failover-lone", LonePort));
try
{
await WaitForUpAsync(lone, expectedMembers: 1);
var (service, audit) = BuildService(lone);
var result = await service.FailOverDriverPrimaryAsync("tester");
result.ShouldBeNull("a single driver member has nowhere to fail over to");
audit.Events.ShouldBeEmpty("a refused failover changed nothing and must not be audited");
// Positive assert: the node is still a live Up member — the refusal did not half-leave it.
Akka.Cluster.Cluster.Get(lone).State.Members
.Count(m => m.Status == MemberStatus.Up)
.ShouldBe(1);
lone.WhenTerminated.IsCompleted.ShouldBeFalse();
}
finally
{
await lone.Terminate();
}
}
/// <summary>
/// The service's target and <see cref="RedundancyStateActor.SelectDriverPrimary"/> are two copies
/// of the same query. This pins them together, so a change to one that is not made to the other
/// fails here rather than in production, where it would mean failing over the wrong node.
/// </summary>
[Fact]
public async Task Parity_with_SelectDriverPrimary()
{
var elected = RedundancyStateActor.SelectDriverPrimary(
Akka.Cluster.Cluster.Get(_oldest!).State.Members);
var target = ManualFailoverService.FailOverCore(_oldest!, dryRun: true);
var snapshotPrimary = BuildService(_oldest!).Service.GetSnapshot().PrimaryAddress;
elected.ShouldNotBeNull();
target.ShouldBe(elected, "the node acted on must be exactly the node the election names Primary");
snapshotPrimary.ShouldBe(elected.ToString(), "the UI panel must name the same node the button acts on");
// The dry run really was dry.
await Task.Delay(500);
Akka.Cluster.Cluster.Get(_oldest!).State.Members
.Count(m => m.Status == MemberStatus.Up)
.ShouldBe(2, "a dry run must not remove anyone");
}
private static async Task<IReadOnlyList<NodeRedundancyState>> CaptureSnapshotAsync(ActorSystem system)
{
var tcs = new TaskCompletionSource<RedundancyStateChanged>(
TaskCreationOptions.RunContinuationsAsynchronously);
system.ActorOf(
RedundancyStateActor.Props(broadcast: msg =>
{
if (msg is RedundancyStateChanged changed) tcs.TrySetResult(changed);
}),
$"redundancy-{Guid.NewGuid():N}");
return (await tcs.Task.WaitAsync(TimeSpan.FromSeconds(15))).Nodes;
}
private sealed class RecordingAuditWriter : IAuditWriter
{
public List<AuditEvent> Events { get; } = new();
public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default)
{
Events.Add(auditEvent);
return Task.CompletedTask;
}
}
}