diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index b637f413..d19fe9bd 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -133,6 +133,14 @@ services: # host raise the VM memory or run fewer host services. mem_limit: 2g mem_reservation: 1g + # Restart supervision. A 2-node keep-oldest SBR downs the LONE survivor when the + # OLDEST node crashes (the survivor is "the side without the oldest" → DownReachable + # → run-coordinated-shutdown-when-down terminates it). Recovery is by exit-and-rejoin: + # the supervisor restarts the exited node and it re-forms/rejoins as a fresh + # incarnation (the ScadaBridge pattern). Without a restart policy the exited node + # stays down and the cluster looks like a total outage. Inherited by every host node + # via the anchor merge. + restart: unless-stopped depends_on: sql: { condition: service_healthy } migrator: { condition: service_completed_successfully } @@ -143,7 +151,11 @@ services: Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "central-1" + # Both redundancy peers are seeds (#459 / ScadaBridge parity) so a restarted node can + # re-join the mesh via EITHER peer, not only central-1. With restart supervision this is + # the 2-node keep-oldest exit-and-rejoin recovery path. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" @@ -204,7 +216,9 @@ services: Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "central-2" + # Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" diff --git a/docs/Redundancy.md b/docs/Redundancy.md index f16e7248..19fe48b7 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -161,23 +161,46 @@ provider reads the `split-brain-resolver` HOCON block in `akka.conf`. On top of `ServiceCollectionExtensions.BuildClusterOptions` sets the typed `ClusterOptions.SplitBrainResolver` (`KeepOldestOption { DownIfAlone = true }`) to make the strategy **explicit in code** rather than relying on the framework default — it is reinforcing, not the sole activator, and yields the same effective behavior. So -the cluster is **not** running `NoDowning`; hard-crashed nodes are downed and both the cluster singletons and -the `driver` role-leader fail over. (Only an *explicit* `NoDowning`, e.g. +the cluster is **not** running `NoDowning`; hard-crashed nodes are downed and the cluster recovers (how it +recovers depends on *which* node was lost — see the table below). (Only an *explicit* `NoDowning`, e.g. `akka.cluster.downing-provider-class = ""`, would leave both redundancy sides at ServiceLevel 240 indefinitely.) The HOCON block carries the tuning: `active-strategy = keep-oldest`, `stable-after = 15s`, `keep-oldest.down-if-alone = on`, `failure-detector.threshold = 10.0` (its `active-strategy` + `down-if-alone` must stay consistent with the typed option; `stable-after` lives only in HOCON because the typed option can't express it). -`keep-oldest` is the correct strategy for a 2-node warm-redundancy pair: under a clean partition the oldest -member (typically the long-running primary) stays up and the smaller (or younger) side downs itself within -~`stable-after` seconds; `down-if-alone` downs a node that loses its only peer. `keep-majority`/`static-quorum` -are wrong for two nodes (no majority in a 1-1 split). The `RedundancyStateActor` on the surviving partition -re-computes from the post-partition `Cluster.State`, and a hard-crashed (not gracefully stopped) node -triggers the same failover — verified by `HardKillFailoverTests` (its negative control confirms failover -survives removing the typed option, and only breaks under an explicit `NoDowning`). +`keep-oldest` is the correct strategy for a 2-node warm-redundancy pair (`keep-majority`/`static-quorum` +are wrong for two nodes — no majority in a 1-1 split), but its two loss cases recover **differently**, and +this is inherent to any 2-node cluster: -There is no operator-driven role swap during a partition. Failover is what the cluster does automatically. +| Node lost | What keep-oldest does | Recovery | +|---|---|---| +| **Younger** node (crash or partition) | The oldest is on the surviving side → it stays up and downs the younger side (`DownUnreachable`). | **In-place, fast.** The oldest keeps its singletons + `driver` role-leadership; `RedundancyStateActor` re-computes from the post-loss `Cluster.State`. | +| **Oldest** node (crash or partition) | The lone survivor is "the side **without** the oldest" → keep-oldest downs the **survivor itself** (`DownReachable "including myself"`), and `run-coordinated-shutdown-when-down = on` terminates it. `down-if-alone` does **not** rescue a lone survivor (its rescue branch needs ≥ 2 surviving members, so it is a 3+-node feature). | **Exit-and-rejoin.** The self-downed survivor **and** the crashed oldest are both restarted by the service supervisor and re-form / rejoin. There is a brief restart-window outage; there is **no** in-place takeover. | + +**Recovery therefore depends on three things being in place** (the [ScadaBridge](../../ScadaBridge/CLAUDE.md) +sister project runs the same pattern): + +1. **A service supervisor that restarts the process** on exit — production `Install-Services.ps1` sets + `sc.exe failure OtOpcUaHost … actions= restart/5000/restart/30000/restart/60000`; the docker-dev rig sets + `restart: unless-stopped`. Without it a downed node stays down and an oldest-crash looks like a total outage. +2. **The recovery watchdog** `ActorSystemTerminationWatchdog` (registered in `Program.cs` after `AddAkka`) — it + watches `ActorSystem.WhenTerminated` and, on an unexpected self-down, calls + `IHostApplicationLifetime.StopApplication()` so the process exits (and the supervisor restarts it) instead of + idling forever with a dead actor system. +3. **Both peers listed in `SeedNodes`** on every node, so a restarted node can re-join the mesh via either peer. + +> **On `HardKillFailoverTests`:** that in-process test hard-kills the oldest and asserts the survivor becomes +> sole `driver` role-leader, and it passes — but it simulates the crash with `provider.Transport.Shutdown()`, +> which leaves node A's `ActorSystem` **alive** (a transport partition with the oldest still running), not a +> real process death. It is therefore **not** representative of an oldest-process crash; a real 2-container +> `docker kill` of the oldest downs the survivor (verified 2026-07-15, see +> `archreview/plans/artifacts/459-oldest-crash-live-finding-2026-07-15.md`). Treat the recovery guarantee for +> the oldest as "exit-and-rejoin under supervision," not "in-place failover." + +There is no operator-driven role swap during a partition. Failover / recovery is what the cluster + supervisor +do automatically. **Instant in-place takeover on *any* single-node loss requires 3+ members** (an odd cluster +or a lightweight witness) — a deliberate future option, not the current 2-node posture. ## Primary data-plane gate (writes, acks, alerts emit) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/ActorSystemTerminationWatchdog.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/ActorSystemTerminationWatchdog.cs new file mode 100644 index 00000000..5662c1a4 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/ActorSystemTerminationWatchdog.cs @@ -0,0 +1,80 @@ +using Akka.Actor; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Host; + +/// +/// Down-if-alone recovery watchdog (arch-review #459). The 2-node keep-oldest split-brain +/// resolver downs the lone survivor when the OLDEST node crashes (the survivor is "the side +/// without the oldest" → DownReachable), and run-coordinated-shutdown-when-down = on +/// terminates that node's . Recovery is by exit-and-rejoin: the +/// service supervisor (Windows sc.exe failure … restart / docker restart: unless-stopped) +/// restarts the exited node and it re-forms / rejoins as a fresh incarnation. +/// +/// This watchdog closes the gap where the terminates but the .NET +/// host process keeps running with a dead actor system (idling forever, never restarted). It watches +/// ; if the system terminates outside a normal host +/// shutdown, it stops the application so the supervisor can restart the process. Mirrors the sister +/// ScadaBridge project's proven pattern. +/// +/// Registered after AddAkka so it starts after Akka's own hosted service has +/// built the system; the is resolved lazily in +/// (never at construction) so it can't race Akka startup. A graceful host stop is distinguished from +/// an unexpected SBR self-down via plus +/// , so normal shutdown never logs a false +/// alarm or double-triggers . +/// +public sealed class ActorSystemTerminationWatchdog : IHostedService +{ + private readonly Func _actorSystemAccessor; + private readonly IHostApplicationLifetime _lifetime; + private readonly ILogger _logger; + private volatile bool _stopRequested; + + /// Constructs the watchdog over a lazy accessor and the host lifetime. + /// Lazy accessor (resolved in , never at construction, so it can't race Akka startup). + /// Host application lifetime used to stop the process on an unexpected self-down. + /// Logger for the critical self-down diagnostic. + public ActorSystemTerminationWatchdog( + Func actorSystemAccessor, + IHostApplicationLifetime lifetime, + ILogger logger) + { + _actorSystemAccessor = actorSystemAccessor; + _lifetime = lifetime; + _logger = logger; + } + + /// Wires the continuation that exits the host on an unexpected self-down. + /// Unused; the watchdog only registers a continuation. + /// A completed task. + public Task StartAsync(CancellationToken cancellationToken) + { + var system = _actorSystemAccessor(); + system.WhenTerminated.ContinueWith( + _ => + { + // Expected shutdown: our StopAsync ran, or the host is already stopping. Stay quiet. + if (_stopRequested || _lifetime.ApplicationStopping.IsCancellationRequested) + return; + + _logger.LogCritical( + "ActorSystem terminated outside host shutdown (SBR self-down / " + + "run-coordinated-shutdown-when-down). Stopping the host so the service supervisor " + + "restarts this node as a fresh incarnation (2-node keep-oldest exit-and-rejoin recovery)."); + _lifetime.StopApplication(); + }, + TaskContinuationOptions.ExecuteSynchronously); + return Task.CompletedTask; + } + + /// Marks a graceful shutdown so the termination continuation stays silent and does not re-trigger stop. + /// Unused. + /// A completed task. + public Task StopAsync(CancellationToken cancellationToken) + { + _stopRequested = true; + return Task.CompletedTask; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index a1340806..49ff37b1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -296,6 +296,16 @@ builder.Services.AddAkka("otopcua", (ab, sp) => ab.WithOtOpcUaRuntimeActors(); }); +// Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own +// hosted service has built the ActorSystem; it resolves the system lazily (never at construction) so +// it can't race startup. On an unexpected SBR self-down it stops the host so the service supervisor +// (sc.exe failure / docker restart: unless-stopped) restarts this node — the 2-node keep-oldest +// exit-and-rejoin recovery path. +builder.Services.AddHostedService(sp => new ActorSystemTerminationWatchdog( + () => sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + if (hasAdmin) { // Auth + AdminUI surface only mounted on admin-role nodes. Driver-only nodes have no UI. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ActorSystemTerminationWatchdogTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ActorSystemTerminationWatchdogTests.cs new file mode 100644 index 00000000..f4fed6ce --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ActorSystemTerminationWatchdogTests.cs @@ -0,0 +1,92 @@ +using Akka.Actor; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Host; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// Unit coverage for the #459 down-if-alone recovery watchdog: an unexpected +/// termination (SBR self-down) must stop the host so the supervisor +/// restarts it, while a graceful host shutdown must stay silent and not re-trigger stop. +/// +[Trait("Category", "Unit")] +public sealed class ActorSystemTerminationWatchdogTests +{ + /// An unexpected ActorSystem termination stops the host so the supervisor restarts the node. + [Fact] + public async Task Unexpected_termination_stops_the_application() + { + var system = ActorSystem.Create("watchdog-unexpected"); + var lifetime = new FakeHostApplicationLifetime(); + var watchdog = new ActorSystemTerminationWatchdog( + () => system, lifetime, NullLogger.Instance); + + await watchdog.StartAsync(CancellationToken.None); + + // Simulate the SBR self-down: the ActorSystem terminates while the host is still running. + await system.Terminate(); + + await WaitForAsync(() => lifetime.StopApplicationCalls > 0, TimeSpan.FromSeconds(5)); + lifetime.StopApplicationCalls.ShouldBe(1, "an out-of-band ActorSystem termination must stop the host once"); + } + + /// A graceful host stop (StopAsync ran) leaves the termination continuation silent. + [Fact] + public async Task Graceful_stop_does_not_stop_the_application() + { + var system = ActorSystem.Create("watchdog-graceful"); + var lifetime = new FakeHostApplicationLifetime(); + var watchdog = new ActorSystemTerminationWatchdog( + () => system, lifetime, NullLogger.Instance); + + await watchdog.StartAsync(CancellationToken.None); + + // Host is shutting down normally: StopAsync runs, THEN the system terminates. + await watchdog.StopAsync(CancellationToken.None); + await system.Terminate(); + + // Give any continuation a chance to run, then assert it stayed silent. + await Task.Delay(500, TestContext.Current.CancellationToken); + lifetime.StopApplicationCalls.ShouldBe(0, "a graceful shutdown must not re-trigger StopApplication"); + } + + /// Termination while the host is already stopping (ApplicationStopping fired) stays silent. + [Fact] + public async Task Termination_while_already_stopping_stays_silent() + { + var system = ActorSystem.Create("watchdog-already-stopping"); + var lifetime = new FakeHostApplicationLifetime(); + var watchdog = new ActorSystemTerminationWatchdog( + () => system, lifetime, NullLogger.Instance); + + await watchdog.StartAsync(CancellationToken.None); + + lifetime.TriggerStopping(); // the host is already tearing down (ApplicationStopping cancelled) + await system.Terminate(); + + await Task.Delay(500, TestContext.Current.CancellationToken); + lifetime.StopApplicationCalls.ShouldBe(0, "termination during an in-progress host shutdown must stay silent"); + } + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(20); + } + + /// Minimal that records calls. + private sealed class FakeHostApplicationLifetime : IHostApplicationLifetime + { + private readonly CancellationTokenSource _stopping = new(); + public int StopApplicationCalls { get; private set; } + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => _stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => StopApplicationCalls++; + public void TriggerStopping() => _stopping.Cancel(); + } +}