From 48e97fee018a2827f1b055bae202f3fca53590bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 16:56:24 -0400 Subject: [PATCH] =?UTF-8?q?feat(host):=20exit=20process=20on=20unexpected?= =?UTF-8?q?=20ActorSystem=20termination=20=E2=80=94=20completes=20the=20do?= =?UTF-8?q?wn-if-alone=20recovery=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WhenTerminated watchdog calls IHostApplicationLifetime.StopApplication() when the ActorSystem dies outside StopAsync (SBR self-down + run-coordinated-shutdown-when-down), so the service supervisor restarts the node as a fresh incarnation. Optional ctor param keeps every existing construction site compiling. Plan's deploy/wonder-app-vd03/install.ps1 service-recovery-actions edit is skipped: that production deploy artifact is not tracked in this repo (Task 16 skipped its appsettings.Central.json for the same reason). --- .../Component-ClusterInfrastructure.md | 11 +++ .../Actors/AkkaHostedService.cs | 38 ++++++++- .../UnexpectedTerminationTests.cs | 84 +++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 tests/ZB.MOM.WW.ScadaBridge.Host.Tests/UnexpectedTerminationTests.cs diff --git a/docs/requirements/Component-ClusterInfrastructure.md b/docs/requirements/Component-ClusterInfrastructure.md index 1535f935..c16f1a86 100644 --- a/docs/requirements/Component-ClusterInfrastructure.md +++ b/docs/requirements/Component-ClusterInfrastructure.md @@ -101,6 +101,17 @@ The system uses the Akka.NET **keep-oldest** split-brain resolver strategy: - **`down-if-alone = on`**: The keep-oldest resolver is configured with `down-if-alone` enabled. If the oldest node finds itself alone (no other reachable members), it downs itself rather than continuing as a single-node cluster. This prevents the oldest node from running in isolation during a network partition while the younger node also forms its own cluster. - **Why keep-oldest**: With only two nodes, quorum-based strategies (static-quorum, keep-majority) cannot distinguish "one node crashed" from "network partition" — both sides see fewer than quorum and both would down themselves, resulting in total cluster shutdown. Keep-oldest with `down-if-alone` provides safe singleton ownership — at most one node runs the cluster singleton at any time. +### Down-if-alone recovery + +When a node downs itself (via `down-if-alone`, or any other SBR decision), the resolver is configured with `run-coordinated-shutdown-when-down = on`, so the self-down runs `CoordinatedShutdown` and **terminates that node's `ActorSystem`**. The Host process must not keep running with a dead actor system — it would serve nothing and be restarted by nobody. The recovery contract is: + +1. Self-down ⇒ `CoordinatedShutdown` ⇒ `ActorSystem` termination. +2. The Host watches `ActorSystem.WhenTerminated`; a termination that is **not** the host's own `StopAsync` triggers `IHostApplicationLifetime.StopApplication()`, so **the process exits**. +3. The service supervisor restarts it — docker `restart: unless-stopped`, or Windows service recovery actions (`sc.exe failure … restart/…`). +4. The restarted process rejoins as a **fresh incarnation** (the keep-oldest resolver handles the rejoin cleanly; there is no stale membership to reconcile). + +The docker failover drill exercises this loop end-to-end (kill the active node, assert singleton migration + clean rejoin). + ## Single-Node Operation `akka.cluster.min-nr-of-members` must be set to **1**. After failover, only one node is running. If set to 2, the surviving node waits for a second member before allowing the Cluster Singleton (Site Runtime Deployment Manager) to start — blocking all data collection and script execution indefinitely. diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs index a51c8ef6..8a5a61d7 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs @@ -33,8 +33,18 @@ public class AkkaHostedService : IHostedService private readonly ClusterOptions _clusterOptions; private readonly CommunicationOptions _communicationOptions; private readonly ILogger _logger; + private readonly IHostApplicationLifetime? _appLifetime; private ActorSystem? _actorSystem; + /// + /// Flips to true the moment begins, so the + /// watchdog can tell an + /// intentional host shutdown apart from an SBR self-down that terminates the + /// system out from under a still-running process. volatile: written on + /// the shutdown thread, read on the Akka termination-callback thread. + /// + private volatile bool _stopRequested; + /// /// Guards the one-time creation of in /// so and a concurrent @@ -78,13 +88,17 @@ public class AkkaHostedService : IHostedService IOptions nodeOptions, IOptions clusterOptions, IOptions communicationOptions, - ILogger logger) + ILogger logger, + IHostApplicationLifetime? appLifetime = null) { _serviceProvider = serviceProvider; _nodeOptions = nodeOptions.Value; _clusterOptions = clusterOptions.Value; _communicationOptions = communicationOptions.Value; _logger = logger; + // Optional so existing construction sites (tests, DI variants) keep + // compiling. When absent the watchdog logs but can't stop the process. + _appLifetime = appLifetime; } /// @@ -185,6 +199,22 @@ public class AkkaHostedService : IHostedService _communicationOptions.TransportHeartbeatInterval.TotalSeconds, _communicationOptions.TransportFailureThreshold.TotalSeconds); + // Down-if-alone recovery watchdog: SBR's keep-oldest down-if-alone plus + // run-coordinated-shutdown-when-down means a self-downed node terminates + // its own ActorSystem. If that happens outside our StopAsync, the Host + // process must exit so the service supervisor (docker + // `restart: unless-stopped` / Windows service recovery) restarts it and + // it rejoins as a fresh incarnation. Without this the node idles forever + // with a dead actor system (review 01 underdeveloped #2). + system.WhenTerminated.ContinueWith(_ => + { + if (_stopRequested) return; + _logger.LogCritical( + "ActorSystem terminated outside host shutdown (SBR down / run-coordinated-shutdown-when-down). " + + "Stopping the host process so the service supervisor restarts this node as a fresh incarnation."); + _appLifetime?.StopApplication(); + }, TaskContinuationOptions.ExecuteSynchronously); + // Publish last so a concurrent reader never observes a half-constructed system. _actorSystem = system; return _actorSystem; @@ -308,6 +338,12 @@ akka {{ /// A task representing the asynchronous operation. public async Task StopAsync(CancellationToken cancellationToken) { + // Mark this as an intentional shutdown FIRST, so the WhenTerminated + // watchdog (wired in GetOrCreateActorSystem) does not mistake the + // CoordinatedShutdown below for an SBR self-down and re-trigger + // StopApplication on an already-stopping host. + _stopRequested = true; + // Dispose auxiliary subscribers (e.g. SiteAuditTelemetryStalledTracker) // BEFORE Akka shuts down so their EventStream unsubscribe calls run // while the system is still alive. Per-tracker Dispose is wrapped in diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/UnexpectedTerminationTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/UnexpectedTerminationTests.cs new file mode 100644 index 00000000..f4e6ee01 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/UnexpectedTerminationTests.cs @@ -0,0 +1,84 @@ +using Akka.Actor; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure; +using ZB.MOM.WW.ScadaBridge.Communication; +using ZB.MOM.WW.ScadaBridge.Host.Actors; + +namespace ZB.MOM.WW.ScadaBridge.Host.Tests; + +/// +/// PLAN-01 Task 20 (review 01 underdeveloped #2): with SBR down-if-alone + +/// run-coordinated-shutdown-when-down live, a self-downed node terminates its +/// ActorSystem out from under the still-running Host process. Unless the process +/// then exits, the node idles forever — serving nothing, restarted by nobody. +/// The recovery contract: unexpected ActorSystem termination ⇒ StopApplication() +/// ⇒ the service supervisor restarts the process ⇒ it rejoins as a fresh +/// incarnation. A termination that IS the host's own StopAsync must NOT trip it. +/// +public sealed class UnexpectedTerminationTests +{ + private static int FreePort() + { + var l = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0); + l.Start(); + var p = ((System.Net.IPEndPoint)l.LocalEndpoint).Port; + l.Stop(); + return p; + } + + private static AkkaHostedService CreateAkkaHostedService(IHostApplicationLifetime lifetime) + { + var port = FreePort(); + var self = $"akka.tcp://scadabridge@127.0.0.1:{port}"; + return new AkkaHostedService( + new ServiceCollection().BuildServiceProvider(), + Options.Create(new NodeOptions { Role = "Site", NodeHostname = "127.0.0.1", RemotingPort = port, SiteId = "site-a" }), + Options.Create(new ClusterOptions + { + SeedNodes = new List { self }, + AllowSingleNodeCluster = true, + SplitBrainResolverStrategy = "keep-oldest", + StableAfter = TimeSpan.FromSeconds(3), + HeartbeatInterval = TimeSpan.FromMilliseconds(500), + FailureDetectionThreshold = TimeSpan.FromSeconds(2), + MinNrOfMembers = 1, + DownIfAlone = true, + }), + Options.Create(new CommunicationOptions()), + NullLogger.Instance, + lifetime); + } + + [Fact] + public async Task ActorSystemTerminatedOutsideStopAsync_StopsHostApplication() + { + var lifetime = Substitute.For(); + var service = CreateAkkaHostedService(lifetime); + // Create the system (StartAsync registers actors; the system is created + // by GetOrCreateActorSystem, which wires the WhenTerminated watchdog). + service.GetOrCreateActorSystem(); + var system = service.ActorSystem!; + + await system.Terminate(); // simulated self-down, NOT StopAsync + await Task.Delay(500); + + lifetime.Received(1).StopApplication(); + } + + [Fact] + public async Task StopAsync_DoesNotTriggerStopApplication() + { + var lifetime = Substitute.For(); + var service = CreateAkkaHostedService(lifetime); + service.GetOrCreateActorSystem(); + + await service.StopAsync(CancellationToken.None); + await Task.Delay(500); + + lifetime.DidNotReceive().StopApplication(); + } +}