feat(host): exit process on unexpected ActorSystem termination — completes the down-if-alone recovery loop

WhenTerminated watchdog calls IHostApplicationLifetime.StopApplication() when the ActorSystem dies outside StopAsync (SBR self-down + run-coordinated-shutdown-when-down), so the service supervisor restarts the node as a fresh incarnation. Optional ctor param keeps every existing construction site compiling. Plan's deploy/wonder-app-vd03/install.ps1 service-recovery-actions edit is skipped: that production deploy artifact is not tracked in this repo (Task 16 skipped its appsettings.Central.json for the same reason).
2026-07-08 16:56:24 -04:00
parent dea69842d5
commit 48e97fee01
3 changed files with 132 additions and 1 deletions
@@ -0,0 +1,84 @@
+using Akka.Actor;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
+using ZB.MOM.WW.ScadaBridge.Communication;
+using ZB.MOM.WW.ScadaBridge.Host.Actors;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
+
+/// <summary>
+/// PLAN-01 Task 20 (review 01 underdeveloped #2): with SBR down-if-alone +
+/// run-coordinated-shutdown-when-down live, a self-downed node terminates its
+/// ActorSystem out from under the still-running Host process. Unless the process
+/// then exits, the node idles forever — serving nothing, restarted by nobody.
+/// The recovery contract: unexpected ActorSystem termination ⇒ StopApplication()
+/// ⇒ the service supervisor restarts the process ⇒ it rejoins as a fresh
+/// incarnation. A termination that IS the host's own StopAsync must NOT trip it.
+/// </summary>
+public sealed class UnexpectedTerminationTests
+{
+    private static int FreePort()
+    {
+        var l = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
+        l.Start();
+        var p = ((System.Net.IPEndPoint)l.LocalEndpoint).Port;
+        l.Stop();
+        return p;
+    }
+
+    private static AkkaHostedService CreateAkkaHostedService(IHostApplicationLifetime lifetime)
+    {
+        var port = FreePort();
+        var self = $"akka.tcp://scadabridge@127.0.0.1:{port}";
+        return new AkkaHostedService(
+            new ServiceCollection().BuildServiceProvider(),
+            Options.Create(new NodeOptions { Role = "Site", NodeHostname = "127.0.0.1", RemotingPort = port, SiteId = "site-a" }),
+            Options.Create(new ClusterOptions
+            {
+                SeedNodes = new List<string> { self },
+                AllowSingleNodeCluster = true,
+                SplitBrainResolverStrategy = "keep-oldest",
+                StableAfter = TimeSpan.FromSeconds(3),
+                HeartbeatInterval = TimeSpan.FromMilliseconds(500),
+                FailureDetectionThreshold = TimeSpan.FromSeconds(2),
+                MinNrOfMembers = 1,
+                DownIfAlone = true,
+            }),
+            Options.Create(new CommunicationOptions()),
+            NullLogger<AkkaHostedService>.Instance,
+            lifetime);
+    }
+
+    [Fact]
+    public async Task ActorSystemTerminatedOutsideStopAsync_StopsHostApplication()
+    {
+        var lifetime = Substitute.For<IHostApplicationLifetime>();
+        var service = CreateAkkaHostedService(lifetime);
+        // Create the system (StartAsync registers actors; the system is created
+        // by GetOrCreateActorSystem, which wires the WhenTerminated watchdog).
+        service.GetOrCreateActorSystem();
+        var system = service.ActorSystem!;
+
+        await system.Terminate(); // simulated self-down, NOT StopAsync
+        await Task.Delay(500);
+
+        lifetime.Received(1).StopApplication();
+    }
+
+    [Fact]
+    public async Task StopAsync_DoesNotTriggerStopApplication()
+    {
+        var lifetime = Substitute.For<IHostApplicationLifetime>();
+        var service = CreateAkkaHostedService(lifetime);
+        service.GetOrCreateActorSystem();
+
+        await service.StopAsync(CancellationToken.None);
+        await Task.Delay(500);
+
+        lifetime.DidNotReceive().StopApplication();
+    }
+}