feat(host): exit process on unexpected ActorSystem termination — completes the down-if-alone recovery loop

WhenTerminated watchdog calls IHostApplicationLifetime.StopApplication() when
the ActorSystem dies outside StopAsync (SBR self-down + run-coordinated-shutdown-when-down),
so the service supervisor restarts the node as a fresh incarnation. Optional
ctor param keeps every existing construction site compiling.

Plan's deploy/wonder-app-vd03/install.ps1 service-recovery-actions edit is
skipped: that production deploy artifact is not tracked in this repo (Task 16
skipped its appsettings.Central.json for the same reason).
This commit is contained in:
Joseph Doherty
2026-07-08 16:56:24 -04:00
parent dea69842d5
commit 48e97fee01
3 changed files with 132 additions and 1 deletions
@@ -33,8 +33,18 @@ public class AkkaHostedService : IHostedService
private readonly ClusterOptions _clusterOptions;
private readonly CommunicationOptions _communicationOptions;
private readonly ILogger<AkkaHostedService> _logger;
private readonly IHostApplicationLifetime? _appLifetime;
private ActorSystem? _actorSystem;
/// <summary>
/// Flips to <c>true</c> the moment <see cref="StopAsync"/> begins, so the
/// <see cref="ActorSystem.WhenTerminated"/> watchdog can tell an
/// intentional host shutdown apart from an SBR self-down that terminates the
/// system out from under a still-running process. <c>volatile</c>: written on
/// the shutdown thread, read on the Akka termination-callback thread.
/// </summary>
private volatile bool _stopRequested;
/// <summary>
/// Guards the one-time creation of <see cref="_actorSystem"/> in
/// <see cref="GetOrCreateActorSystem"/> so <see cref="StartAsync"/> and a concurrent
@@ -78,13 +88,17 @@ public class AkkaHostedService : IHostedService
IOptions<NodeOptions> nodeOptions,
IOptions<ClusterOptions> clusterOptions,
IOptions<CommunicationOptions> communicationOptions,
ILogger<AkkaHostedService> logger)
ILogger<AkkaHostedService> logger,
IHostApplicationLifetime? appLifetime = null)
{
_serviceProvider = serviceProvider;
_nodeOptions = nodeOptions.Value;
_clusterOptions = clusterOptions.Value;
_communicationOptions = communicationOptions.Value;
_logger = logger;
// Optional so existing construction sites (tests, DI variants) keep
// compiling. When absent the watchdog logs but can't stop the process.
_appLifetime = appLifetime;
}
/// <summary>
@@ -185,6 +199,22 @@ public class AkkaHostedService : IHostedService
_communicationOptions.TransportHeartbeatInterval.TotalSeconds,
_communicationOptions.TransportFailureThreshold.TotalSeconds);
// Down-if-alone recovery watchdog: SBR's keep-oldest down-if-alone plus
// run-coordinated-shutdown-when-down means a self-downed node terminates
// its own ActorSystem. If that happens outside our StopAsync, the Host
// process must exit so the service supervisor (docker
// `restart: unless-stopped` / Windows service recovery) restarts it and
// it rejoins as a fresh incarnation. Without this the node idles forever
// with a dead actor system (review 01 underdeveloped #2).
system.WhenTerminated.ContinueWith(_ =>
{
if (_stopRequested) return;
_logger.LogCritical(
"ActorSystem terminated outside host shutdown (SBR down / run-coordinated-shutdown-when-down). "
+ "Stopping the host process so the service supervisor restarts this node as a fresh incarnation.");
_appLifetime?.StopApplication();
}, TaskContinuationOptions.ExecuteSynchronously);
// Publish last so a concurrent reader never observes a half-constructed system.
_actorSystem = system;
return _actorSystem;
@@ -308,6 +338,12 @@ akka {{
/// <returns>A task representing the asynchronous operation.</returns>
public async Task StopAsync(CancellationToken cancellationToken)
{
// Mark this as an intentional shutdown FIRST, so the WhenTerminated
// watchdog (wired in GetOrCreateActorSystem) does not mistake the
// CoordinatedShutdown below for an SBR self-down and re-trigger
// StopApplication on an already-stopping host.
_stopRequested = true;
// Dispose auxiliary subscribers (e.g. SiteAuditTelemetryStalledTracker)
// BEFORE Akka shuts down so their EventStream unsubscribe calls run
// while the system is still alive. Per-tracker Dispose is wrapped in