feat(host): exit process on unexpected ActorSystem termination — completes the down-if-alone recovery loop
WhenTerminated watchdog calls IHostApplicationLifetime.StopApplication() when the ActorSystem dies outside StopAsync (SBR self-down + run-coordinated-shutdown-when-down), so the service supervisor restarts the node as a fresh incarnation. Optional ctor param keeps every existing construction site compiling. Plan's deploy/wonder-app-vd03/install.ps1 service-recovery-actions edit is skipped: that production deploy artifact is not tracked in this repo (Task 16 skipped its appsettings.Central.json for the same reason).
This commit is contained in:
@@ -33,8 +33,18 @@ public class AkkaHostedService : IHostedService
|
||||
private readonly ClusterOptions _clusterOptions;
|
||||
private readonly CommunicationOptions _communicationOptions;
|
||||
private readonly ILogger<AkkaHostedService> _logger;
|
||||
private readonly IHostApplicationLifetime? _appLifetime;
|
||||
private ActorSystem? _actorSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Flips to <c>true</c> the moment <see cref="StopAsync"/> begins, so the
|
||||
/// <see cref="ActorSystem.WhenTerminated"/> watchdog can tell an
|
||||
/// intentional host shutdown apart from an SBR self-down that terminates the
|
||||
/// system out from under a still-running process. <c>volatile</c>: written on
|
||||
/// the shutdown thread, read on the Akka termination-callback thread.
|
||||
/// </summary>
|
||||
private volatile bool _stopRequested;
|
||||
|
||||
/// <summary>
|
||||
/// Guards the one-time creation of <see cref="_actorSystem"/> in
|
||||
/// <see cref="GetOrCreateActorSystem"/> so <see cref="StartAsync"/> and a concurrent
|
||||
@@ -78,13 +88,17 @@ public class AkkaHostedService : IHostedService
|
||||
IOptions<NodeOptions> nodeOptions,
|
||||
IOptions<ClusterOptions> clusterOptions,
|
||||
IOptions<CommunicationOptions> communicationOptions,
|
||||
ILogger<AkkaHostedService> logger)
|
||||
ILogger<AkkaHostedService> logger,
|
||||
IHostApplicationLifetime? appLifetime = null)
|
||||
{
|
||||
_serviceProvider = serviceProvider;
|
||||
_nodeOptions = nodeOptions.Value;
|
||||
_clusterOptions = clusterOptions.Value;
|
||||
_communicationOptions = communicationOptions.Value;
|
||||
_logger = logger;
|
||||
// Optional so existing construction sites (tests, DI variants) keep
|
||||
// compiling. When absent the watchdog logs but can't stop the process.
|
||||
_appLifetime = appLifetime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -185,6 +199,22 @@ public class AkkaHostedService : IHostedService
|
||||
_communicationOptions.TransportHeartbeatInterval.TotalSeconds,
|
||||
_communicationOptions.TransportFailureThreshold.TotalSeconds);
|
||||
|
||||
// Down-if-alone recovery watchdog: SBR's keep-oldest down-if-alone plus
|
||||
// run-coordinated-shutdown-when-down means a self-downed node terminates
|
||||
// its own ActorSystem. If that happens outside our StopAsync, the Host
|
||||
// process must exit so the service supervisor (docker
|
||||
// `restart: unless-stopped` / Windows service recovery) restarts it and
|
||||
// it rejoins as a fresh incarnation. Without this the node idles forever
|
||||
// with a dead actor system (review 01 underdeveloped #2).
|
||||
system.WhenTerminated.ContinueWith(_ =>
|
||||
{
|
||||
if (_stopRequested) return;
|
||||
_logger.LogCritical(
|
||||
"ActorSystem terminated outside host shutdown (SBR down / run-coordinated-shutdown-when-down). "
|
||||
+ "Stopping the host process so the service supervisor restarts this node as a fresh incarnation.");
|
||||
_appLifetime?.StopApplication();
|
||||
}, TaskContinuationOptions.ExecuteSynchronously);
|
||||
|
||||
// Publish last so a concurrent reader never observes a half-constructed system.
|
||||
_actorSystem = system;
|
||||
return _actorSystem;
|
||||
@@ -308,6 +338,12 @@ akka {{
|
||||
/// <returns>A task representing the asynchronous operation.</returns>
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
// Mark this as an intentional shutdown FIRST, so the WhenTerminated
|
||||
// watchdog (wired in GetOrCreateActorSystem) does not mistake the
|
||||
// CoordinatedShutdown below for an SBR self-down and re-trigger
|
||||
// StopApplication on an already-stopping host.
|
||||
_stopRequested = true;
|
||||
|
||||
// Dispose auxiliary subscribers (e.g. SiteAuditTelemetryStalledTracker)
|
||||
// BEFORE Akka shuts down so their EventStream unsubscribe calls run
|
||||
// while the system is still alive. Per-tracker Dispose is wrapped in
|
||||
|
||||
Reference in New Issue
Block a user