lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.Host/ActorSystemTerminationWatchdog.cs

using Akka.Actor;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace ZB.MOM.WW.OtOpcUa.Host;

/// <summary>
///     Down-if-alone recovery watchdog (arch-review #459). The 2-node <c>keep-oldest</c> split-brain
///     resolver downs the <b>lone survivor</b> when the OLDEST node crashes (the survivor is "the side
///     without the oldest" → <c>DownReachable</c>), and <c>run-coordinated-shutdown-when-down = on</c>
///     terminates that node's <see cref="ActorSystem"/>. Recovery is by <b>exit-and-rejoin</b>: the
///     service supervisor (Windows <c>sc.exe failure … restart</c> / docker <c>restart: unless-stopped</c>)
///     restarts the exited node and it re-forms / rejoins as a fresh incarnation.
///
///     <para>This watchdog closes the gap where the <see cref="ActorSystem"/> terminates but the .NET
///     host process keeps running with a dead actor system (idling forever, never restarted). It watches
///     <see cref="ActorSystem.WhenTerminated"/>; if the system terminates <b>outside</b> a normal host
///     shutdown, it stops the application so the supervisor can restart the process. Mirrors the sister
///     ScadaBridge project's proven pattern.</para>
///
///     <para>Registered <b>after</b> <c>AddAkka</c> so it starts after Akka's own hosted service has
///     built the system; the <see cref="ActorSystem"/> is resolved lazily in <see cref="StartAsync"/>
///     (never at construction) so it can't race Akka startup. A graceful host stop is distinguished from
///     an unexpected SBR self-down via <see cref="_stopRequested"/> plus
///     <see cref="IHostApplicationLifetime.ApplicationStopping"/>, so normal shutdown never logs a false
///     alarm or double-triggers <see cref="IHostApplicationLifetime.StopApplication"/>.</para>
/// </summary>
public sealed class ActorSystemTerminationWatchdog : IHostedService
{
    private readonly Func<ActorSystem> _actorSystemAccessor;
    private readonly IHostApplicationLifetime _lifetime;
    private readonly ILogger<ActorSystemTerminationWatchdog> _logger;
    private volatile bool _stopRequested;

    /// <summary>Constructs the watchdog over a lazy <see cref="ActorSystem"/> accessor and the host lifetime.</summary>
    /// <param name="actorSystemAccessor">Lazy accessor (resolved in <see cref="StartAsync"/>, never at construction, so it can't race Akka startup).</param>
    /// <param name="lifetime">Host application lifetime used to stop the process on an unexpected self-down.</param>
    /// <param name="logger">Logger for the critical self-down diagnostic.</param>
    public ActorSystemTerminationWatchdog(
        Func<ActorSystem> actorSystemAccessor,
        IHostApplicationLifetime lifetime,
        ILogger<ActorSystemTerminationWatchdog> logger)
    {
        _actorSystemAccessor = actorSystemAccessor;
        _lifetime = lifetime;
        _logger = logger;
    }

    /// <summary>Wires the <see cref="ActorSystem.WhenTerminated"/> continuation that exits the host on an unexpected self-down.</summary>
    /// <param name="cancellationToken">Unused; the watchdog only registers a continuation.</param>
    /// <returns>A completed task.</returns>
    public Task StartAsync(CancellationToken cancellationToken)
    {
        var system = _actorSystemAccessor();
        system.WhenTerminated.ContinueWith(
            _ =>
            {
                // Expected shutdown: our StopAsync ran, or the host is already stopping. Stay quiet.
                if (_stopRequested || _lifetime.ApplicationStopping.IsCancellationRequested)
                    return;

                _logger.LogCritical(
                    "ActorSystem terminated outside host shutdown (SBR self-down / "
                    + "run-coordinated-shutdown-when-down). Stopping the host so the service supervisor "
                    + "restarts this node as a fresh incarnation (2-node keep-oldest exit-and-rejoin recovery).");
                _lifetime.StopApplication();
            },
            TaskContinuationOptions.ExecuteSynchronously);
        return Task.CompletedTask;
    }

    /// <summary>Marks a graceful shutdown so the termination continuation stays silent and does not re-trigger stop.</summary>
    /// <param name="cancellationToken">Unused.</param>
    /// <returns>A completed task.</returns>
    public Task StopAsync(CancellationToken cancellationToken)
    {
        _stopRequested = true;
        return Task.CompletedTask;
    }
}