ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/UnexpectedTerminationTests.cs

using Akka.Actor;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Host.Actors;

namespace ZB.MOM.WW.ScadaBridge.Host.Tests;

/// <summary>
/// With SBR down-if-alone +
/// run-coordinated-shutdown-when-down live, a self-downed node terminates its
/// ActorSystem out from under the still-running Host process. Unless the process
/// then exits, the node idles forever — serving nothing, restarted by nobody.
/// The recovery contract: unexpected ActorSystem termination ⇒ StopApplication()
/// ⇒ the service supervisor restarts the process ⇒ it rejoins as a fresh
/// incarnation. A termination that IS the host's own StopAsync must NOT trip it.
/// </summary>
public sealed class UnexpectedTerminationTests
{
    private static int FreePort()
    {
        var l = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
        l.Start();
        var p = ((System.Net.IPEndPoint)l.LocalEndpoint).Port;
        l.Stop();
        return p;
    }

    private static AkkaHostedService CreateAkkaHostedService(IHostApplicationLifetime lifetime)
    {
        var port = FreePort();
        var self = $"akka.tcp://scadabridge@127.0.0.1:{port}";
        return new AkkaHostedService(
            new ServiceCollection().BuildServiceProvider(),
            Options.Create(new NodeOptions { Role = "Site", NodeHostname = "127.0.0.1", RemotingPort = port, SiteId = "site-a" }),
            Options.Create(new ClusterOptions
            {
                SeedNodes = new List<string> { self },
                AllowSingleNodeCluster = true,
                SplitBrainResolverStrategy = "keep-oldest",
                StableAfter = TimeSpan.FromSeconds(3),
                HeartbeatInterval = TimeSpan.FromMilliseconds(500),
                FailureDetectionThreshold = TimeSpan.FromSeconds(2),
                MinNrOfMembers = 1,
                DownIfAlone = true,
            }),
            Options.Create(new CommunicationOptions()),
            NullLogger<AkkaHostedService>.Instance,
            lifetime);
    }

    /// <summary>Verifies that the host application is stopped when the actor system terminates outside of a stop request.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ActorSystemTerminatedOutsideStopAsync_StopsHostApplication()
    {
        var lifetime = Substitute.For<IHostApplicationLifetime>();
        var service = CreateAkkaHostedService(lifetime);
        // Create the system (StartAsync registers actors; the system is created
        // by GetOrCreateActorSystem, which wires the WhenTerminated watchdog).
        service.GetOrCreateActorSystem();
        var system = service.ActorSystem!;

        await system.Terminate(); // simulated self-down, NOT StopAsync
        await Task.Delay(500);

        lifetime.Received(1).StopApplication();
        // ScadaBridge#132: Program turns this flag into the non-zero SelfDownExitCode.
        Assert.True(service.TerminatedOutsideHostShutdown);
    }

    /// <summary>Verifies that a normal stop does not trigger an application stop.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task StopAsync_DoesNotTriggerStopApplication()
    {
        var lifetime = Substitute.For<IHostApplicationLifetime>();
        var service = CreateAkkaHostedService(lifetime);
        service.GetOrCreateActorSystem();

        await service.StopAsync(CancellationToken.None);
        await Task.Delay(500);

        lifetime.DidNotReceive().StopApplication();
        Assert.False(service.TerminatedOutsideHostShutdown);
    }

    /// <summary>
    /// ScadaBridge#140: the lifecycle hook that runs before Kestrel drains raises ApplicationStopping
    /// FIRST (so the site's stream cancellation still precedes actor teardown), then completes the
    /// CoordinatedShutdown — and a planned stop is never mistaken for a self-down (no exit code 75).
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task StoppingAsync_RaisesStopping_ThenShutsDown_AndIsNotASelfDown()
    {
        var lifetime = Substitute.For<IHostApplicationLifetime>();
        var service = CreateAkkaHostedService(lifetime);
        var system = service.GetOrCreateActorSystem();
        bool? terminatedWhenStoppingRaised = null;
        lifetime.When(l => l.StopApplication()).Do(_ => terminatedWhenStoppingRaised = system.WhenTerminated.IsCompleted);

        await service.StoppingAsync(CancellationToken.None);

        Assert.True(system.WhenTerminated.IsCompleted, "StoppingAsync must complete the CoordinatedShutdown");
        Assert.Equal(false, terminatedWhenStoppingRaised);
        Assert.Equal(CoordinatedShutdown.ClrExitReason.Instance, CoordinatedShutdown.Get(system).ShutdownReason);

        // The host's later StopAsync is the same, completed shutdown — not a second one.
        await service.StopAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
        await Task.Delay(500);

        lifetime.Received(1).StopApplication();
        Assert.False(service.TerminatedOutsideHostShutdown);
    }

    /// <summary>A host whose Akka service never created its system: StoppingAsync must not create one.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task StoppingAsync_WithoutAnActorSystem_DoesNotCreateOne()
    {
        var lifetime = Substitute.For<IHostApplicationLifetime>();
        var service = CreateAkkaHostedService(lifetime);

        await service.StoppingAsync(CancellationToken.None);

        Assert.Null(service.ActorSystem);
        lifetime.Received(1).StopApplication();
    }

    /// <summary>ScadaBridge#140: the host only calls StoppingAsync on a service that declares the lifecycle interface.</summary>
    [Fact]
    public void AkkaHostedService_IsAHostedLifecycleService()
        => Assert.True(typeof(IHostedLifecycleService).IsAssignableFrom(typeof(AkkaHostedService)));

    /// <summary>
    /// ScadaBridge#132: the self-down exit code is non-zero (Windows service recovery with
    /// failureflag 1 restarts only on a non-zero exit) and is only pushed into a Windows service
    /// lifetime — a console / docker lifetime is left alone.
    /// </summary>
    [Fact]
    public void SelfDownExitCode_IsNonZero_AndOnlyAppliedToAServiceLifetime()
    {
        Assert.NotEqual(0, AkkaHostedService.SelfDownExitCode);
        Assert.False(AkkaHostedService.MarkServiceSelfDownExit(null));
        Assert.False(AkkaHostedService.MarkServiceSelfDownExit(Substitute.For<IHostLifetime>()));
    }
}