ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ShutOutRestartScenarioTests.cs

using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Akka.Remote;
using Akka.Remote.Transport;
using Xunit;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Host;
using ZB.MOM.WW.ScadaBridge.Host.Actors;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;

/// <summary>
/// ScadaBridge#132, on real in-process two-node clusters built from the production
/// <c>BuildHocon</c> (same harness as <see cref="LeaderAutoDownScenarioTests"/>, with Akka's
/// throttler transport to cut and restore the network without killing either node):
/// <list type="bullet">
/// <item>the FROZEN-node shape — only one side downed the other before the network came back —
/// ends with the shut-out node restarting (its <c>ActorSystem</c> terminates) and the other
/// staying Up;</item>
/// <item>a partition held past the window and then HEALED — each side downed the other — ends
/// with exactly one restart, the higher address; a fresh incarnation on that address rejoins the
/// survivor as a 2-member cluster, and the survivor is still Up after its grace;</item>
/// <item>with the switch OFF, the healed partition keeps both sides up (the #116 gate-4 contract
/// before #132) — the positive control that the restart comes from the feature.</item>
/// </list>
/// "Restart" in-process is the termination of the node's <c>ActorSystem</c>; the Host turns that
/// into a process exit (covered by <c>UnexpectedTerminationTests</c>).
/// </summary>
public sealed class ShutOutRestartScenarioTests : IAsyncLifetime
{
    private static readonly TimeSpan Heartbeat = TimeSpan.FromMilliseconds(500);
    private static readonly TimeSpan Threshold = TimeSpan.FromSeconds(2);

    private readonly List<ActorSystem> _systems = new();

    private ActorSystem StartNode(
        int selfPort, int peerPort, TimeSpan stableAfter, bool restartWhenShutOut = true, TimeSpan? grace = null)
    {
        const string scheme = "akka.trttl.tcp";
        var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = selfPort };
        var clusterOptions = new ClusterOptions
        {
            SeedNodes = new List<string>
            {
                $"{scheme}://scadabridge@127.0.0.1:{selfPort}",
                $"{scheme}://scadabridge@127.0.0.1:{peerPort}",
            },
            SplitBrainResolverStrategy = "leader-auto-down",
            StableAfter = stableAfter,
            HeartbeatInterval = Heartbeat,
            FailureDetectionThreshold = Threshold,
            MinNrOfMembers = 1,
            LeaderAutoDown = new LeaderAutoDownOptions
            {
                RestartWhenShutOut = restartWhenShutOut,
                ShutOutGrace = grace ?? TimeSpan.FromSeconds(20),
            },
        };
        var hocon = AkkaHostedService.BuildHocon(
            nodeOptions, clusterOptions, new[] { "Central" },
            TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(60));
        var overrides = "akka.coordinated-shutdown.run-by-actor-system-terminate = off\n"
            + "akka.remote.dot-netty.tcp.applied-adapters = [trttl]\n"
            + "akka.remote.retry-gate-closed-for = 500ms\n"
            // Fast probes so a healed partition is noticed within the test's patience.
            + "scadabridge.cluster.leader-auto-down.shut-out-probe-interval = 1s\n"
            + "scadabridge.cluster.leader-auto-down.shut-out-probe-max-interval = 2s\n";
        var config = ConfigurationFactory.ParseString(overrides)
            .WithFallback(ConfigurationFactory.ParseString(hocon));
        var system = ActorSystem.Create("scadabridge", config);
        _systems.Add(system);
        return system;
    }

    private static Task<bool> Throttle(ActorSystem from, ActorSystem to, ThrottleMode mode) =>
        ((RemoteActorRefProvider)((ExtendedActorSystem)from).Provider).Transport.ManagementCommand(
            new SetThrottle(Akka.Cluster.Cluster.Get(to).SelfAddress, ThrottleTransportAdapter.Direction.Both, mode));

    private async Task<(ActorSystem Lower, ActorSystem Higher, int LowerPort, int HigherPort)> StartPair(
        TimeSpan stableLower, TimeSpan stableHigher, bool restartWhenShutOut = true)
    {
        var p1 = TwoNodeClusterFixture.GetFreeTcpPort();
        var p2 = TwoNodeClusterFixture.GetFreeTcpPort();
        var (lowerPort, higherPort) = p1 < p2 ? (p1, p2) : (p2, p1);
        var lower = StartNode(lowerPort, higherPort, stableLower, restartWhenShutOut);
        await TwoNodeClusterFixture.WaitForMembersUp(lower, 1, TimeSpan.FromSeconds(30));
        var higher = StartNode(higherPort, lowerPort, stableHigher, restartWhenShutOut);
        await TwoNodeClusterFixture.WaitForMembersUp(lower, 2, TimeSpan.FromSeconds(30));
        await TwoNodeClusterFixture.WaitForMembersUp(higher, 2, TimeSpan.FromSeconds(30));
        return (lower, higher, lowerPort, higherPort);
    }

    private static async Task<bool> Terminates(ActorSystem system, TimeSpan within)
    {
        var done = await Task.WhenAny(system.WhenTerminated, Task.Delay(within));
        return done == system.WhenTerminated;
    }

    /// <summary>
    /// Frozen-node shape: only the LOWER node downs the higher one (the higher one's window is far
    /// longer than the cut), then the network returns. The higher node — still counting the lower as
    /// a member — is refused, restarts without a tiebreak; the lower node stays Up. Then the same with
    /// roles swapped, so the single-side rule is proven independent of the address order.
    /// </summary>
    /// <param name="shutOutIsHigher">Whether the shut-out (frozen) node has the higher address.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public async Task SingleSideShutOut_ShutOutNodeRestarts_OtherStaysUp(bool shutOutIsHigher)
    {
        var quick = TimeSpan.FromSeconds(2);
        var never = TimeSpan.FromMinutes(5);
        var (lower, higher, _, _) = shutOutIsHigher
            ? await StartPair(quick, never)
            : await StartPair(never, quick);
        var (survivor, shutOut) = shutOutIsHigher ? (lower, higher) : (higher, lower);

        Assert.True(await Throttle(lower, higher, Blackhole.Instance));
        await TwoNodeClusterFixture.WaitForMemberRemoved(survivor, Akka.Cluster.Cluster.Get(shutOut).SelfAddress, TimeSpan.FromSeconds(30));
        Assert.Contains(Akka.Cluster.Cluster.Get(shutOut).State.Members,
            m => m.Address == Akka.Cluster.Cluster.Get(survivor).SelfAddress); // it still counts the survivor
        Assert.True(await Throttle(lower, higher, Unthrottled.Instance));

        Assert.True(await Terminates(shutOut, TimeSpan.FromSeconds(30)),
            "the shut-out node kept running as a second active node");
        Assert.False(await Terminates(survivor, TimeSpan.FromSeconds(5)), "the survivor restarted too");
        Assert.Equal(MemberStatus.Up, Akka.Cluster.Cluster.Get(survivor).SelfMember.Status);
    }

    /// <summary>
    /// A partition past the window, then healed: both sides downed and removed each other, so both
    /// are refused — exactly ONE restarts, the higher address. A fresh incarnation on the higher
    /// port (what the service supervisor starts) rejoins the survivor, and the survivor is still Up
    /// after its grace has run out.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task HealedPartition_BothShutOut_OnlyTheHigherAddressRestarts_AndRejoins()
    {
        var stable = TimeSpan.FromSeconds(2);
        var (lower, higher, lowerPort, higherPort) = await StartPair(stable, stable);
        var lowerAddress = Akka.Cluster.Cluster.Get(lower).SelfAddress;
        var higherAddress = Akka.Cluster.Cluster.Get(higher).SelfAddress;

        Assert.True(await Throttle(lower, higher, Blackhole.Instance));
        await TwoNodeClusterFixture.WaitForMemberRemoved(lower, higherAddress, TimeSpan.FromSeconds(30));
        await TwoNodeClusterFixture.WaitForMemberRemoved(higher, lowerAddress, TimeSpan.FromSeconds(30));
        Assert.False(lower.WhenTerminated.IsCompleted || higher.WhenTerminated.IsCompleted,
            "a node restarted while the partition was still in place");
        Assert.True(await Throttle(lower, higher, Unthrottled.Instance));

        Assert.True(await Terminates(higher, TimeSpan.FromSeconds(30)),
            "the higher-address node did not restart after the partition healed");
        Assert.False(lower.WhenTerminated.IsCompleted, "the lower-address node restarted as well");

        // The supervisor's restart: a fresh incarnation on the same address rejoins the survivor.
        var reborn = StartNode(higherPort, lowerPort, stable);
        await TwoNodeClusterFixture.WaitForMembersUp(lower, 2, TimeSpan.FromSeconds(60));
        await TwoNodeClusterFixture.WaitForMembersUp(reborn, 2, TimeSpan.FromSeconds(60));

        // Past the survivor's 20s grace: it saw the peer rejoin, so it stays.
        Assert.False(await Terminates(lower, TimeSpan.FromSeconds(25)), "the survivor restarted after its peer rejoined");
        Assert.Equal(2, Akka.Cluster.Cluster.Get(lower).State.Members.Count(m => m.Status == MemberStatus.Up));
        Assert.Empty(Akka.Cluster.Cluster.Get(lower).State.Unreachable);
    }

    /// <summary>
    /// Switch OFF: the same healed partition keeps BOTH sides running (the pre-#132 dual-active)
    /// — the positive control that the restarts above come from the feature.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task HealedPartition_SwitchOff_BothStayUp()
    {
        var stable = TimeSpan.FromSeconds(2);
        var (lower, higher, _, _) = await StartPair(stable, stable, restartWhenShutOut: false);
        var lowerAddress = Akka.Cluster.Cluster.Get(lower).SelfAddress;
        var higherAddress = Akka.Cluster.Cluster.Get(higher).SelfAddress;

        Assert.True(await Throttle(lower, higher, Blackhole.Instance));
        await TwoNodeClusterFixture.WaitForMemberRemoved(lower, higherAddress, TimeSpan.FromSeconds(30));
        await TwoNodeClusterFixture.WaitForMemberRemoved(higher, lowerAddress, TimeSpan.FromSeconds(30));
        Assert.True(await Throttle(lower, higher, Unthrottled.Instance));

        Assert.False(await Terminates(higher, TimeSpan.FromSeconds(15)), "a node restarted with the switch off");
        Assert.False(lower.WhenTerminated.IsCompleted, "a node restarted with the switch off");
    }

    /// <summary>Prepares the fixture before the tests run.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public Task InitializeAsync() => Task.CompletedTask;

    /// <summary>Releases the resources held by this fixture asynchronously.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async Task DisposeAsync()
    {
        foreach (var s in _systems)
        {
            try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
        }
    }
}