ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/LeaderAutoDownTests.cs

using System.Collections.Immutable;
using System.Reflection;
using Akka.Actor;
using Akka.Cluster;
using Akka.TestKit.Xunit2;
using Akka.Util;
using ZB.MOM.WW.ScadaBridge.Host.Actors;

namespace ZB.MOM.WW.ScadaBridge.Host.Tests;

/// <summary>
/// ScadaBridge#106: the decision logic of <see cref="LeaderAutoDownBase"/> must match Akka.NET
/// 1.5.62 <c>AutoDownBase</c> exactly. These cases are the ones Akka's own <c>AutoDownSpec</c>
/// pins — cluster events are fed straight to the actor and the <c>Down</c> seam reports to a
/// probe, so the leader-only and reachability semantics are proven without a running cluster.
/// The two-node real-cluster behaviour is proven separately in IntegrationTests
/// (<c>LeaderAutoDownFailoverTests</c>).
/// </summary>
public class LeaderAutoDownTests : TestKit
{
    /// <summary>INFO so the decision-log assertions can see the Info-level decisions.</summary>
    public LeaderAutoDownTests()
        : base("akka.loglevel = INFO")
    {
    }

    private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
    private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
    private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);

    private sealed record DownCalled(Address Address);

    /// <summary>Test double: <c>Down</c> reports to a probe instead of calling the cluster.</summary>
    private sealed class TestLeaderAutoDown : LeaderAutoDownBase
    {
        private readonly IActorRef _probe;

        public TestLeaderAutoDown(Address selfAddress, TimeSpan unreachableAfter, IActorRef probe)
            : base(unreachableAfter)
        {
            SelfAddress = selfAddress;
            _probe = probe;
        }

        public override Address SelfAddress { get; }

        public override IScheduler Scheduler => Context.System.Scheduler;

        public override void Down(Address node)
        {
            // Report EVERY call, leader or not: a base that wrongly downs while not leader must
            // show up as an unexpected message, not be swallowed by a throw-and-restart.
            _probe.Tell(new DownCalled(node));
            if (!IsLeader)
            {
                _probe.Tell("down-called-while-not-leader");
            }
        }
    }

    /// <summary>Akka's <c>Member</c> constructors are internal; tests build members via the internal factory.</summary>
    private static Member CreateMember(Address address, MemberStatus status)
    {
        var create = typeof(Member).GetMethod(
            "Create",
            BindingFlags.Static | BindingFlags.NonPublic,
            new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet<string>), typeof(AppVersion) })
            ?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed.");
        return (Member)create.Invoke(null, new object[]
        {
            new UniqueAddress(address, 1), 1, status, ImmutableHashSet<string>.Empty, AppVersion.Zero,
        })!;
    }

    private IActorRef AutoDownActor(TimeSpan unreachableAfter) =>
        Sys.ActorOf(Props.Create(() => new TestLeaderAutoDown(MemberA, unreachableAfter, TestActor)));

    /// <summary>A non-leader never downs an unreachable member.</summary>
    [Fact]
    public void NotLeader_DoesNotDownUnreachable()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
        ExpectNoMsg(TimeSpan.FromSeconds(1));
    }

    /// <summary>The leader downs an unreachable member immediately when the window is zero.</summary>
    [Fact]
    public void Leader_DownsUnreachable_WhenWindowIsZero()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
        ExpectMsg(new DownCalled(MemberB));
    }

    /// <summary>The leader downs an unreachable member only after the full window.</summary>
    [Fact]
    public void Leader_DownsUnreachable_AfterWindow()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(2));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
        ExpectNoMsg(TimeSpan.FromSeconds(1));
        ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
    }

    /// <summary>The initial snapshot drives leadership and schedules members already unreachable.</summary>
    [Fact]
    public void CurrentClusterState_Leader_DownsAlreadyUnreachable()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.CurrentClusterState(
            ImmutableSortedSet<Member>.Empty,
            ImmutableHashSet.Create(b),
            ImmutableHashSet<Address>.Empty,
            MemberA,
            ImmutableDictionary<string, Address>.Empty));
        ExpectMsg(new DownCalled(MemberB));
    }

    /// <summary>A pending member is downed as soon as this node becomes leader.</summary>
    [Fact]
    public void BecomingLeader_DownsPendingUnreachable()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
        ExpectNoMsg(TimeSpan.FromMilliseconds(500));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        ExpectMsg(new DownCalled(MemberC));
    }

    /// <summary>A member that timed out while not leader is downed on becoming leader.</summary>
    [Fact]
    public void BecomingLeader_AfterWindow_DownsPendingUnreachable()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(1));
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
        ExpectNoMsg(TimeSpan.FromSeconds(2));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        ExpectMsg(new DownCalled(MemberC));
    }

    /// <summary>Losing leadership inside the window means no Down when the timer fires.</summary>
    [Fact]
    public void LosingLeadership_InsideWindow_DoesNotDown()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(2));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        ExpectNoMsg(TimeSpan.FromSeconds(3));
    }

    /// <summary>A member that becomes reachable again inside the window is not downed.</summary>
    [Fact]
    public void ReachableAgain_InsideWindow_DoesNotDown()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(2));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(b));
        a.Tell(new ClusterEvent.ReachableMember(b));
        ExpectNoMsg(TimeSpan.FromSeconds(3));
    }

    /// <summary>A pending member that becomes reachable again is not downed on becoming leader.</summary>
    [Fact]
    public void ReachableAgain_WhilePending_IsForgotten()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        var c = CreateMember(MemberC, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(c));
        a.Tell(new ClusterEvent.ReachableMember(c));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        ExpectNoMsg(TimeSpan.FromSeconds(1));
    }

    /// <summary>A member removed inside the window is not downed.</summary>
    [Fact]
    public void RemovedInsideWindow_DoesNotDown()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(2));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(b));
        a.Tell(new ClusterEvent.MemberRemoved(b.Copy(MemberStatus.Removed), MemberStatus.Up));
        ExpectNoMsg(TimeSpan.FromSeconds(3));
    }

    /// <summary>Members already Down or Exiting are skipped (Akka's ConvergenceSkipUnreachableWithMemberStatus).</summary>
    /// <param name="status">The member status.</param>
    [Theory]
    [InlineData(MemberStatus.Down)]
    [InlineData(MemberStatus.Exiting)]
    public void AlreadyDownOrExiting_IsNotDownedAgain(MemberStatus status)
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, status)));
        ExpectNoMsg(TimeSpan.FromSeconds(1));
    }

    /// <summary>A repeated UnreachableMember for an already-scheduled member does not schedule a second Down.</summary>
    [Fact]
    public void RepeatedUnreachable_DownsOnce()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(1));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(b));
        a.Tell(new ClusterEvent.UnreachableMember(b));
        ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
        ExpectNoMsg(TimeSpan.FromSeconds(2));
    }

    /// <summary>
    /// A flapping member — unreachable, reachable again inside the window, repeatedly, for far
    /// longer than one window in total — is never downed.
    /// </summary>
    [Fact]
    public void FlappingMember_ReachableAgainInsideEachWindow_IsNeverDowned()
    {
        var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        for (var i = 0; i < 5; i++)
        {
            a.Tell(new ClusterEvent.UnreachableMember(b));
            ExpectNoMsg(TimeSpan.FromMilliseconds(700));
            a.Tell(new ClusterEvent.ReachableMember(b));
            ExpectNoMsg(TimeSpan.FromMilliseconds(100));
        }

        ExpectNoMsg(TimeSpan.FromSeconds(2));
    }

    /// <summary>
    /// When a flapping member finally stays unreachable, the window restarts from its LAST
    /// unreachable — an earlier flap's timer must not fire — and it is downed exactly once.
    /// </summary>
    [Fact]
    public void FlappingMember_ThatStaysUnreachable_IsDownedOneFullWindowAfterLastUnreachable()
    {
        var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(b));
        ExpectNoMsg(TimeSpan.FromMilliseconds(500));
        a.Tell(new ClusterEvent.ReachableMember(b));
        ExpectNoMsg(TimeSpan.FromMilliseconds(200));
        a.Tell(new ClusterEvent.UnreachableMember(b));
        ExpectNoMsg(TimeSpan.FromMilliseconds(500));
        a.Tell(new ClusterEvent.ReachableMember(b));
        ExpectNoMsg(TimeSpan.FromMilliseconds(200));
        a.Tell(new ClusterEvent.UnreachableMember(b)); // final: stays unreachable from here

        // The first flap's timer would fire ~0.1s from here; the final window ends ~1.5s from here.
        ExpectNoMsg(TimeSpan.FromMilliseconds(1100));
        ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(2));
        ExpectNoMsg(TimeSpan.FromSeconds(2));
    }

    /// <summary>The Down decision is logged at Warning, naming the member and the window.</summary>
    [Fact]
    public void DecisionLog_Down_IsLoggedAtWarning()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        EventFilter.Warning(contains: $"[leader-auto-down] DOWNING {MemberB}").ExpectOne(() =>
            a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
        ExpectMsg(new DownCalled(MemberB));
    }

    /// <summary>Observing an unreachable member (and its window) is logged at Warning.</summary>
    [Fact]
    public void DecisionLog_UnreachableScheduled_IsLoggedAtWarning()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(30));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        EventFilter.Warning(contains: $"{MemberB} is unreachable; it will be downed if still unreachable after").ExpectOne(() =>
            a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
    }

    /// <summary>A cancelled Down (member reachable again) is logged.</summary>
    [Fact]
    public void DecisionLog_ReachableAgain_CancelIsLogged()
    {
        var a = AutoDownActor(TimeSpan.FromSeconds(30));
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        var b = CreateMember(MemberB, MemberStatus.Up);
        a.Tell(new ClusterEvent.UnreachableMember(b));
        EventFilter.Info(contains: $"{MemberB} became reachable again; its pending Down is cancelled").ExpectOne(() =>
            a.Tell(new ClusterEvent.ReachableMember(b)));
    }

    /// <summary>A timeout on a non-leader (kept pending) is logged, and so is the later Down on becoming leader.</summary>
    [Fact]
    public void DecisionLog_PendingOnNonLeader_ThenDownOnBecomingLeader_AreLogged()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberB));
        EventFilter.Info(contains: "is not leader; kept pending").ExpectOne(() =>
            a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))));
        EventFilter.Warning(contains: $"DOWNING {MemberC}: it timed out as unreachable while this node was not leader").ExpectOne(() =>
            a.Tell(new ClusterEvent.LeaderChanged(MemberA)));
        ExpectMsg(new DownCalled(MemberC));
    }

    /// <summary>Skipping an already Down member is logged.</summary>
    [Fact]
    public void DecisionLog_SkippedDownMember_IsLogged()
    {
        var a = AutoDownActor(TimeSpan.Zero);
        a.Tell(new ClusterEvent.LeaderChanged(MemberA));
        EventFilter.Info(contains: "but already Down; not scheduling a Down").ExpectOne(() =>
            a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Down))));
    }
}