using System.Collections.Immutable; using System.Reflection; using Akka.Actor; using Akka.Cluster; using Akka.TestKit.Xunit2; using Akka.Util; using ZB.MOM.WW.ScadaBridge.Host.Actors; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; /// /// ScadaBridge#106: the decision logic of must match Akka.NET /// 1.5.62 AutoDownBase exactly. These cases are the ones Akka's own AutoDownSpec /// pins — cluster events are fed straight to the actor and the Down seam reports to a /// probe, so the leader-only and reachability semantics are proven without a running cluster. /// The two-node real-cluster behaviour is proven separately in IntegrationTests /// (LeaderAutoDownFailoverTests). /// public class LeaderAutoDownTests : TestKit { private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552); private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552); private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552); private sealed record DownCalled(Address Address); /// Test double: Down reports to a probe instead of calling the cluster. private sealed class TestLeaderAutoDown : LeaderAutoDownBase { private readonly IActorRef _probe; public TestLeaderAutoDown(Address selfAddress, TimeSpan unreachableAfter, IActorRef probe) : base(unreachableAfter) { SelfAddress = selfAddress; _probe = probe; } public override Address SelfAddress { get; } public override IScheduler Scheduler => Context.System.Scheduler; public override void Down(Address node) { // Report EVERY call, leader or not: a base that wrongly downs while not leader must // show up as an unexpected message, not be swallowed by a throw-and-restart. _probe.Tell(new DownCalled(node)); if (!IsLeader) { _probe.Tell("down-called-while-not-leader"); } } } /// Akka's Member constructors are internal; tests build members via the internal factory. private static Member CreateMember(Address address, MemberStatus status) { var create = typeof(Member).GetMethod( "Create", BindingFlags.Static | BindingFlags.NonPublic, new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet), typeof(AppVersion) }) ?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed."); return (Member)create.Invoke(null, new object[] { new UniqueAddress(address, 1), 1, status, ImmutableHashSet.Empty, AppVersion.Zero, })!; } private IActorRef AutoDownActor(TimeSpan unreachableAfter) => Sys.ActorOf(Props.Create(() => new TestLeaderAutoDown(MemberA, unreachableAfter, TestActor))); /// A non-leader never downs an unreachable member. [Fact] public void NotLeader_DoesNotDownUnreachable() { var a = AutoDownActor(TimeSpan.Zero); a.Tell(new ClusterEvent.LeaderChanged(MemberB)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))); ExpectNoMsg(TimeSpan.FromSeconds(1)); } /// The leader downs an unreachable member immediately when the window is zero. [Fact] public void Leader_DownsUnreachable_WhenWindowIsZero() { var a = AutoDownActor(TimeSpan.Zero); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))); ExpectMsg(new DownCalled(MemberB)); } /// The leader downs an unreachable member only after the full window. [Fact] public void Leader_DownsUnreachable_AfterWindow() { var a = AutoDownActor(TimeSpan.FromSeconds(2)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))); ExpectNoMsg(TimeSpan.FromSeconds(1)); ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3)); } /// The initial snapshot drives leadership and schedules members already unreachable. [Fact] public void CurrentClusterState_Leader_DownsAlreadyUnreachable() { var a = AutoDownActor(TimeSpan.Zero); var b = CreateMember(MemberB, MemberStatus.Up); a.Tell(new ClusterEvent.CurrentClusterState( ImmutableSortedSet.Empty, ImmutableHashSet.Create(b), ImmutableHashSet
.Empty, MemberA, ImmutableDictionary.Empty)); ExpectMsg(new DownCalled(MemberB)); } /// A pending member is downed as soon as this node becomes leader. [Fact] public void BecomingLeader_DownsPendingUnreachable() { var a = AutoDownActor(TimeSpan.Zero); a.Tell(new ClusterEvent.LeaderChanged(MemberB)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))); ExpectNoMsg(TimeSpan.FromMilliseconds(500)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); ExpectMsg(new DownCalled(MemberC)); } /// A member that timed out while not leader is downed on becoming leader. [Fact] public void BecomingLeader_AfterWindow_DownsPendingUnreachable() { var a = AutoDownActor(TimeSpan.FromSeconds(1)); a.Tell(new ClusterEvent.LeaderChanged(MemberB)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))); ExpectNoMsg(TimeSpan.FromSeconds(2)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); ExpectMsg(new DownCalled(MemberC)); } /// Losing leadership inside the window means no Down when the timer fires. [Fact] public void LosingLeadership_InsideWindow_DoesNotDown() { var a = AutoDownActor(TimeSpan.FromSeconds(2)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))); a.Tell(new ClusterEvent.LeaderChanged(MemberB)); ExpectNoMsg(TimeSpan.FromSeconds(3)); } /// A member that becomes reachable again inside the window is not downed. [Fact] public void ReachableAgain_InsideWindow_DoesNotDown() { var a = AutoDownActor(TimeSpan.FromSeconds(2)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); var b = CreateMember(MemberB, MemberStatus.Up); a.Tell(new ClusterEvent.UnreachableMember(b)); a.Tell(new ClusterEvent.ReachableMember(b)); ExpectNoMsg(TimeSpan.FromSeconds(3)); } /// A pending member that becomes reachable again is not downed on becoming leader. [Fact] public void ReachableAgain_WhilePending_IsForgotten() { var a = AutoDownActor(TimeSpan.Zero); a.Tell(new ClusterEvent.LeaderChanged(MemberB)); var c = CreateMember(MemberC, MemberStatus.Up); a.Tell(new ClusterEvent.UnreachableMember(c)); a.Tell(new ClusterEvent.ReachableMember(c)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); ExpectNoMsg(TimeSpan.FromSeconds(1)); } /// A member removed inside the window is not downed. [Fact] public void RemovedInsideWindow_DoesNotDown() { var a = AutoDownActor(TimeSpan.FromSeconds(2)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); var b = CreateMember(MemberB, MemberStatus.Up); a.Tell(new ClusterEvent.UnreachableMember(b)); a.Tell(new ClusterEvent.MemberRemoved(b.Copy(MemberStatus.Removed), MemberStatus.Up)); ExpectNoMsg(TimeSpan.FromSeconds(3)); } /// Members already Down or Exiting are skipped (Akka's ConvergenceSkipUnreachableWithMemberStatus). /// The member status. [Theory] [InlineData(MemberStatus.Down)] [InlineData(MemberStatus.Exiting)] public void AlreadyDownOrExiting_IsNotDownedAgain(MemberStatus status) { var a = AutoDownActor(TimeSpan.Zero); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, status))); ExpectNoMsg(TimeSpan.FromSeconds(1)); } /// A repeated UnreachableMember for an already-scheduled member does not schedule a second Down. [Fact] public void RepeatedUnreachable_DownsOnce() { var a = AutoDownActor(TimeSpan.FromSeconds(1)); a.Tell(new ClusterEvent.LeaderChanged(MemberA)); var b = CreateMember(MemberB, MemberStatus.Up); a.Tell(new ClusterEvent.UnreachableMember(b)); a.Tell(new ClusterEvent.UnreachableMember(b)); ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3)); ExpectNoMsg(TimeSpan.FromSeconds(2)); } }