using System.Collections.Immutable;
using System.Reflection;
using Akka.Actor;
using Akka.Cluster;
using Akka.TestKit.Xunit2;
using Akka.Util;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
///
/// ScadaBridge#106: the decision logic of must match Akka.NET
/// 1.5.62 AutoDownBase exactly. These cases are the ones Akka's own AutoDownSpec
/// pins — cluster events are fed straight to the actor and the Down seam reports to a
/// probe, so the leader-only and reachability semantics are proven without a running cluster.
/// The two-node real-cluster behaviour is proven separately in IntegrationTests
/// (LeaderAutoDownFailoverTests).
///
public class LeaderAutoDownTests : TestKit
{
private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);
private sealed record DownCalled(Address Address);
/// Test double: Down reports to a probe instead of calling the cluster.
private sealed class TestLeaderAutoDown : LeaderAutoDownBase
{
private readonly IActorRef _probe;
public TestLeaderAutoDown(Address selfAddress, TimeSpan unreachableAfter, IActorRef probe)
: base(unreachableAfter)
{
SelfAddress = selfAddress;
_probe = probe;
}
public override Address SelfAddress { get; }
public override IScheduler Scheduler => Context.System.Scheduler;
public override void Down(Address node)
{
// Report EVERY call, leader or not: a base that wrongly downs while not leader must
// show up as an unexpected message, not be swallowed by a throw-and-restart.
_probe.Tell(new DownCalled(node));
if (!IsLeader)
{
_probe.Tell("down-called-while-not-leader");
}
}
}
/// Akka's Member constructors are internal; tests build members via the internal factory.
private static Member CreateMember(Address address, MemberStatus status)
{
var create = typeof(Member).GetMethod(
"Create",
BindingFlags.Static | BindingFlags.NonPublic,
new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet), typeof(AppVersion) })
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed.");
return (Member)create.Invoke(null, new object[]
{
new UniqueAddress(address, 1), 1, status, ImmutableHashSet.Empty, AppVersion.Zero,
})!;
}
private IActorRef AutoDownActor(TimeSpan unreachableAfter) =>
Sys.ActorOf(Props.Create(() => new TestLeaderAutoDown(MemberA, unreachableAfter, TestActor)));
/// A non-leader never downs an unreachable member.
[Fact]
public void NotLeader_DoesNotDownUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// The leader downs an unreachable member immediately when the window is zero.
[Fact]
public void Leader_DownsUnreachable_WhenWindowIsZero()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
ExpectMsg(new DownCalled(MemberB));
}
/// The leader downs an unreachable member only after the full window.
[Fact]
public void Leader_DownsUnreachable_AfterWindow()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
}
/// The initial snapshot drives leadership and schedules members already unreachable.
[Fact]
public void CurrentClusterState_Leader_DownsAlreadyUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.CurrentClusterState(
ImmutableSortedSet.Empty,
ImmutableHashSet.Create(b),
ImmutableHashSet.Empty,
MemberA,
ImmutableDictionary.Empty));
ExpectMsg(new DownCalled(MemberB));
}
/// A pending member is downed as soon as this node becomes leader.
[Fact]
public void BecomingLeader_DownsPendingUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectMsg(new DownCalled(MemberC));
}
/// A member that timed out while not leader is downed on becoming leader.
[Fact]
public void BecomingLeader_AfterWindow_DownsPendingUnreachable()
{
var a = AutoDownActor(TimeSpan.FromSeconds(1));
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectMsg(new DownCalled(MemberC));
}
/// Losing leadership inside the window means no Down when the timer fires.
[Fact]
public void LosingLeadership_InsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// A member that becomes reachable again inside the window is not downed.
[Fact]
public void ReachableAgain_InsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// A pending member that becomes reachable again is not downed on becoming leader.
[Fact]
public void ReachableAgain_WhilePending_IsForgotten()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
var c = CreateMember(MemberC, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(c));
a.Tell(new ClusterEvent.ReachableMember(c));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// A member removed inside the window is not downed.
[Fact]
public void RemovedInsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.MemberRemoved(b.Copy(MemberStatus.Removed), MemberStatus.Up));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// Members already Down or Exiting are skipped (Akka's ConvergenceSkipUnreachableWithMemberStatus).
/// The member status.
[Theory]
[InlineData(MemberStatus.Down)]
[InlineData(MemberStatus.Exiting)]
public void AlreadyDownOrExiting_IsNotDownedAgain(MemberStatus status)
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, status)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// A repeated UnreachableMember for an already-scheduled member does not schedule a second Down.
[Fact]
public void RepeatedUnreachable_DownsOnce()
{
var a = AutoDownActor(TimeSpan.FromSeconds(1));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
}