using System.Collections.Immutable;
using System.Reflection;
using Akka.Actor;
using Akka.Cluster;
using Akka.TestKit.Xunit2;
using Akka.Util;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
///
/// ScadaBridge#106: the decision logic of must match Akka.NET
/// 1.5.62 AutoDownBase exactly. These cases are the ones Akka's own AutoDownSpec
/// pins — cluster events are fed straight to the actor and the Down seam reports to a
/// probe, so the leader-only and reachability semantics are proven without a running cluster.
/// The two-node real-cluster behaviour is proven separately in IntegrationTests
/// (LeaderAutoDownFailoverTests).
///
public class LeaderAutoDownTests : TestKit
{
/// INFO so the decision-log assertions can see the Info-level decisions.
public LeaderAutoDownTests()
: base("akka.loglevel = INFO")
{
}
private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);
private sealed record DownCalled(Address Address);
/// Test double: Down reports to a probe instead of calling the cluster.
private sealed class TestLeaderAutoDown : LeaderAutoDownBase
{
private readonly IActorRef _probe;
public TestLeaderAutoDown(Address selfAddress, TimeSpan unreachableAfter, IActorRef probe)
: base(unreachableAfter)
{
SelfAddress = selfAddress;
_probe = probe;
}
public override Address SelfAddress { get; }
public override IScheduler Scheduler => Context.System.Scheduler;
public override void Down(Address node)
{
// Report EVERY call, leader or not: a base that wrongly downs while not leader must
// show up as an unexpected message, not be swallowed by a throw-and-restart.
_probe.Tell(new DownCalled(node));
if (!IsLeader)
{
_probe.Tell("down-called-while-not-leader");
}
}
}
/// Akka's Member constructors are internal; tests build members via the internal factory.
private static Member CreateMember(Address address, MemberStatus status)
{
var create = typeof(Member).GetMethod(
"Create",
BindingFlags.Static | BindingFlags.NonPublic,
new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet), typeof(AppVersion) })
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed.");
return (Member)create.Invoke(null, new object[]
{
new UniqueAddress(address, 1), 1, status, ImmutableHashSet.Empty, AppVersion.Zero,
})!;
}
private IActorRef AutoDownActor(TimeSpan unreachableAfter) =>
Sys.ActorOf(Props.Create(() => new TestLeaderAutoDown(MemberA, unreachableAfter, TestActor)));
/// A non-leader never downs an unreachable member.
[Fact]
public void NotLeader_DoesNotDownUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// The leader downs an unreachable member immediately when the window is zero.
[Fact]
public void Leader_DownsUnreachable_WhenWindowIsZero()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
ExpectMsg(new DownCalled(MemberB));
}
/// The leader downs an unreachable member only after the full window.
[Fact]
public void Leader_DownsUnreachable_AfterWindow()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
}
/// The initial snapshot drives leadership and schedules members already unreachable.
[Fact]
public void CurrentClusterState_Leader_DownsAlreadyUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.CurrentClusterState(
ImmutableSortedSet.Empty,
ImmutableHashSet.Create(b),
ImmutableHashSet.Empty,
MemberA,
ImmutableDictionary.Empty));
ExpectMsg(new DownCalled(MemberB));
}
/// A pending member is downed as soon as this node becomes leader.
[Fact]
public void BecomingLeader_DownsPendingUnreachable()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectMsg(new DownCalled(MemberC));
}
/// A member that timed out while not leader is downed on becoming leader.
[Fact]
public void BecomingLeader_AfterWindow_DownsPendingUnreachable()
{
var a = AutoDownActor(TimeSpan.FromSeconds(1));
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
ExpectNoMsg(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectMsg(new DownCalled(MemberC));
}
/// Losing leadership inside the window means no Down when the timer fires.
[Fact]
public void LosingLeadership_InsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// A member that becomes reachable again inside the window is not downed.
[Fact]
public void ReachableAgain_InsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// A pending member that becomes reachable again is not downed on becoming leader.
[Fact]
public void ReachableAgain_WhilePending_IsForgotten()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
var c = CreateMember(MemberC, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(c));
a.Tell(new ClusterEvent.ReachableMember(c));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// A member removed inside the window is not downed.
[Fact]
public void RemovedInsideWindow_DoesNotDown()
{
var a = AutoDownActor(TimeSpan.FromSeconds(2));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.MemberRemoved(b.Copy(MemberStatus.Removed), MemberStatus.Up));
ExpectNoMsg(TimeSpan.FromSeconds(3));
}
/// Members already Down or Exiting are skipped (Akka's ConvergenceSkipUnreachableWithMemberStatus).
/// The member status.
[Theory]
[InlineData(MemberStatus.Down)]
[InlineData(MemberStatus.Exiting)]
public void AlreadyDownOrExiting_IsNotDownedAgain(MemberStatus status)
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, status)));
ExpectNoMsg(TimeSpan.FromSeconds(1));
}
/// A repeated UnreachableMember for an already-scheduled member does not schedule a second Down.
[Fact]
public void RepeatedUnreachable_DownsOnce()
{
var a = AutoDownActor(TimeSpan.FromSeconds(1));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
///
/// A flapping member — unreachable, reachable again inside the window, repeatedly, for far
/// longer than one window in total — is never downed.
///
[Fact]
public void FlappingMember_ReachableAgainInsideEachWindow_IsNeverDowned()
{
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
for (var i = 0; i < 5; i++)
{
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(700));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
}
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
///
/// When a flapping member finally stays unreachable, the window restarts from its LAST
/// unreachable — an earlier flap's timer must not fire — and it is downed exactly once.
///
[Fact]
public void FlappingMember_ThatStaysUnreachable_IsDownedOneFullWindowAfterLastUnreachable()
{
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
a.Tell(new ClusterEvent.UnreachableMember(b)); // final: stays unreachable from here
// The first flap's timer would fire ~0.1s from here; the final window ends ~1.5s from here.
ExpectNoMsg(TimeSpan.FromMilliseconds(1100));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(2));
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
/// The Down decision is logged at Warning, naming the member and the window.
[Fact]
public void DecisionLog_Down_IsLoggedAtWarning()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Warning(contains: $"[leader-auto-down] DOWNING {MemberB}").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
ExpectMsg(new DownCalled(MemberB));
}
/// Observing an unreachable member (and its window) is logged at Warning.
[Fact]
public void DecisionLog_UnreachableScheduled_IsLoggedAtWarning()
{
var a = AutoDownActor(TimeSpan.FromSeconds(30));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Warning(contains: $"{MemberB} is unreachable; it will be downed if still unreachable after").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
}
/// A cancelled Down (member reachable again) is logged.
[Fact]
public void DecisionLog_ReachableAgain_CancelIsLogged()
{
var a = AutoDownActor(TimeSpan.FromSeconds(30));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
EventFilter.Info(contains: $"{MemberB} became reachable again; its pending Down is cancelled").ExpectOne(() =>
a.Tell(new ClusterEvent.ReachableMember(b)));
}
/// A timeout on a non-leader (kept pending) is logged, and so is the later Down on becoming leader.
[Fact]
public void DecisionLog_PendingOnNonLeader_ThenDownOnBecomingLeader_AreLogged()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
EventFilter.Info(contains: "is not leader; kept pending").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))));
EventFilter.Warning(contains: $"DOWNING {MemberC}: it timed out as unreachable while this node was not leader").ExpectOne(() =>
a.Tell(new ClusterEvent.LeaderChanged(MemberA)));
ExpectMsg(new DownCalled(MemberC));
}
/// Skipping an already Down member is logged.
[Fact]
public void DecisionLog_SkippedDownMember_IsLogged()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Info(contains: "but already Down; not scheduling a Down").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Down))));
}
}