using System.Collections.Immutable; using System.Reflection; using Akka.Actor; using Akka.Cluster; using Akka.Util; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy; /// Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7). public sealed class WarmRoleAuthorityTests { private static readonly NodeId Self = NodeId.Parse("a:1"); private static readonly NodeId Peer = NodeId.Parse("b:1"); private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true, NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) => new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote); [Fact] public void Active_iff_membership_and_the_accepted_snapshot_both_name_self() { WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active); WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby); } [Theory] [InlineData(true)] [InlineData(false)] public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf) { var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self)); verdict.Role.ShouldBe(WarmRole.Standby); verdict.Disagreement.ShouldBeTrue(); } [Fact] public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership() { var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null)); verdict.Role.ShouldBe(WarmRole.Standby); verdict.Reason.ShouldBe("boot"); } [Fact] public void Not_up_or_endpoint_failed_is_Standby() { WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby); WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby); } /// Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes. [Fact] public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option() { var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true); WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active); WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby); WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby, "a reachable Primary is never overridden"); WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role .ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby"); } /// /// T13 live gate (2026-11-13): auto-down removes the crashed Primary ~15 s after the survivor promoted, while the /// survivor's own singleton has not yet published a snapshot naming it. A Primary that is Down or no longer a /// member is as gone as an unreachable one, so the survivor stays Active through the removal. A Primary that is /// present, reachable and merely Leaving/Exiting (a graceful hand-over) is not gone. /// [Fact] public void A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active() { var self = MemberAt(Self); var peer = MemberAt(Peer); var none = new HashSet(); var seenRemoved = new HashSet { Peer }; var noneRemoved = new HashSet(); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], new HashSet { peer }, noneRemoved).ShouldBeTrue("unreachable"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Down)], none, noneRemoved).ShouldBeTrue("downed"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved).ShouldBeTrue("removed by auto-down"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, noneRemoved).ShouldBeFalse("never seen as a member: not assumed gone"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, seenRemoved).ShouldBeFalse("back as a reachable Up member"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, noneRemoved).ShouldBeFalse("a reachable Up Primary is never overridden"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Leaving)], none, noneRemoved).ShouldBeFalse("graceful leave"); WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Exiting)], none, noneRemoved).ShouldBeFalse("graceful exit"); WarmRoleAuthority.SnapshotPrimaryGone(null, [self], none, seenRemoved).ShouldBeFalse("no snapshot Primary"); // The live sequence on the survivor: promoted while the Primary was unreachable, then the Primary is removed. var removed = In(oldest: Self, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved)); WarmRoleAuthority.Evaluate(removed).Role.ShouldBe(WarmRole.Active); WarmRoleAuthority.Evaluate(removed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby); } private static int _memberUid; // Akka's Member.Create factories are internal; same reflection idiom as DriverHostActorHistorizationRoleViewTests. private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod( "Create", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public, binder: null, types: [ typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet), typeof(AppVersion), ], modifiers: null) ?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found; Akka API changed"); private static Member MemberAt(NodeId node, MemberStatus status = MemberStatus.Up) { var text = node.ToString(); var colon = text.LastIndexOf(':'); return (Member)MemberCreate.Invoke(null, [ new UniqueAddress( new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])), Interlocked.Increment(ref _memberUid)), 1, status, ImmutableHashSet.Create("driver"), AppVersion.Zero, ])!; } [Fact] public void A_late_bind_replays_the_retained_state_but_not_the_actions() { var deferred = new DeferredStandbyProjection(); deferred.SetDataProjection(false); deferred.SetStandbyDelivery(false); deferred.SetAlarmSuppression(false); deferred.QueueCurrentToLiveItems(false).ShouldBe(0); deferred.RefreshAllEventItems().ShouldBe(0); var inner = new RecordingProjection(); deferred.SetInner(inner); inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" }); deferred.QueueCurrentToLiveItems(true); inner.Log.Last().ShouldBe("queue:True"); } } /// Records every step into a shared, ordered log. internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue? log = null) : IStandbyProjection { public System.Collections.Concurrent.ConcurrentQueue Log { get; } = log ?? new(); public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}"); public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}"); public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}"); public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; } public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; } }