Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Redundancy/WarmRoleAuthorityTests.cs
T
Joseph Doherty c569f80694 fix(717): Warm crash survivor stays Active through auto-down; verdict reads its own events
Two defects on the crash-failover path, found by the T13 live gate on an isolated docker pair:

1. When auto-down removed the unreachable Primary (~15 s after the survivor promoted), the
   snapshot's Primary left the unreachable set, so (3') failed and the survivor demoted to
   Standby (BadOutOfService, ServiceLevel 50) for ~15 s, until its own singleton published
   after the hand-over retries. A Primary that is Down, or that this node saw removed, now
   counts as gone (WarmRoleAuthority.SnapshotPrimaryGone). A member that is merely
   Leaving/Exiting, or that this node never saw, is not.

2. The verdict read Cluster.State / SelfMember (Akka's ClusterReadView, a separate subscriber
   to the same events), which can lag the event being handled. Live, the survivor stayed
   Standby 4.3 s after its own UnreachableMember event (promotion waited for the next health
   tick). The Warm verdict now uses a membership view built from this actor's own
   member/reachability subscription, falling back to the read view only before the first event.

Tests: WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active,
OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal
(synthetic events the read view never sees). Mutations: read-view-only verdict and the removed
arm switched off each fail the actor test. Runtime.Tests 1058 passed, 3 skipped.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 04:36:58 -05:00

161 lines
8.4 KiB
C#

using System.Collections.Immutable;
using System.Reflection;
using Akka.Actor;
using Akka.Cluster;
using Akka.Util;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
/// <summary>Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7).</summary>
public sealed class WarmRoleAuthorityTests
{
private static readonly NodeId Self = NodeId.Parse("a:1");
private static readonly NodeId Peer = NodeId.Parse("b:1");
private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true,
NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) =>
new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote);
[Fact]
public void Active_iff_membership_and_the_accepted_snapshot_both_name_self()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf)
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Disagreement.ShouldBeTrue();
}
[Fact]
public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership()
{
var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null));
verdict.Role.ShouldBe(WarmRole.Standby);
verdict.Reason.ShouldBe("boot");
}
[Fact]
public void Not_up_or_endpoint_failed_is_Standby()
{
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby);
}
/// <summary>Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes.</summary>
[Fact]
public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option()
{
var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true);
WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby,
"a reachable Primary is never overridden");
WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role
.ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby");
}
/// <summary>
/// T13 live gate (2026-11-13): auto-down removes the crashed Primary ~15 s after the survivor promoted, while the
/// survivor's own singleton has not yet published a snapshot naming it. A Primary that is Down or no longer a
/// member is as gone as an unreachable one, so the survivor stays Active through the removal. A Primary that is
/// present, reachable and merely Leaving/Exiting (a graceful hand-over) is not gone.
/// </summary>
[Fact]
public void A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active()
{
var self = MemberAt(Self);
var peer = MemberAt(Peer);
var none = new HashSet<Member>();
var seenRemoved = new HashSet<NodeId> { Peer };
var noneRemoved = new HashSet<NodeId>();
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], new HashSet<Member> { peer }, noneRemoved).ShouldBeTrue("unreachable");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Down)], none, noneRemoved).ShouldBeTrue("downed");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved).ShouldBeTrue("removed by auto-down");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, noneRemoved).ShouldBeFalse("never seen as a member: not assumed gone");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, seenRemoved).ShouldBeFalse("back as a reachable Up member");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, noneRemoved).ShouldBeFalse("a reachable Up Primary is never overridden");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Leaving)], none, noneRemoved).ShouldBeFalse("graceful leave");
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Exiting)], none, noneRemoved).ShouldBeFalse("graceful exit");
WarmRoleAuthority.SnapshotPrimaryGone(null, [self], none, seenRemoved).ShouldBeFalse("no snapshot Primary");
// The live sequence on the survivor: promoted while the Primary was unreachable, then the Primary is removed.
var removed = In(oldest: Self, oldestReachable: Self, snapshotPrimary: Peer,
primaryUnreachable: WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved));
WarmRoleAuthority.Evaluate(removed).Role.ShouldBe(WarmRole.Active);
WarmRoleAuthority.Evaluate(removed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
}
private static int _memberUid;
// Akka's Member.Create factories are internal; same reflection idiom as DriverHostActorHistorizationRoleViewTests.
private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod(
"Create",
BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public,
binder: null,
types:
[
typeof(UniqueAddress), typeof(int), typeof(MemberStatus),
typeof(ImmutableHashSet<string>), typeof(AppVersion),
],
modifiers: null)
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found; Akka API changed");
private static Member MemberAt(NodeId node, MemberStatus status = MemberStatus.Up)
{
var text = node.ToString();
var colon = text.LastIndexOf(':');
return (Member)MemberCreate.Invoke(null,
[
new UniqueAddress(
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])),
Interlocked.Increment(ref _memberUid)),
1,
status,
ImmutableHashSet.Create("driver"),
AppVersion.Zero,
])!;
}
[Fact]
public void A_late_bind_replays_the_retained_state_but_not_the_actions()
{
var deferred = new DeferredStandbyProjection();
deferred.SetDataProjection(false);
deferred.SetStandbyDelivery(false);
deferred.SetAlarmSuppression(false);
deferred.QueueCurrentToLiveItems(false).ShouldBe(0);
deferred.RefreshAllEventItems().ShouldBe(0);
var inner = new RecordingProjection();
deferred.SetInner(inner);
inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" });
deferred.QueueCurrentToLiveItems(true);
inner.Log.Last().ShouldBe("queue:True");
}
}
/// <summary>Records every <see cref="IStandbyProjection"/> step into a shared, ordered log.</summary>
internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue<string>? log = null) : IStandbyProjection
{
public System.Collections.Concurrent.ConcurrentQueue<string> Log { get; } = log ?? new();
public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}");
public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}");
public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}");
public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; }
public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; }
}