Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Health/PeerProbeSupervisorTests.cs
T
Joseph Doherty 50b55ee4b9 fix(redundancy): elect the Primary by cluster age, not by address
RedundancyStateActor derived the driver Primary from ClusterState.RoleLeader("driver").
Akka offers two different notions of a "first" member and they are not the same
one: role leader is the lowest-ADDRESSED Up member (host, then port), while
ClusterSingletonManager places singletons on the OLDEST — lowest up-number.

They agree on a freshly-formed cluster, which is why every existing test passed.
They diverge after any restart: the restarted node re-joins as the youngest while
keeping its address, so if it holds the lower address it becomes role leader while
the singletons stay put. The snapshot would then name a Primary that is not
hosting the work, and every Primary-gated surface follows it — inbound device
writes, native-alarm acks, the fleet-wide alerts emit, and the alarm-history
drain would all enable on the wrong node while the node actually running the
singletons stayed gated off.

BuildSnapshot now selects the oldest Up member carrying the driver role, matching
singleton placement. Leaving members are excluded: a node handing its singletons
over must not be named Primary.

NodeRedundancyState.IsRoleLeaderForDriver is renamed IsDriverPrimary, and
NodeHealthInputs.IsDriverRoleLeader likewise. Keeping the old names would have
left the wire contract asserting a derivation the code no longer uses — the same
drift that made this defect invisible.

Proven by a real two-node cluster rather than a mock. RedundancyPrimaryElectionTests
binds the first-joining node to the HIGHER port, so oldest and lowest-address name
different nodes, and includes a fixture assertion that the divergence actually
occurred — without it the real assertion could pass for the wrong reason. Positive
control: restoring the RoleLeader derivation turns exactly the two election tests
red while the fixture check stays green.

Runtime.Tests 440 passed, ControlPlane.Tests 82, Cluster.Tests 36.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-21 18:39:07 -04:00

159 lines
7.1 KiB
C#

using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
/// <summary>
/// Tests for <see cref="PeerProbeSupervisor"/>: it maintains exactly one peer-OPC-UA-probe child
/// per OTHER, non-Detached driver node named in the latest <see cref="RedundancyStateChanged"/>
/// snapshot, spawning/stopping children as the topology changes.
/// </summary>
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
{
private static readonly NodeId Local = NodeId.Parse("local:4053");
private static readonly NodeId Peer = NodeId.Parse("peer:4053");
private static readonly NodeId Adm = NodeId.Parse("adm:4053");
/// <summary>No-op child actor stub so we can count children without real TCP probes.</summary>
private sealed class NoopActor : ReceiveActor { }
private static Props NoopProps() => Akka.Actor.Props.Create(() => new NoopActor());
/// <summary>
/// No-op child stub that records its own <see cref="ActorBase.Self"/> into a shared list on
/// start, in spawn order — lets a test grab a specific (e.g. the FIRST, old-generation) child
/// ref so it can deliver a synthetic <see cref="Terminated"/> for it.
/// </summary>
private sealed class RecordingNoopActor : ReceiveActor
{
public RecordingNoopActor(List<IActorRef> spawned) => spawned.Add(Self);
}
private static Props RecordingProps(List<IActorRef> spawned) =>
Akka.Actor.Props.Create(() => new RecordingNoopActor(spawned));
private static NodeRedundancyState State(NodeId id, RedundancyRole role) =>
new(id, role, IsClusterLeader: false, IsDriverPrimary: false, DateTime.UtcNow);
private static RedundancyStateChanged Snapshot(params NodeRedundancyState[] nodes) =>
new(nodes, CorrelationId.NewId());
/// <summary>Verifies one child is spawned per non-self, non-Detached peer — self and Detached
/// nodes are excluded.</summary>
[Fact]
public void Spawns_one_child_per_non_self_non_detached_peer()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary),
State(Adm, RedundancyRole.Detached)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies the child for a departed peer is stopped when the next snapshot omits it.</summary>
[Fact]
public void Stops_child_for_departed_peer()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies a single-node snapshot (just the local node) spawns no children.</summary>
[Fact]
public void Single_node_snapshot_spawns_no_children()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies a previously-removed peer is respawned when it re-appears, without an
/// "actor name not unique" collision on the sanitized child name.</summary>
[Fact]
public void Re_adding_a_previously_removed_peer_respawns_it()
{
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
/// <summary>Locks in the stale-Terminated guard: when an OLD (already-replaced) child's
/// <see cref="Terminated"/> for a peer arrives AFTER a NEW child for the SAME peer was spawned,
/// the fresh child must NOT be evicted (removal is keyed by current-child ref-equality, not by
/// peer key). Without this guard a late stale Terminated would silently drop a live probe.</summary>
[Fact]
public void Stale_terminated_for_old_child_does_not_evict_fresh_peer_child()
{
var spawned = new List<IActorRef>();
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
PeerProbeSupervisor.PropsForTests(Local, _ => RecordingProps(spawned)));
// First add of the peer -> child #0 (the OLD ref).
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
AwaitAssert(() => spawned.Count.ShouldBe(1), duration: TimeSpan.FromMilliseconds(500));
var oldRef = spawned[0];
// Drop the peer -> child #0 stopped, ChildCount back to 0.
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
duration: TimeSpan.FromMilliseconds(500));
// Re-add the SAME peer -> a NEW child #1 (the FRESH ref) is spawned.
sup.Tell(Snapshot(
State(Local, RedundancyRole.Primary),
State(Peer, RedundancyRole.Secondary)));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
AwaitAssert(() => spawned.Count.ShouldBe(2), duration: TimeSpan.FromMilliseconds(500));
// Now deliver a STALE Terminated for the OLD ref. The current child for Peer is the fresh
// child #1, so ref-equality finds no match and the supervisor must leave ChildCount at 1.
sup.Tell(new Terminated(oldRef, existenceConfirmed: true, addressTerminated: false));
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
duration: TimeSpan.FromMilliseconds(500));
}
}