Joseph Doherty
240d7aa8aa
`AwaitAssert` returns on its FIRST success, so an assertion that is already true at t=0 passes instantly and spends none of its duration. All three sites asserted an absence against a collection that starts empty, so none of them ever gave the handler the time their comments claimed. Replaced with mailbox-ordering positive controls rather than sleeps: - `RouteNativeAlarmAck_unknown_node_is_dropped` — follow the unmapped ack with a MAPPED one. The host forwards via `Tell` and the child handles `RouteAlarmAck` with `ReceiveAsync` (which suspends its own mailbox), so once the control reaches the driver the unmapped one has provably been handled. - `RouteNativeAlarmAck_on_non_primary_is_dropped` — the role itself is the control: return the node to Primary and re-send. Comments (`gated` / `control`) are the discriminator, so a leaked ack is visible rather than merged into a count. - `PeerProbeSupervisorTests.Single_node_snapshot_spawns_no_children` — a FOURTH site the issue did not list. It says the other `ChildCount.ShouldBe(0)` waits are each preceded by a `ShouldBe(1)`; that is true of three of the four, but not this one, which asserts the initial state. Now followed by a two-node snapshot: the count reaching 1 proves the single-node snapshot was processed, and a local-node child would have made it 2. The issue warned these may turn red, which would be a real defect surfacing. They did not — the routing is correct. Proven by breaking it deliberately instead: disabling the Primary gate reddens the Secondary test, and routing unmapped ids to an arbitrary mapping reddens the unknown-node test. Under the old form both breaks passed.
174 lines
8.0 KiB
C#
174 lines
8.0 KiB
C#
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
|
|
|
/// <summary>
|
|
/// Tests for <see cref="PeerProbeSupervisor"/>: it maintains exactly one peer-OPC-UA-probe child
|
|
/// per OTHER, non-Detached driver node named in the latest <see cref="RedundancyStateChanged"/>
|
|
/// snapshot, spawning/stopping children as the topology changes.
|
|
/// </summary>
|
|
public sealed class PeerProbeSupervisorTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId Local = NodeId.Parse("local:4053");
|
|
private static readonly NodeId Peer = NodeId.Parse("peer:4053");
|
|
private static readonly NodeId Adm = NodeId.Parse("adm:4053");
|
|
|
|
/// <summary>No-op child actor stub so we can count children without real TCP probes.</summary>
|
|
private sealed class NoopActor : ReceiveActor { }
|
|
|
|
private static Props NoopProps() => Akka.Actor.Props.Create(() => new NoopActor());
|
|
|
|
/// <summary>
|
|
/// No-op child stub that records its own <see cref="ActorBase.Self"/> into a shared list on
|
|
/// start, in spawn order — lets a test grab a specific (e.g. the FIRST, old-generation) child
|
|
/// ref so it can deliver a synthetic <see cref="Terminated"/> for it.
|
|
/// </summary>
|
|
private sealed class RecordingNoopActor : ReceiveActor
|
|
{
|
|
public RecordingNoopActor(List<IActorRef> spawned) => spawned.Add(Self);
|
|
}
|
|
|
|
private static Props RecordingProps(List<IActorRef> spawned) =>
|
|
Akka.Actor.Props.Create(() => new RecordingNoopActor(spawned));
|
|
|
|
private static NodeRedundancyState State(NodeId id, RedundancyRole role) =>
|
|
new(id, role, IsClusterLeader: false, IsDriverPrimary: false, DateTime.UtcNow);
|
|
|
|
private static RedundancyStateChanged Snapshot(params NodeRedundancyState[] nodes) =>
|
|
new(nodes, CorrelationId.NewId());
|
|
|
|
/// <summary>Verifies one child is spawned per non-self, non-Detached peer — self and Detached
|
|
/// nodes are excluded.</summary>
|
|
[Fact]
|
|
public void Spawns_one_child_per_non_self_non_detached_peer()
|
|
{
|
|
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
|
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
|
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary),
|
|
State(Adm, RedundancyRole.Detached)));
|
|
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
}
|
|
|
|
/// <summary>Verifies the child for a departed peer is stopped when the next snapshot omits it.</summary>
|
|
[Fact]
|
|
public void Stops_child_for_departed_peer()
|
|
{
|
|
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
|
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
|
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
|
|
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
|
duration: PresenceBudget);
|
|
}
|
|
|
|
/// <summary>Verifies a single-node snapshot (just the local node) spawns no children — the local node
|
|
/// is never its own probe peer. Established against a following two-node snapshot as a positive
|
|
/// control, so the count of 1 (rather than 2) is what carries the "no local child" claim.</summary>
|
|
[Fact]
|
|
public void Single_node_snapshot_spawns_no_children()
|
|
{
|
|
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
|
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
|
|
|
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
|
|
|
// POSITIVE CONTROL (#501). This is an ABSENCE assertion, and ChildCount is 0 before the snapshot
|
|
// above has been processed at all — so asserting it directly passes at t=0 and proves nothing
|
|
// (AwaitAssert returns on its first success, spending none of its duration). Unlike the other
|
|
// ChildCount.ShouldBe(0) waits in this class, there is no preceding ShouldBe(1) to make it a
|
|
// transition.
|
|
//
|
|
// Follow with a snapshot that DOES spawn exactly one child. The supervisor processes its mailbox
|
|
// in order, so once the count reaches 1 the single-node snapshot has provably been handled — and
|
|
// had it spawned a child for the local node, the count would be 2.
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
}
|
|
|
|
/// <summary>Verifies a previously-removed peer is respawned when it re-appears, without an
|
|
/// "actor name not unique" collision on the sanitized child name.</summary>
|
|
[Fact]
|
|
public void Re_adding_a_previously_removed_peer_respawns_it()
|
|
{
|
|
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
|
PeerProbeSupervisor.PropsForTests(Local, _ => NoopProps()));
|
|
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
|
|
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
|
duration: PresenceBudget);
|
|
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
}
|
|
|
|
/// <summary>Locks in the stale-Terminated guard: when an OLD (already-replaced) child's
|
|
/// <see cref="Terminated"/> for a peer arrives AFTER a NEW child for the SAME peer was spawned,
|
|
/// the fresh child must NOT be evicted (removal is keyed by current-child ref-equality, not by
|
|
/// peer key). Without this guard a late stale Terminated would silently drop a live probe.</summary>
|
|
[Fact]
|
|
public void Stale_terminated_for_old_child_does_not_evict_fresh_peer_child()
|
|
{
|
|
var spawned = new List<IActorRef>();
|
|
var sup = ActorOfAsTestActorRef<PeerProbeSupervisor>(
|
|
PeerProbeSupervisor.PropsForTests(Local, _ => RecordingProps(spawned)));
|
|
|
|
// First add of the peer -> child #0 (the OLD ref).
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
AwaitAssert(() => spawned.Count.ShouldBe(1), duration: PresenceBudget);
|
|
var oldRef = spawned[0];
|
|
|
|
// Drop the peer -> child #0 stopped, ChildCount back to 0.
|
|
sup.Tell(Snapshot(State(Local, RedundancyRole.Primary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(0),
|
|
duration: PresenceBudget);
|
|
|
|
// Re-add the SAME peer -> a NEW child #1 (the FRESH ref) is spawned.
|
|
sup.Tell(Snapshot(
|
|
State(Local, RedundancyRole.Primary),
|
|
State(Peer, RedundancyRole.Secondary)));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
AwaitAssert(() => spawned.Count.ShouldBe(2), duration: PresenceBudget);
|
|
|
|
// Now deliver a STALE Terminated for the OLD ref. The current child for Peer is the fresh
|
|
// child #1, so ref-equality finds no match and the supervisor must leave ChildCount at 1.
|
|
sup.Tell(new Terminated(oldRef, existenceConfirmed: true, addressTerminated: false));
|
|
AwaitAssert(() => sup.UnderlyingActor.ChildCount.ShouldBe(1),
|
|
duration: PresenceBudget);
|
|
}
|
|
}
|