5cb0e72166
Phase 2 Task 5. DriverHostActor and ScriptedAlarmHostActor now subscribe to the node-local EventStream alongside their existing DPS topic subscriptions, so the transport flag lives ENTIRELY on the publishing side -- exactly one of the two ever publishes, and the switch can be flipped or reverted without touching a single subscriber. Note the alarm case is not symmetric: the DPS subscribe there still carries the node-LOCAL publisher (OtOpcUaServerHostedService's OPC UA Part 9 method calls), which never crosses the boundary and stays on DPS in both modes. Only the central AdminUI leg moves. Renamed DriverHostActor._coordinatorOverride to _ackRouter. Under ClusterClient mode that ref is the NodeCommunicationActor, emphatically not a coordinator, and the old name would send the next reader looking for one. Adds two wiring tests, because the unit tests either side of this seam BOTH pass with the subscription deleted -- the comm actor's test asserts a probe receives the message, and the host tests drive the actor by direct Tell. Neither notices a missing PreStart subscribe. Sabotage-verified: deleting either subscription reddens exactly its own wiring test and nothing else. Three things went wrong while writing those tests, all worth keeping: - The first version raced. ActorOf returns before PreStart runs, and an EventStream.Publish with no subscriber is dropped silently -- no buffering, no dead letter. Fixed with a warm-up whose ack proves PreStart completed; the comment says why it is not ceremony. - The alarm version was VACUOUS: it told the host directly INSIDE the assertion window alongside the relay, so the direct Tell alone produced the log the assertion waited for. Split into warm-up (outside) and relay (inside). - The alarm test needs its own ActorSystem: the shared harness pins loglevel = WARNING and the only signal an unowned alarm gives is a Debug line. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
212 lines
9.4 KiB
C#
212 lines
9.4 KiB
C#
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// The bridge that carries the Primary-gate decision out of the actor and into
|
|
/// <see cref="IRedundancyRoleView"/>, where the alarm store-and-forward drain can read it.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive
|
|
/// <see cref="RedundancyStateChanged"/> itself — but it must be Primary-scoped, because
|
|
/// the queue it drains replicates to the pair peer and two draining nodes would deliver
|
|
/// every alarm event twice.
|
|
/// </para>
|
|
/// <para>
|
|
/// These cases deliberately DIVERGE from <see cref="DriverHostActorPrimaryGateTests"/> in the
|
|
/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate
|
|
/// history row is cheap and already contemplated by the at-least-once contract; a silently
|
|
/// un-drained queue is not. See <c>AlarmHistoryDrainGatePolicyTests</c>.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test");
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>
|
|
/// Before any snapshot arrives the view must read open, not closed.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// A deployment that runs no redundancy never publishes here at all. Defaulting closed
|
|
/// would silently stop its alarm history forever — the exact failure the drain gate exists
|
|
/// to avoid causing.
|
|
/// </remarks>
|
|
[Fact]
|
|
public void An_unpublished_view_reads_as_primary()
|
|
{
|
|
new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>A Secondary closes the view — but only because the snapshot also names a Primary.</summary>
|
|
[Fact]
|
|
public void Secondary_role_closes_the_view_when_a_primary_exists()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2);
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of
|
|
/// this node's rows, so deferring to it would deliver them never.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged(
|
|
[
|
|
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
|
|
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
|
|
// A Primary DOES exist — in another pair. It is not this node's replication peer.
|
|
new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary,
|
|
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId()));
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
[Fact]
|
|
public void Primary_role_opens_the_view()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2);
|
|
|
|
// Published, not current — `true` is the seed. See the note in the unknown-role theory.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Unknown role ⇒ the view stays <b>open</b>, even with driver peers present — the deliberate
|
|
/// divergence from the write gate's default-DENY.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// This case previously asserted the opposite, on the stated goal of mirroring the write
|
|
/// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs
|
|
/// four driver nodes in one cluster with no <c>Redundancy</c> section, so every role was
|
|
/// unknown and the driver count was 4 — and every node logged "Historian drain suspended",
|
|
/// including the two unpaired site-b nodes with no peer and no replication. Alarm history
|
|
/// drained nowhere, where before Phase 2 it drained fine.
|
|
/// </para>
|
|
/// <para>
|
|
/// See <c>AlarmHistoryDrainGatePolicyTests</c> for why the asymmetry of the two error costs
|
|
/// makes fail-open correct here and fail-closed correct for device writes.
|
|
/// </para>
|
|
/// </remarks>
|
|
[Theory]
|
|
[InlineData(1)]
|
|
[InlineData(2)]
|
|
[InlineData(4)]
|
|
public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount)
|
|
{
|
|
// A snapshot that never names this node is how the role stays unknown in practice — the
|
|
// documented identity-mismatch shape, not a contrived case.
|
|
var view = SpawnAndTellForeignSnapshot(driverMemberCount);
|
|
|
|
// Assert on what was PUBLISHED, never on the current value. `true` is also the seeded
|
|
// default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the
|
|
// first poll — before the actor has processed the snapshot — and therefore passes just as
|
|
// happily when the publish is wrong or absent. Verified: with the old write-gate verdict
|
|
// restored, the current-value form stayed green and only this form goes red.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>A demotion must move the view, not just an initial promotion.</summary>
|
|
[Fact]
|
|
public void A_demotion_closes_a_previously_open_view()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
var host = SpawnHost(view, driverMemberCount: 2);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Primary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Secondary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout);
|
|
}
|
|
|
|
// ---------------- helpers ----------------
|
|
|
|
/// <summary>
|
|
/// A view that remembers every published decision, so a test can assert on the actor's
|
|
/// <i>action</i> rather than on a state the seed already satisfies.
|
|
/// </summary>
|
|
private sealed class RecordingRoleView : IRedundancyRoleView
|
|
{
|
|
private readonly List<bool> _published = [];
|
|
private readonly Lock _gate = new();
|
|
private readonly RedundancyRoleView _inner = new();
|
|
|
|
/// <summary>Every value published so far, in order.</summary>
|
|
public IReadOnlyList<bool> Published
|
|
{
|
|
get { lock (_gate) return _published.ToArray(); }
|
|
}
|
|
|
|
public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory;
|
|
|
|
public void Publish(bool shouldDrainAlarmHistory)
|
|
{
|
|
lock (_gate) _published.Add(shouldDrainAlarmHistory);
|
|
_inner.Publish(shouldDrainAlarmHistory);
|
|
}
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role));
|
|
return view;
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary));
|
|
return view;
|
|
}
|
|
|
|
/// <summary>The peer this node replicates with — the only node it may ever stand down for.</summary>
|
|
private const string PeerHost = "the-peer";
|
|
|
|
private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) =>
|
|
Sys.ActorOf(DriverHostActor.Props(
|
|
NewInMemoryDbFactory(),
|
|
TestNode,
|
|
ackRouter: null,
|
|
driverMemberCountProvider: () => driverMemberCount,
|
|
redundancyRoleView: view,
|
|
replicationPeerHost: PeerHost));
|
|
|
|
/// <summary>
|
|
/// A snapshot of a real pair: this node in <paramref name="role"/>, plus a peer holding the
|
|
/// other half. The peer matters — a Secondary only stands down when a Primary exists, so a
|
|
/// single-entry snapshot would silently change what these cases test.
|
|
/// </summary>
|
|
private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) =>
|
|
new(
|
|
[
|
|
new NodeRedundancyState(node, role,
|
|
IsClusterLeader: role == RedundancyRole.Primary,
|
|
IsDriverPrimary: role == RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
new NodeRedundancyState(
|
|
NodeId.Parse($"{PeerHost}:4053"),
|
|
role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary,
|
|
IsClusterLeader: role != RedundancyRole.Primary,
|
|
IsDriverPrimary: role != RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId());
|
|
}
|