Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs
T
Joseph Doherty 3be8a1217e fix(runtime): the passive half of a LocalDb pair stands its alarm drain down for its mesh partner (review R6 MINOR-4)
The drain gate's 'is my queue-sharing peer Primary?' used only the host of
LocalDb:Replication:PeerAddress, which the passive half (ApiKey +
SyncListenPort) does not set - so as Secondary it kept draining beside its
Primary partner and every alarm event reached the historian twice.
DriverHostActor takes a sharesAlarmQueueWithLocal predicate used when no
peer host is known; WithOtOpcUaRuntimeActors builds it
(BuildAlarmQueuePeerPredicate) for a SharesRowsWithPeer node with no
PeerAddress as 'a member of my own cluster-{ClusterId} mesh other than
me'. A node sharing no rows, or with no mesh role, gets none and keeps
draining. AlarmHistorian.md / Redundancy.md / pair-replication runbook
updated.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 08:43:06 -04:00

249 lines
11 KiB
C#

using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// The bridge that carries the Primary-gate decision out of the actor and into
/// <see cref="IRedundancyRoleView"/>, where the alarm store-and-forward drain can read it.
/// </summary>
/// <remarks>
/// <para>
/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive
/// <see cref="RedundancyStateChanged"/> itself — but it must be Primary-scoped, because
/// the queue it drains replicates to the pair peer and two draining nodes would deliver
/// every alarm event twice.
/// </para>
/// <para>
/// These cases deliberately DIVERGE from <see cref="DriverHostActorPrimaryGateTests"/> in the
/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate
/// history row is cheap and already contemplated by the at-least-once contract; a silently
/// un-drained queue is not. See <c>AlarmHistoryDrainGatePolicyTests</c>.
/// </para>
/// </remarks>
public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test");
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
/// <summary>
/// Before any snapshot arrives the view must read open, not closed.
/// </summary>
/// <remarks>
/// A deployment that runs no redundancy never publishes here at all. Defaulting closed
/// would silently stop its alarm history forever — the exact failure the drain gate exists
/// to avoid causing.
/// </remarks>
[Fact]
public void An_unpublished_view_reads_as_primary()
{
new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue();
}
/// <summary>A Secondary closes the view — but only because the snapshot also names a Primary.</summary>
[Fact]
public void Secondary_role_closes_the_view_when_a_primary_exists()
{
var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2);
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
}
/// <summary>
/// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of
/// this node's rows, so deferring to it would deliver them never.
/// </summary>
[Fact]
public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer()
{
var view = new RecordingRoleView();
SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged(
[
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
// A Primary DOES exist — in another pair. It is not this node's replication peer.
new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary,
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
],
CorrelationId.NewId()));
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
[Fact]
public void Primary_role_opens_the_view()
{
var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2);
// Published, not current — `true` is the seed. See the note in the unknown-role theory.
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
/// <summary>
/// Unknown role ⇒ the view stays <b>open</b>, even with driver peers present — the deliberate
/// divergence from the write gate's default-DENY.
/// </summary>
/// <remarks>
/// <para>
/// This case previously asserted the opposite, on the stated goal of mirroring the write
/// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs
/// four driver nodes in one cluster with no <c>Redundancy</c> section, so every role was
/// unknown and the driver count was 4 — and every node logged "Historian drain suspended",
/// including the two unpaired site-b nodes with no peer and no replication. Alarm history
/// drained nowhere, where before Phase 2 it drained fine.
/// </para>
/// <para>
/// See <c>AlarmHistoryDrainGatePolicyTests</c> for why the asymmetry of the two error costs
/// makes fail-open correct here and fail-closed correct for device writes.
/// </para>
/// </remarks>
[Theory]
[InlineData(1)]
[InlineData(2)]
[InlineData(4)]
public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount)
{
// A snapshot that never names this node is how the role stays unknown in practice — the
// documented identity-mismatch shape, not a contrived case.
var view = SpawnAndTellForeignSnapshot(driverMemberCount);
// Assert on what was PUBLISHED, never on the current value. `true` is also the seeded
// default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the
// first poll — before the actor has processed the snapshot — and therefore passes just as
// happily when the publish is wrong or absent. Verified: with the old write-gate verdict
// restored, the current-value form stayed green and only this form goes red.
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
}
/// <summary>A demotion must move the view, not just an initial promotion.</summary>
[Fact]
public void A_demotion_closes_a_previously_open_view()
{
var view = new RecordingRoleView();
var host = SpawnHost(view, driverMemberCount: 2);
host.Tell(Snapshot(TestNode, RedundancyRole.Primary));
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
host.Tell(Snapshot(TestNode, RedundancyRole.Secondary));
AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout);
}
/// <summary>
/// Review R6 MINOR-4 — the PASSIVE half of a pair dials nobody (no <c>PeerAddress</c>, so no peer host) but
/// holds the same replicated rows, so as Secondary it must stand down for its Primary partner too, which it
/// identifies through the queue-sharing predicate (its mesh partner). Before the fix it published
/// <c>true</c> here and both halves drained. A Primary the predicate does NOT name is the control: the
/// passive half keeps draining, so the deny comes from identifying the partner, not from being Secondary.
/// </summary>
[Fact]
public void A_passive_secondary_closes_the_view_for_its_mesh_partner_and_only_for_it()
{
var partner = NodeId.Parse("site-a-1:4053");
foreach (var (primary, expected) in new[] { (partner, false), (NodeId.Parse("site-b-1:4053"), true) })
{
var view = new RecordingRoleView();
var host = Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(),
TestNode,
ackRouter: null,
driverMemberCountProvider: () => 2,
redundancyRoleView: view,
replicationPeerHost: null,
sharesAlarmQueueWithLocal: n => n == partner));
host.Tell(new RedundancyStateChanged(
[
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
new NodeRedundancyState(primary, RedundancyRole.Primary,
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
],
CorrelationId.NewId()));
AwaitAssert(() => view.Published.ShouldBe([expected]), duration: Timeout);
Sys.Stop(host);
}
}
// ---------------- helpers ----------------
/// <summary>
/// A view that remembers every published decision, so a test can assert on the actor's
/// <i>action</i> rather than on a state the seed already satisfies.
/// </summary>
private sealed class RecordingRoleView : IRedundancyRoleView
{
private readonly List<bool> _published = [];
private readonly Lock _gate = new();
private readonly RedundancyRoleView _inner = new();
/// <summary>Every value published so far, in order.</summary>
public IReadOnlyList<bool> Published
{
get { lock (_gate) return _published.ToArray(); }
}
public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory;
public void Publish(bool shouldDrainAlarmHistory)
{
lock (_gate) _published.Add(shouldDrainAlarmHistory);
_inner.Publish(shouldDrainAlarmHistory);
}
}
private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount)
{
var view = new RecordingRoleView();
SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role));
return view;
}
private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount)
{
var view = new RecordingRoleView();
SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary));
return view;
}
/// <summary>The peer this node replicates with — the only node it may ever stand down for.</summary>
private const string PeerHost = "the-peer";
private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) =>
Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(),
TestNode,
ackRouter: null,
driverMemberCountProvider: () => driverMemberCount,
redundancyRoleView: view,
replicationPeerHost: PeerHost));
/// <summary>
/// A snapshot of a real pair: this node in <paramref name="role"/>, plus a peer holding the
/// other half. The peer matters — a Secondary only stands down when a Primary exists, so a
/// single-entry snapshot would silently change what these cases test.
/// </summary>
private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) =>
new(
[
new NodeRedundancyState(node, role,
IsClusterLeader: role == RedundancyRole.Primary,
IsDriverPrimary: role == RedundancyRole.Primary,
AsOfUtc: DateTime.UtcNow),
new NodeRedundancyState(
NodeId.Parse($"{PeerHost}:4053"),
role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary,
IsClusterLeader: role != RedundancyRole.Primary,
IsDriverPrimary: role != RedundancyRole.Primary,
AsOfUtc: DateTime.UtcNow),
],
CorrelationId.NewId());
}