3be8a1217e
The drain gate's 'is my queue-sharing peer Primary?' used only the host of
LocalDb:Replication:PeerAddress, which the passive half (ApiKey +
SyncListenPort) does not set - so as Secondary it kept draining beside its
Primary partner and every alarm event reached the historian twice.
DriverHostActor takes a sharesAlarmQueueWithLocal predicate used when no
peer host is known; WithOtOpcUaRuntimeActors builds it
(BuildAlarmQueuePeerPredicate) for a SharesRowsWithPeer node with no
PeerAddress as 'a member of my own cluster-{ClusterId} mesh other than
me'. A node sharing no rows, or with no mesh role, gets none and keeps
draining. AlarmHistorian.md / Redundancy.md / pair-replication runbook
updated.
Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
249 lines
11 KiB
C#
249 lines
11 KiB
C#
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// The bridge that carries the Primary-gate decision out of the actor and into
|
|
/// <see cref="IRedundancyRoleView"/>, where the alarm store-and-forward drain can read it.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive
|
|
/// <see cref="RedundancyStateChanged"/> itself — but it must be Primary-scoped, because
|
|
/// the queue it drains replicates to the pair peer and two draining nodes would deliver
|
|
/// every alarm event twice.
|
|
/// </para>
|
|
/// <para>
|
|
/// These cases deliberately DIVERGE from <see cref="DriverHostActorPrimaryGateTests"/> in the
|
|
/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate
|
|
/// history row is cheap and already contemplated by the at-least-once contract; a silently
|
|
/// un-drained queue is not. See <c>AlarmHistoryDrainGatePolicyTests</c>.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test");
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>
|
|
/// Before any snapshot arrives the view must read open, not closed.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// A deployment that runs no redundancy never publishes here at all. Defaulting closed
|
|
/// would silently stop its alarm history forever — the exact failure the drain gate exists
|
|
/// to avoid causing.
|
|
/// </remarks>
|
|
[Fact]
|
|
public void An_unpublished_view_reads_as_primary()
|
|
{
|
|
new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>A Secondary closes the view — but only because the snapshot also names a Primary.</summary>
|
|
[Fact]
|
|
public void Secondary_role_closes_the_view_when_a_primary_exists()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2);
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of
|
|
/// this node's rows, so deferring to it would deliver them never.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged(
|
|
[
|
|
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
|
|
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
|
|
// A Primary DOES exist — in another pair. It is not this node's replication peer.
|
|
new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary,
|
|
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId()));
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
[Fact]
|
|
public void Primary_role_opens_the_view()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2);
|
|
|
|
// Published, not current — `true` is the seed. See the note in the unknown-role theory.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Unknown role ⇒ the view stays <b>open</b>, even with driver peers present — the deliberate
|
|
/// divergence from the write gate's default-DENY.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// This case previously asserted the opposite, on the stated goal of mirroring the write
|
|
/// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs
|
|
/// four driver nodes in one cluster with no <c>Redundancy</c> section, so every role was
|
|
/// unknown and the driver count was 4 — and every node logged "Historian drain suspended",
|
|
/// including the two unpaired site-b nodes with no peer and no replication. Alarm history
|
|
/// drained nowhere, where before Phase 2 it drained fine.
|
|
/// </para>
|
|
/// <para>
|
|
/// See <c>AlarmHistoryDrainGatePolicyTests</c> for why the asymmetry of the two error costs
|
|
/// makes fail-open correct here and fail-closed correct for device writes.
|
|
/// </para>
|
|
/// </remarks>
|
|
[Theory]
|
|
[InlineData(1)]
|
|
[InlineData(2)]
|
|
[InlineData(4)]
|
|
public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount)
|
|
{
|
|
// A snapshot that never names this node is how the role stays unknown in practice — the
|
|
// documented identity-mismatch shape, not a contrived case.
|
|
var view = SpawnAndTellForeignSnapshot(driverMemberCount);
|
|
|
|
// Assert on what was PUBLISHED, never on the current value. `true` is also the seeded
|
|
// default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the
|
|
// first poll — before the actor has processed the snapshot — and therefore passes just as
|
|
// happily when the publish is wrong or absent. Verified: with the old write-gate verdict
|
|
// restored, the current-value form stayed green and only this form goes red.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>A demotion must move the view, not just an initial promotion.</summary>
|
|
[Fact]
|
|
public void A_demotion_closes_a_previously_open_view()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
var host = SpawnHost(view, driverMemberCount: 2);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Primary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Secondary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review R6 MINOR-4 — the PASSIVE half of a pair dials nobody (no <c>PeerAddress</c>, so no peer host) but
|
|
/// holds the same replicated rows, so as Secondary it must stand down for its Primary partner too, which it
|
|
/// identifies through the queue-sharing predicate (its mesh partner). Before the fix it published
|
|
/// <c>true</c> here and both halves drained. A Primary the predicate does NOT name is the control: the
|
|
/// passive half keeps draining, so the deny comes from identifying the partner, not from being Secondary.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_passive_secondary_closes_the_view_for_its_mesh_partner_and_only_for_it()
|
|
{
|
|
var partner = NodeId.Parse("site-a-1:4053");
|
|
foreach (var (primary, expected) in new[] { (partner, false), (NodeId.Parse("site-b-1:4053"), true) })
|
|
{
|
|
var view = new RecordingRoleView();
|
|
var host = Sys.ActorOf(DriverHostActor.Props(
|
|
NewInMemoryDbFactory(),
|
|
TestNode,
|
|
ackRouter: null,
|
|
driverMemberCountProvider: () => 2,
|
|
redundancyRoleView: view,
|
|
replicationPeerHost: null,
|
|
sharesAlarmQueueWithLocal: n => n == partner));
|
|
|
|
host.Tell(new RedundancyStateChanged(
|
|
[
|
|
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
|
|
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
|
|
new NodeRedundancyState(primary, RedundancyRole.Primary,
|
|
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId()));
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([expected]), duration: Timeout);
|
|
Sys.Stop(host);
|
|
}
|
|
}
|
|
|
|
// ---------------- helpers ----------------
|
|
|
|
/// <summary>
|
|
/// A view that remembers every published decision, so a test can assert on the actor's
|
|
/// <i>action</i> rather than on a state the seed already satisfies.
|
|
/// </summary>
|
|
private sealed class RecordingRoleView : IRedundancyRoleView
|
|
{
|
|
private readonly List<bool> _published = [];
|
|
private readonly Lock _gate = new();
|
|
private readonly RedundancyRoleView _inner = new();
|
|
|
|
/// <summary>Every value published so far, in order.</summary>
|
|
public IReadOnlyList<bool> Published
|
|
{
|
|
get { lock (_gate) return _published.ToArray(); }
|
|
}
|
|
|
|
public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory;
|
|
|
|
public void Publish(bool shouldDrainAlarmHistory)
|
|
{
|
|
lock (_gate) _published.Add(shouldDrainAlarmHistory);
|
|
_inner.Publish(shouldDrainAlarmHistory);
|
|
}
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role));
|
|
return view;
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary));
|
|
return view;
|
|
}
|
|
|
|
/// <summary>The peer this node replicates with — the only node it may ever stand down for.</summary>
|
|
private const string PeerHost = "the-peer";
|
|
|
|
private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) =>
|
|
Sys.ActorOf(DriverHostActor.Props(
|
|
NewInMemoryDbFactory(),
|
|
TestNode,
|
|
ackRouter: null,
|
|
driverMemberCountProvider: () => driverMemberCount,
|
|
redundancyRoleView: view,
|
|
replicationPeerHost: PeerHost));
|
|
|
|
/// <summary>
|
|
/// A snapshot of a real pair: this node in <paramref name="role"/>, plus a peer holding the
|
|
/// other half. The peer matters — a Secondary only stands down when a Primary exists, so a
|
|
/// single-entry snapshot would silently change what these cases test.
|
|
/// </summary>
|
|
private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) =>
|
|
new(
|
|
[
|
|
new NodeRedundancyState(node, role,
|
|
IsClusterLeader: role == RedundancyRole.Primary,
|
|
IsDriverPrimary: role == RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
new NodeRedundancyState(
|
|
NodeId.Parse($"{PeerHost}:4053"),
|
|
role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary,
|
|
IsClusterLeader: role != RedundancyRole.Primary,
|
|
IsDriverPrimary: role != RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId());
|
|
}
|