50b55ee4b9
RedundancyStateActor derived the driver Primary from ClusterState.RoleLeader("driver").
Akka offers two different notions of a "first" member and they are not the same
one: role leader is the lowest-ADDRESSED Up member (host, then port), while
ClusterSingletonManager places singletons on the OLDEST — lowest up-number.
They agree on a freshly-formed cluster, which is why every existing test passed.
They diverge after any restart: the restarted node re-joins as the youngest while
keeping its address, so if it holds the lower address it becomes role leader while
the singletons stay put. The snapshot would then name a Primary that is not
hosting the work, and every Primary-gated surface follows it — inbound device
writes, native-alarm acks, the fleet-wide alerts emit, and the alarm-history
drain would all enable on the wrong node while the node actually running the
singletons stayed gated off.
BuildSnapshot now selects the oldest Up member carrying the driver role, matching
singleton placement. Leaving members are excluded: a node handing its singletons
over must not be named Primary.
NodeRedundancyState.IsRoleLeaderForDriver is renamed IsDriverPrimary, and
NodeHealthInputs.IsDriverRoleLeader likewise. Keeping the old names would have
left the wire contract asserting a derivation the code no longer uses — the same
drift that made this defect invisible.
Proven by a real two-node cluster rather than a mock. RedundancyPrimaryElectionTests
binds the first-joining node to the HIGHER port, so oldest and lowest-address name
different nodes, and includes a fixture assertion that the divergence actually
occurred — without it the real assertion could pass for the wrong reason. Positive
control: restoring the RoleLeader derivation turns exactly the two election tests
red while the fixture check stays green.
Runtime.Tests 440 passed, ControlPlane.Tests 82, Cluster.Tests 36.
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
212 lines
9.4 KiB
C#
212 lines
9.4 KiB
C#
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// The bridge that carries the Primary-gate decision out of the actor and into
|
|
/// <see cref="IRedundancyRoleView"/>, where the alarm store-and-forward drain can read it.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The drain runs on a timer owned by the sink, not on a mailbox, so it cannot receive
|
|
/// <see cref="RedundancyStateChanged"/> itself — but it must be Primary-scoped, because
|
|
/// the queue it drains replicates to the pair peer and two draining nodes would deliver
|
|
/// every alarm event twice.
|
|
/// </para>
|
|
/// <para>
|
|
/// These cases deliberately DIVERGE from <see cref="DriverHostActorPrimaryGateTests"/> in the
|
|
/// unknown-role case: the write gate denies when a peer may exist, this one allows. A duplicate
|
|
/// history row is cheap and already contemplated by the at-least-once contract; a silently
|
|
/// un-drained queue is not. See <c>AlarmHistoryDrainGatePolicyTests</c>.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-roleview-test");
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>
|
|
/// Before any snapshot arrives the view must read open, not closed.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// A deployment that runs no redundancy never publishes here at all. Defaulting closed
|
|
/// would silently stop its alarm history forever — the exact failure the drain gate exists
|
|
/// to avoid causing.
|
|
/// </remarks>
|
|
[Fact]
|
|
public void An_unpublished_view_reads_as_primary()
|
|
{
|
|
new RedundancyRoleView().ShouldDrainAlarmHistory.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>A Secondary closes the view — but only because the snapshot also names a Primary.</summary>
|
|
[Fact]
|
|
public void Secondary_role_closes_the_view_when_a_primary_exists()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Secondary, driverMemberCount: 2);
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([false]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A Secondary whose Primary is some OTHER pair's node keeps draining: that node holds none of
|
|
/// this node's rows, so deferring to it would deliver them never.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Secondary_role_keeps_the_view_open_when_the_primary_is_not_its_peer()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount: 4).Tell(new RedundancyStateChanged(
|
|
[
|
|
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
|
|
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
|
|
// A Primary DOES exist — in another pair. It is not this node's replication peer.
|
|
new NodeRedundancyState(NodeId.Parse("some-other-pair:4053"), RedundancyRole.Primary,
|
|
IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId()));
|
|
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
[Fact]
|
|
public void Primary_role_opens_the_view()
|
|
{
|
|
var view = SpawnAndTellRole(RedundancyRole.Primary, driverMemberCount: 2);
|
|
|
|
// Published, not current — `true` is the seed. See the note in the unknown-role theory.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Unknown role ⇒ the view stays <b>open</b>, even with driver peers present — the deliberate
|
|
/// divergence from the write gate's default-DENY.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// This case previously asserted the opposite, on the stated goal of mirroring the write
|
|
/// gate. The Phase 2 live gate showed that goal to be wrong for this consumer: the rig runs
|
|
/// four driver nodes in one cluster with no <c>Redundancy</c> section, so every role was
|
|
/// unknown and the driver count was 4 — and every node logged "Historian drain suspended",
|
|
/// including the two unpaired site-b nodes with no peer and no replication. Alarm history
|
|
/// drained nowhere, where before Phase 2 it drained fine.
|
|
/// </para>
|
|
/// <para>
|
|
/// See <c>AlarmHistoryDrainGatePolicyTests</c> for why the asymmetry of the two error costs
|
|
/// makes fail-open correct here and fail-closed correct for device writes.
|
|
/// </para>
|
|
/// </remarks>
|
|
[Theory]
|
|
[InlineData(1)]
|
|
[InlineData(2)]
|
|
[InlineData(4)]
|
|
public void Unknown_role_leaves_the_view_open_whatever_the_peer_count(int driverMemberCount)
|
|
{
|
|
// A snapshot that never names this node is how the role stays unknown in practice — the
|
|
// documented identity-mismatch shape, not a contrived case.
|
|
var view = SpawnAndTellForeignSnapshot(driverMemberCount);
|
|
|
|
// Assert on what was PUBLISHED, never on the current value. `true` is also the seeded
|
|
// default, so `AwaitAssert(() => view.ShouldDrainAlarmHistory.ShouldBeTrue())` passes at the
|
|
// first poll — before the actor has processed the snapshot — and therefore passes just as
|
|
// happily when the publish is wrong or absent. Verified: with the old write-gate verdict
|
|
// restored, the current-value form stayed green and only this form goes red.
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
}
|
|
|
|
/// <summary>A demotion must move the view, not just an initial promotion.</summary>
|
|
[Fact]
|
|
public void A_demotion_closes_a_previously_open_view()
|
|
{
|
|
var view = new RecordingRoleView();
|
|
var host = SpawnHost(view, driverMemberCount: 2);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Primary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true]), duration: Timeout);
|
|
|
|
host.Tell(Snapshot(TestNode, RedundancyRole.Secondary));
|
|
AwaitAssert(() => view.Published.ShouldBe([true, false]), duration: Timeout);
|
|
}
|
|
|
|
// ---------------- helpers ----------------
|
|
|
|
/// <summary>
|
|
/// A view that remembers every published decision, so a test can assert on the actor's
|
|
/// <i>action</i> rather than on a state the seed already satisfies.
|
|
/// </summary>
|
|
private sealed class RecordingRoleView : IRedundancyRoleView
|
|
{
|
|
private readonly List<bool> _published = [];
|
|
private readonly Lock _gate = new();
|
|
private readonly RedundancyRoleView _inner = new();
|
|
|
|
/// <summary>Every value published so far, in order.</summary>
|
|
public IReadOnlyList<bool> Published
|
|
{
|
|
get { lock (_gate) return _published.ToArray(); }
|
|
}
|
|
|
|
public bool ShouldDrainAlarmHistory => _inner.ShouldDrainAlarmHistory;
|
|
|
|
public void Publish(bool shouldDrainAlarmHistory)
|
|
{
|
|
lock (_gate) _published.Add(shouldDrainAlarmHistory);
|
|
_inner.Publish(shouldDrainAlarmHistory);
|
|
}
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellRole(RedundancyRole role, int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(TestNode, role));
|
|
return view;
|
|
}
|
|
|
|
private RecordingRoleView SpawnAndTellForeignSnapshot(int driverMemberCount)
|
|
{
|
|
var view = new RecordingRoleView();
|
|
SpawnHost(view, driverMemberCount).Tell(Snapshot(NodeId.Parse("some-other-node"), RedundancyRole.Primary));
|
|
return view;
|
|
}
|
|
|
|
/// <summary>The peer this node replicates with — the only node it may ever stand down for.</summary>
|
|
private const string PeerHost = "the-peer";
|
|
|
|
private IActorRef SpawnHost(IRedundancyRoleView view, int driverMemberCount) =>
|
|
Sys.ActorOf(DriverHostActor.Props(
|
|
NewInMemoryDbFactory(),
|
|
TestNode,
|
|
coordinator: null,
|
|
driverMemberCountProvider: () => driverMemberCount,
|
|
redundancyRoleView: view,
|
|
replicationPeerHost: PeerHost));
|
|
|
|
/// <summary>
|
|
/// A snapshot of a real pair: this node in <paramref name="role"/>, plus a peer holding the
|
|
/// other half. The peer matters — a Secondary only stands down when a Primary exists, so a
|
|
/// single-entry snapshot would silently change what these cases test.
|
|
/// </summary>
|
|
private static RedundancyStateChanged Snapshot(NodeId node, RedundancyRole role) =>
|
|
new(
|
|
[
|
|
new NodeRedundancyState(node, role,
|
|
IsClusterLeader: role == RedundancyRole.Primary,
|
|
IsDriverPrimary: role == RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
new NodeRedundancyState(
|
|
NodeId.Parse($"{PeerHost}:4053"),
|
|
role == RedundancyRole.Primary ? RedundancyRole.Secondary : RedundancyRole.Primary,
|
|
IsClusterLeader: role != RedundancyRole.Primary,
|
|
IsDriverPrimary: role != RedundancyRole.Primary,
|
|
AsOfUtc: DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId());
|
|
}
|