feat(localdb): alarm S&F on LocalDb with primary-gated drain (cutover)

Moves the alarm store-and-forward buffer out of its own alarm-historian.db
and into the node's consolidated LocalDb, where it replicates to the
redundant pair peer. A node that dies holding undelivered alarm history no
longer takes it to the grave.

Tasks 2 and 3 land together, as the plan anticipated. They are not separable:
the gate is a constructor argument of the rewritten sink, and a commit that
replicated the queue without gating the drain would be a commit in which both
nodes of a pair deliver every alarm event, continuously.

That drain gate is the load-bearing part of the change, and the recon
explains why it is new work rather than a refinement. Exactly-once delivery
across a pair is enforced today on the ENQUEUE side, by
HistorianAdapterActor: only the Primary enqueues, so the Secondary's queue is
empty and its ungated drain has nothing to send. Replicating the table
destroys that invariant.

The gate is a Func<bool> the caller supplies, because the drain runs on a
timer the sink owns rather than on a mailbox, and Core.AlarmHistorian cannot
reference PrimaryGatePolicy in Runtime. Runtime supplies it via a new
IRedundancyRoleView singleton that DriverHostActor publishes its Primary-gate
verdict to -- the same verdict the inbound-write and native-ack gates use, so
there is no second notion of am-I-the-Primary to drift.

Two failure modes are deliberately closed:

- The view is seeded OPEN, matching the policy's own answer for an unknown
  role with no driver peer. A deployment that runs no redundancy never
  publishes to it, and defaulting closed would silently stop its alarm
  history forever.

- A gate that throws is read as not-now, never as permission, and a closed
  gate reports the new HistorianDrainState.NotPrimary rather than Idle. A
  Secondary's rising queue is supposed to look different from a stalled
  drain, and if BOTH nodes report NotPrimary the pair is misconfigured and
  says so instead of quietly filling toward the capacity ceiling.

Row ids are a hash of the payload rather than fresh GUIDs. Both adapters
accept the same fanned transition in the window before the first redundancy
snapshot arrives, and under last-writer-wins an equal key converges those two
accepts into one row instead of duplicating them.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-21 04:13:00 -04:00
parent 8a9cb40a72
commit 71379816e7
14 changed files with 804 additions and 365 deletions
@@ -24,6 +24,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
@@ -252,6 +253,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
private readonly Func<int>? _driverMemberCountProvider;
/// <summary>Where the Primary-gate decision is published for consumers that live outside a mailbox —
/// today, the alarm store-and-forward drain. Null on nodes that do not wire one.</summary>
private readonly IRedundancyRoleView? _redundancyRoleView;
/// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
/// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
private bool _warnedSnapshotMissingLocalNode;
@@ -366,7 +371,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null) =>
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -376,7 +382,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache));
deploymentArtifactCache, redundancyRoleView));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -426,9 +432,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null,
IDeploymentArtifactCache? deploymentArtifactCache = null)
IDeploymentArtifactCache? deploymentArtifactCache = null,
IRedundancyRoleView? redundancyRoleView = null)
{
_deploymentArtifactCache = deploymentArtifactCache;
_redundancyRoleView = redundancyRoleView;
_dbFactory = dbFactory;
_localNode = localNode;
_coordinatorOverride = coordinator;
@@ -1476,19 +1484,23 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
if (local is not null)
{
_localRole = local.Role;
return;
}
// S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
// Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
// identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
{
// S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
// Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
// identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
_warnedSnapshotMissingLocalNode = true;
_log.Warning(
"DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
_localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
}
// Publish on EVERY snapshot, including ones that leave the cached role untouched: the other
// half of the decision is the driver member count, which moves with cluster membership and
// not with this message. Snapshots are re-published on a heartbeat, so a membership change
// is picked up within one heartbeat rather than waiting for a role change that may never come.
_redundancyRoleView?.Publish(ShouldServiceAsPrimary());
}
private void Stale()