feat(localdb): alarm S&F on LocalDb with primary-gated drain (cutover)

Moves the alarm store-and-forward buffer out of its own alarm-historian.db
and into the node's consolidated LocalDb, where it replicates to the
redundant pair peer. A node that dies holding undelivered alarm history no
longer takes it to the grave.

Tasks 2 and 3 land together, as the plan anticipated. They are not separable:
the gate is a constructor argument of the rewritten sink, and a commit that
replicated the queue without gating the drain would be a commit in which both
nodes of a pair deliver every alarm event, continuously.

That drain gate is the load-bearing part of the change, and the recon
explains why it is new work rather than a refinement. Exactly-once delivery
across a pair is enforced today on the ENQUEUE side, by
HistorianAdapterActor: only the Primary enqueues, so the Secondary's queue is
empty and its ungated drain has nothing to send. Replicating the table
destroys that invariant.

The gate is a Func<bool> the caller supplies, because the drain runs on a
timer the sink owns rather than on a mailbox, and Core.AlarmHistorian cannot
reference PrimaryGatePolicy in Runtime. Runtime supplies it via a new
IRedundancyRoleView singleton that DriverHostActor publishes its Primary-gate
verdict to -- the same verdict the inbound-write and native-ack gates use, so
there is no second notion of am-I-the-Primary to drift.

Two failure modes are deliberately closed:

- The view is seeded OPEN, matching the policy's own answer for an unknown
  role with no driver peer. A deployment that runs no redundancy never
  publishes to it, and defaulting closed would silently stop its alarm
  history forever.

- A gate that throws is read as not-now, never as permission, and a closed
  gate reports the new HistorianDrainState.NotPrimary rather than Idle. A
  Secondary's rising queue is supposed to look different from a stalled
  drain, and if BOTH nodes report NotPrimary the pair is misconfigured and
  says so instead of quietly filling toward the capacity ceiling.

Row ids are a hash of the payload rather than fresh GUIDs. Both adapters
accept the same fanned transition in the window before the first redundancy
snapshot arrives, and under last-writer-wins an equal key converges those two
accepts into one row instead of duplicating them.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -24,6 +24,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
 using ZB.MOM.WW.OtOpcUa.OpcUaServer;
 using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
+using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
@@ -252,6 +253,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
     private readonly Func<int>? _driverMemberCountProvider;
 
+    /// <summary>Where the Primary-gate decision is published for consumers that live outside a mailbox —
+    /// today, the alarm store-and-forward drain. Null on nodes that do not wire one.</summary>
+    private readonly IRedundancyRoleView? _redundancyRoleView;
+
     /// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
     /// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
     private bool _warnedSnapshotMissingLocalNode;
@@ -366,7 +371,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         IActorRef? scriptedAlarmHostOverride = null,
         IDriverCapabilityInvokerFactory? invokerFactory = null,
         Func<int>? driverMemberCountProvider = null,
-        IDeploymentArtifactCache? deploymentArtifactCache = null) =>
+        IDeploymentArtifactCache? deploymentArtifactCache = null,
+        IRedundancyRoleView? redundancyRoleView = null) =>
         // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
         // expression tree. Six IActorRef? parameters and several interface-typed ones mean a
         // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -376,7 +382,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
             dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
             healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
             loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
-            deploymentArtifactCache));
+            deploymentArtifactCache, redundancyRoleView));
 
     /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
     /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -426,9 +432,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         IActorRef? scriptedAlarmHostOverride = null,
         IDriverCapabilityInvokerFactory? invokerFactory = null,
         Func<int>? driverMemberCountProvider = null,
-        IDeploymentArtifactCache? deploymentArtifactCache = null)
+        IDeploymentArtifactCache? deploymentArtifactCache = null,
+        IRedundancyRoleView? redundancyRoleView = null)
     {
         _deploymentArtifactCache = deploymentArtifactCache;
+        _redundancyRoleView = redundancyRoleView;
         _dbFactory = dbFactory;
         _localNode = localNode;
         _coordinatorOverride = coordinator;
@@ -1476,19 +1484,23 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         if (local is not null)
         {
             _localRole = local.Role;
-            return;
         }
-
-        // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
-        // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
-        // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
-        if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
+        else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
         {
+            // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
+            // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
+            // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
             _warnedSnapshotMissingLocalNode = true;
             _log.Warning(
                 "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
                 _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
         }
+
+        // Publish on EVERY snapshot, including ones that leave the cached role untouched: the other
+        // half of the decision is the driver member count, which moves with cluster membership and
+        // not with this message. Snapshots are re-published on a heartbeat, so a membership change
+        // is picked up within one heartbeat rather than waiting for a role change that may never come.
+        _redundancyRoleView?.Publish(ShouldServiceAsPrimary());
     }
 
     private void Stale()

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.IO;
 using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
 
 namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
@@ -7,11 +6,12 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 /// <summary>
 ///     Binds the <c>AlarmHistorian</c> configuration section that gates the durable
 ///     store-and-forward alarm sink. When <see cref="Enabled"/> is <c>true</c>,
-///     <c>AddAlarmHistorian</c> registers a <c>SqliteStoreAndForwardSink</c> (draining to the
+///     <c>AddAlarmHistorian</c> registers a <c>LocalDbStoreAndForwardSink</c> (draining to the
 ///     gateway alarm writer supplied by the Host) in place of the
 ///     <c>NullAlarmHistorianSink</c> default; otherwise the Null default survives. This section
-///     supplies only the <see cref="Enabled"/> gate and the SQLite store-and-forward knobs — the
-///     downstream connection (endpoint/key/TLS) is sourced from the <c>ServerHistorian</c> section.
+///     supplies only the <see cref="Enabled"/> gate and the store-and-forward knobs — the
+///     downstream connection (endpoint/key/TLS) is sourced from the <c>ServerHistorian</c> section,
+///     and the queue's storage location is the node's consolidated <c>LocalDb:Path</c> database.
 /// </summary>
 public sealed class AlarmHistorianOptions
 {
@@ -24,9 +24,6 @@ public sealed class AlarmHistorianOptions
     /// </summary>
     public bool Enabled { get; init; }
 
-    /// <summary>Filesystem path to the local SQLite store-and-forward queue database.</summary>
-    public string DatabasePath { get; init; } = "alarm-historian.db";
-
     /// <summary>Maximum number of queued rows the drain worker forwards in a single batch.</summary>
     public int BatchSize { get; init; } = 100;
 
@@ -34,15 +31,15 @@ public sealed class AlarmHistorianOptions
     public int DrainIntervalSeconds { get; init; } = 5;
 
     /// <summary>Maximum queued rows before the sink evicts the oldest. Defaults to 1,000,000
-    /// (matches <c>SqliteStoreAndForwardSink</c>'s <c>DefaultCapacity</c>).</summary>
-    public long Capacity { get; init; } = SqliteStoreAndForwardSink.DefaultCapacity;
+    /// (matches <c>LocalDbStoreAndForwardSink</c>'s <c>DefaultCapacity</c>).</summary>
+    public long Capacity { get; init; } = LocalDbStoreAndForwardSink.DefaultCapacity;
 
     /// <summary>Days to retain dead-lettered rows before purge. Defaults to 30.</summary>
     public int DeadLetterRetentionDays { get; init; } = 30;
 
     /// <summary>Maximum delivery attempts before a perpetually-retrying (poison) row is dead-lettered.
-    /// Defaults to 10 (matches <c>SqliteStoreAndForwardSink</c>'s <c>DefaultMaxAttempts</c>).</summary>
-    public int MaxAttempts { get; init; } = SqliteStoreAndForwardSink.DefaultMaxAttempts;
+    /// Defaults to 10 (matches <c>LocalDbStoreAndForwardSink</c>'s <c>DefaultMaxAttempts</c>).</summary>
+    public int MaxAttempts { get; init; } = LocalDbStoreAndForwardSink.DefaultMaxAttempts;
 
     /// <summary>Returns operator-facing misconfiguration warnings for an <c>Enabled</c> historian
     /// (empty when disabled or correctly configured). Pure — the registration logs each entry.</summary>
@@ -51,8 +48,6 @@ public sealed class AlarmHistorianOptions
     {
         var warnings = new List<string>();
         if (!Enabled) return warnings;
-        if (!Path.IsPathRooted(DatabasePath))
-            warnings.Add($"AlarmHistorian:DatabasePath '{DatabasePath}' is relative — it resolves against the process working directory (e.g. System32 for a Windows service). Set an absolute path.");
         if (DrainIntervalSeconds <= 0)
             warnings.Add($"AlarmHistorian:DrainIntervalSeconds is {DrainIntervalSeconds} — must be > 0; the drain timer will throw or spin at startup.");
         if (Capacity <= 0)

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs

@@ -0,0 +1,46 @@
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
+
+/// <summary>
+///     A singleton snapshot of the Primary-gate decision, so code that lives outside an actor can
+///     ask "should this node be servicing Primary-only work right now?".
+/// </summary>
+/// <remarks>
+///     <para>
+///         Exists for the alarm store-and-forward drain. That drain runs on a timer owned by the
+///         sink, not on an actor mailbox, so it cannot receive <c>RedundancyStateChanged</c>
+///         directly — but it must be Primary-scoped, because the queue it drains replicates to the
+///         pair peer and two draining nodes would deliver every event twice.
+///     </para>
+///     <para>
+///         Deliberately publishes the <i>decision</i> rather than the role and member count that
+///         produced it, so <see cref="PrimaryGatePolicy"/> stays the single place that decides.
+///     </para>
+/// </remarks>
+public interface IRedundancyRoleView
+{
+    /// <summary>Whether this node should service Primary-only work right now.</summary>
+    bool ShouldServiceAsPrimary { get; }
+
+    /// <summary>Records a fresh decision. Called by <c>DriverHostActor</c> on every redundancy snapshot.</summary>
+    /// <param name="shouldServiceAsPrimary">The decision <see cref="PrimaryGatePolicy"/> produced.</param>
+    void Publish(bool shouldServiceAsPrimary);
+}
+
+/// <summary>Thread-safe <see cref="IRedundancyRoleView"/> backed by a volatile field.</summary>
+public sealed class RedundancyRoleView : IRedundancyRoleView
+{
+    // Seeded with the decision for "role unknown, no driver peer" rather than a bare false. An
+    // unpublished view and a node with no peer are genuinely the same situation, and they must
+    // behave the same: a deployment that runs no redundancy at all never publishes here, and
+    // defaulting closed would silently stop its alarm history forever.
+    private volatile bool _shouldServiceAsPrimary =
+        PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 0);
+
+    /// <inheritdoc/>
+    public bool ShouldServiceAsPrimary => _shouldServiceAsPrimary;
+
+    /// <inheritdoc/>
+    public void Publish(bool shouldServiceAsPrimary) => _shouldServiceAsPrimary = shouldServiceAsPrimary;
+}

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs

@@ -20,7 +20,9 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
 using ZB.MOM.WW.OtOpcUa.Runtime.Health;
 using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
+using ZB.MOM.WW.LocalDb;
 
 namespace ZB.MOM.WW.OtOpcUa.Runtime;
 
@@ -65,12 +67,23 @@ public static class ServiceCollectionExtensions
 
     /// <summary>
     ///     Config-gated durable alarm-historian sink. When the <c>AlarmHistorian</c> section has
-    ///     <c>Enabled=true</c>, registers a <see cref="SqliteStoreAndForwardSink"/> (draining via the
+    ///     <c>Enabled=true</c>, registers a <see cref="LocalDbStoreAndForwardSink"/> (draining via the
     ///     <paramref name="writerFactory"/>-supplied writer) as the <see cref="IAlarmHistorianSink"/>,
     ///     overriding the <see cref="NullAlarmHistorianSink"/> default. Otherwise a no-op (Null stays).
     ///     The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be
     ///     supplied by the Host, which is the only project that references it.
     /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         The queue lives in the node's consolidated <see cref="ILocalDb"/>, which replicates it
+    ///         to the redundant pair peer — so undelivered alarm history survives losing the node
+    ///         holding it. That is also why the drain is gated on <see cref="IRedundancyRoleView"/>:
+    ///         the Secondary holds a full replica of the Primary's queue, and an ungated drain there
+    ///         would re-deliver every event. Both services are resolved from the provider, so a
+    ///         deployment that enables this section without registering LocalDb fails loudly at
+    ///         resolution rather than silently buffering to nowhere.
+    ///     </para>
+    /// </remarks>
     /// <param name="services">The service collection to register with.</param>
     /// <param name="configuration">The configuration carrying the <c>AlarmHistorian</c> section.</param>
     /// <param name="writerFactory">
@@ -87,21 +100,28 @@ public static class ServiceCollectionExtensions
         if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime
 
         foreach (var warning in opts.Validate())
-            Serilog.Log.Logger.ForContext<SqliteStoreAndForwardSink>().Warning("Historian config: {HistorianConfigWarning}", warning);
+            Serilog.Log.Logger.ForContext<LocalDbStoreAndForwardSink>().Warning("Historian config: {HistorianConfigWarning}", warning);
+
+        // The view is a plain singleton so it exists even on nodes whose DriverHostActor never
+        // publishes to it — an unpublished view reads as "no peer, drain", which is the correct
+        // posture for a deployment that runs no redundancy at all.
+        services.TryAddSingleton<IRedundancyRoleView, RedundancyRoleView>();
 
         services.AddSingleton<IAlarmHistorianSink>(sp =>
         {
-            // SqliteStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging).
+            // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging).
             // Resolve it off the host's configured static logger so the drain worker's WARN/INFO
             // lines land in the same sinks as the rest of the process.
-            var sink = new SqliteStoreAndForwardSink(
-                opts.DatabasePath,
+            var roleView = sp.GetRequiredService<IRedundancyRoleView>();
+            var sink = new LocalDbStoreAndForwardSink(
+                sp.GetRequiredService<ILocalDb>(),
                 writerFactory(opts, sp),
-                Serilog.Log.Logger.ForContext<SqliteStoreAndForwardSink>(),
+                Serilog.Log.Logger.ForContext<LocalDbStoreAndForwardSink>(),
                 batchSize: opts.BatchSize,
                 capacity: opts.Capacity,
                 deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays),
-                maxAttempts: opts.MaxAttempts);
+                maxAttempts: opts.MaxAttempts,
+                drainGate: () => roleView.ShouldServiceAsPrimary);
             sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds));
             return sink;
         });
@@ -227,6 +247,11 @@ public static class ServiceCollectionExtensions
             // harnesses) rather than given a null-object, so DriverHostActor skips caching outright
             // instead of pretending to cache into a sink that drops everything.
             var deploymentArtifactCache = resolver.GetService<IDeploymentArtifactCache>();
+            // Where this actor publishes its Primary-gate verdict for the alarm store-and-forward
+            // drain, which runs on a timer and so cannot read RedundancyStateChanged itself.
+            // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which
+            // case there is nothing downstream to inform.
+            var redundancyRoleView = resolver.GetService<IRedundancyRoleView>();
             // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in
             // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses,
             // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host.
@@ -345,7 +370,8 @@ public static class ServiceCollectionExtensions
                     loggerFactory: loggerFactory,
                     scriptRootLogger: scriptRootLogger,
                     invokerFactory: invokerFactory,
-                    deploymentArtifactCache: deploymentArtifactCache),
+                    deploymentArtifactCache: deploymentArtifactCache,
+                    redundancyRoleView: redundancyRoleView),
                 DriverHostActorName);
             registry.Register<DriverHostActorKey>(driverHost);