feat(localdb): alarm S&F on LocalDb with primary-gated drain (cutover)

Moves the alarm store-and-forward buffer out of its own alarm-historian.db
and into the node's consolidated LocalDb, where it replicates to the
redundant pair peer. A node that dies holding undelivered alarm history no
longer takes it to the grave.

Tasks 2 and 3 land together, as the plan anticipated. They are not separable:
the gate is a constructor argument of the rewritten sink, and a commit that
replicated the queue without gating the drain would be a commit in which both
nodes of a pair deliver every alarm event, continuously.

That drain gate is the load-bearing part of the change, and the recon
explains why it is new work rather than a refinement. Exactly-once delivery
across a pair is enforced today on the ENQUEUE side, by
HistorianAdapterActor: only the Primary enqueues, so the Secondary's queue is
empty and its ungated drain has nothing to send. Replicating the table
destroys that invariant.

The gate is a Func<bool> the caller supplies, because the drain runs on a
timer the sink owns rather than on a mailbox, and Core.AlarmHistorian cannot
reference PrimaryGatePolicy in Runtime. Runtime supplies it via a new
IRedundancyRoleView singleton that DriverHostActor publishes its Primary-gate
verdict to -- the same verdict the inbound-write and native-ack gates use, so
there is no second notion of am-I-the-Primary to drift.

Two failure modes are deliberately closed:

- The view is seeded OPEN, matching the policy's own answer for an unknown
  role with no driver peer. A deployment that runs no redundancy never
  publishes to it, and defaulting closed would silently stop its alarm
  history forever.

- A gate that throws is read as not-now, never as permission, and a closed
  gate reports the new HistorianDrainState.NotPrimary rather than Idle. A
  Secondary's rising queue is supposed to look different from a stalled
  drain, and if BOTH nodes report NotPrimary the pair is misconfigured and
  says so instead of quietly filling toward the capacity ceiling.

Row ids are a hash of the payload rather than fresh GUIDs. Both adapters
accept the same fanned transition in the window before the first redundancy
snapshot arrives, and under last-writer-wins an equal key converges those two
accepts into one row instead of duplicating them.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

src/Server/ZB.MOM.WW.OtOpcUa.Host/appsettings.json

@@ -52,9 +52,8 @@
     "MaxBackoffSeconds": 30
   },
   "AlarmHistorian": {
-    "_comment": "Durable SQLite store-and-forward alarm sink. Drains alarm events to the ServerHistorian gateway's SendEvent path; the downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section.",
+    "_comment": "Durable store-and-forward alarm sink. Buffers into the node's consolidated LocalDb (see the LocalDb section) so the queue replicates to the redundant pair peer, and drains alarm events to the ServerHistorian gateway's SendEvent path from whichever node holds the Primary role. The downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section.",
     "Enabled": false,
-    "DatabasePath": "alarm-historian.db",
     "DrainIntervalSeconds": 5,
     "Capacity": 1000000,
     "DeadLetterRetentionDays": 30

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -24,6 +24,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
 using ZB.MOM.WW.OtOpcUa.OpcUaServer;
 using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
+using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 using CommonsNodeId = ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId;
@@ -252,6 +253,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
     private readonly Func<int>? _driverMemberCountProvider;
 
+    /// <summary>Where the Primary-gate decision is published for consumers that live outside a mailbox —
+    /// today, the alarm store-and-forward drain. Null on nodes that do not wire one.</summary>
+    private readonly IRedundancyRoleView? _redundancyRoleView;
+
     /// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
     /// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
     private bool _warnedSnapshotMissingLocalNode;
@@ -366,7 +371,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         IActorRef? scriptedAlarmHostOverride = null,
         IDriverCapabilityInvokerFactory? invokerFactory = null,
         Func<int>? driverMemberCountProvider = null,
-        IDeploymentArtifactCache? deploymentArtifactCache = null) =>
+        IDeploymentArtifactCache? deploymentArtifactCache = null,
+        IRedundancyRoleView? redundancyRoleView = null) =>
         // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
         // expression tree. Six IActorRef? parameters and several interface-typed ones mean a
         // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -376,7 +382,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
             dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
             healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
             loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
-            deploymentArtifactCache));
+            deploymentArtifactCache, redundancyRoleView));
 
     /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
     /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -426,9 +432,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         IActorRef? scriptedAlarmHostOverride = null,
         IDriverCapabilityInvokerFactory? invokerFactory = null,
         Func<int>? driverMemberCountProvider = null,
-        IDeploymentArtifactCache? deploymentArtifactCache = null)
+        IDeploymentArtifactCache? deploymentArtifactCache = null,
+        IRedundancyRoleView? redundancyRoleView = null)
     {
         _deploymentArtifactCache = deploymentArtifactCache;
+        _redundancyRoleView = redundancyRoleView;
         _dbFactory = dbFactory;
         _localNode = localNode;
         _coordinatorOverride = coordinator;
@@ -1476,19 +1484,23 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         if (local is not null)
         {
             _localRole = local.Role;
-            return;
         }
-
-        // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
-        // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
-        // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
-        if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
+        else if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
         {
+            // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
+            // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
+            // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
             _warnedSnapshotMissingLocalNode = true;
             _log.Warning(
                 "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
                 _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
         }
+
+        // Publish on EVERY snapshot, including ones that leave the cached role untouched: the other
+        // half of the decision is the driver member count, which moves with cluster membership and
+        // not with this message. Snapshots are re-published on a heartbeat, so a membership change
+        // is picked up within one heartbeat rather than waiting for a role change that may never come.
+        _redundancyRoleView?.Publish(ShouldServiceAsPrimary());
     }
 
     private void Stale()

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/AlarmHistorianOptions.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.IO;
 using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
 
 namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
@@ -7,11 +6,12 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 /// <summary>
 ///     Binds the <c>AlarmHistorian</c> configuration section that gates the durable
 ///     store-and-forward alarm sink. When <see cref="Enabled"/> is <c>true</c>,
-///     <c>AddAlarmHistorian</c> registers a <c>SqliteStoreAndForwardSink</c> (draining to the
+///     <c>AddAlarmHistorian</c> registers a <c>LocalDbStoreAndForwardSink</c> (draining to the
 ///     gateway alarm writer supplied by the Host) in place of the
 ///     <c>NullAlarmHistorianSink</c> default; otherwise the Null default survives. This section
-///     supplies only the <see cref="Enabled"/> gate and the SQLite store-and-forward knobs — the
-///     downstream connection (endpoint/key/TLS) is sourced from the <c>ServerHistorian</c> section.
+///     supplies only the <see cref="Enabled"/> gate and the store-and-forward knobs — the
+///     downstream connection (endpoint/key/TLS) is sourced from the <c>ServerHistorian</c> section,
+///     and the queue's storage location is the node's consolidated <c>LocalDb:Path</c> database.
 /// </summary>
 public sealed class AlarmHistorianOptions
 {
@@ -24,9 +24,6 @@ public sealed class AlarmHistorianOptions
     /// </summary>
     public bool Enabled { get; init; }
 
-    /// <summary>Filesystem path to the local SQLite store-and-forward queue database.</summary>
-    public string DatabasePath { get; init; } = "alarm-historian.db";
-
     /// <summary>Maximum number of queued rows the drain worker forwards in a single batch.</summary>
     public int BatchSize { get; init; } = 100;
 
@@ -34,15 +31,15 @@ public sealed class AlarmHistorianOptions
     public int DrainIntervalSeconds { get; init; } = 5;
 
     /// <summary>Maximum queued rows before the sink evicts the oldest. Defaults to 1,000,000
-    /// (matches <c>SqliteStoreAndForwardSink</c>'s <c>DefaultCapacity</c>).</summary>
-    public long Capacity { get; init; } = SqliteStoreAndForwardSink.DefaultCapacity;
+    /// (matches <c>LocalDbStoreAndForwardSink</c>'s <c>DefaultCapacity</c>).</summary>
+    public long Capacity { get; init; } = LocalDbStoreAndForwardSink.DefaultCapacity;
 
     /// <summary>Days to retain dead-lettered rows before purge. Defaults to 30.</summary>
     public int DeadLetterRetentionDays { get; init; } = 30;
 
     /// <summary>Maximum delivery attempts before a perpetually-retrying (poison) row is dead-lettered.
-    /// Defaults to 10 (matches <c>SqliteStoreAndForwardSink</c>'s <c>DefaultMaxAttempts</c>).</summary>
-    public int MaxAttempts { get; init; } = SqliteStoreAndForwardSink.DefaultMaxAttempts;
+    /// Defaults to 10 (matches <c>LocalDbStoreAndForwardSink</c>'s <c>DefaultMaxAttempts</c>).</summary>
+    public int MaxAttempts { get; init; } = LocalDbStoreAndForwardSink.DefaultMaxAttempts;
 
     /// <summary>Returns operator-facing misconfiguration warnings for an <c>Enabled</c> historian
     /// (empty when disabled or correctly configured). Pure — the registration logs each entry.</summary>
@@ -51,8 +48,6 @@ public sealed class AlarmHistorianOptions
     {
         var warnings = new List<string>();
         if (!Enabled) return warnings;
-        if (!Path.IsPathRooted(DatabasePath))
-            warnings.Add($"AlarmHistorian:DatabasePath '{DatabasePath}' is relative — it resolves against the process working directory (e.g. System32 for a Windows service). Set an absolute path.");
         if (DrainIntervalSeconds <= 0)
             warnings.Add($"AlarmHistorian:DrainIntervalSeconds is {DrainIntervalSeconds} — must be > 0; the drain timer will throw or spin at startup.");
         if (Capacity <= 0)

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Redundancy/IRedundancyRoleView.cs

@@ -0,0 +1,46 @@
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
+
+/// <summary>
+///     A singleton snapshot of the Primary-gate decision, so code that lives outside an actor can
+///     ask "should this node be servicing Primary-only work right now?".
+/// </summary>
+/// <remarks>
+///     <para>
+///         Exists for the alarm store-and-forward drain. That drain runs on a timer owned by the
+///         sink, not on an actor mailbox, so it cannot receive <c>RedundancyStateChanged</c>
+///         directly — but it must be Primary-scoped, because the queue it drains replicates to the
+///         pair peer and two draining nodes would deliver every event twice.
+///     </para>
+///     <para>
+///         Deliberately publishes the <i>decision</i> rather than the role and member count that
+///         produced it, so <see cref="PrimaryGatePolicy"/> stays the single place that decides.
+///     </para>
+/// </remarks>
+public interface IRedundancyRoleView
+{
+    /// <summary>Whether this node should service Primary-only work right now.</summary>
+    bool ShouldServiceAsPrimary { get; }
+
+    /// <summary>Records a fresh decision. Called by <c>DriverHostActor</c> on every redundancy snapshot.</summary>
+    /// <param name="shouldServiceAsPrimary">The decision <see cref="PrimaryGatePolicy"/> produced.</param>
+    void Publish(bool shouldServiceAsPrimary);
+}
+
+/// <summary>Thread-safe <see cref="IRedundancyRoleView"/> backed by a volatile field.</summary>
+public sealed class RedundancyRoleView : IRedundancyRoleView
+{
+    // Seeded with the decision for "role unknown, no driver peer" rather than a bare false. An
+    // unpublished view and a node with no peer are genuinely the same situation, and they must
+    // behave the same: a deployment that runs no redundancy at all never publishes here, and
+    // defaulting closed would silently stop its alarm history forever.
+    private volatile bool _shouldServiceAsPrimary =
+        PrimaryGatePolicy.ShouldServiceAsPrimary(localRole: null, driverMemberCount: 0);
+
+    /// <inheritdoc/>
+    public bool ShouldServiceAsPrimary => _shouldServiceAsPrimary;
+
+    /// <inheritdoc/>
+    public void Publish(bool shouldServiceAsPrimary) => _shouldServiceAsPrimary = shouldServiceAsPrimary;
+}

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs

@@ -20,7 +20,9 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
 using ZB.MOM.WW.OtOpcUa.Runtime.Health;
 using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
+using ZB.MOM.WW.LocalDb;
 
 namespace ZB.MOM.WW.OtOpcUa.Runtime;
 
@@ -65,12 +67,23 @@ public static class ServiceCollectionExtensions
 
     /// <summary>
     ///     Config-gated durable alarm-historian sink. When the <c>AlarmHistorian</c> section has
-    ///     <c>Enabled=true</c>, registers a <see cref="SqliteStoreAndForwardSink"/> (draining via the
+    ///     <c>Enabled=true</c>, registers a <see cref="LocalDbStoreAndForwardSink"/> (draining via the
     ///     <paramref name="writerFactory"/>-supplied writer) as the <see cref="IAlarmHistorianSink"/>,
     ///     overriding the <see cref="NullAlarmHistorianSink"/> default. Otherwise a no-op (Null stays).
     ///     The writer is injected so the durable downstream (the HistorianGateway alarm writer) can be
     ///     supplied by the Host, which is the only project that references it.
     /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         The queue lives in the node's consolidated <see cref="ILocalDb"/>, which replicates it
+    ///         to the redundant pair peer — so undelivered alarm history survives losing the node
+    ///         holding it. That is also why the drain is gated on <see cref="IRedundancyRoleView"/>:
+    ///         the Secondary holds a full replica of the Primary's queue, and an ungated drain there
+    ///         would re-deliver every event. Both services are resolved from the provider, so a
+    ///         deployment that enables this section without registering LocalDb fails loudly at
+    ///         resolution rather than silently buffering to nowhere.
+    ///     </para>
+    /// </remarks>
     /// <param name="services">The service collection to register with.</param>
     /// <param name="configuration">The configuration carrying the <c>AlarmHistorian</c> section.</param>
     /// <param name="writerFactory">
@@ -87,21 +100,28 @@ public static class ServiceCollectionExtensions
         if (opts is not { Enabled: true }) return services; // leave the Null default from AddOtOpcUaRuntime
 
         foreach (var warning in opts.Validate())
-            Serilog.Log.Logger.ForContext<SqliteStoreAndForwardSink>().Warning("Historian config: {HistorianConfigWarning}", warning);
+            Serilog.Log.Logger.ForContext<LocalDbStoreAndForwardSink>().Warning("Historian config: {HistorianConfigWarning}", warning);
+
+        // The view is a plain singleton so it exists even on nodes whose DriverHostActor never
+        // publishes to it — an unpublished view reads as "no peer, drain", which is the correct
+        // posture for a deployment that runs no redundancy at all.
+        services.TryAddSingleton<IRedundancyRoleView, RedundancyRoleView>();
 
         services.AddSingleton<IAlarmHistorianSink>(sp =>
         {
-            // SqliteStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging).
+            // LocalDbStoreAndForwardSink takes a Serilog ILogger (not Microsoft.Extensions.Logging).
             // Resolve it off the host's configured static logger so the drain worker's WARN/INFO
             // lines land in the same sinks as the rest of the process.
-            var sink = new SqliteStoreAndForwardSink(
-                opts.DatabasePath,
+            var roleView = sp.GetRequiredService<IRedundancyRoleView>();
+            var sink = new LocalDbStoreAndForwardSink(
+                sp.GetRequiredService<ILocalDb>(),
                 writerFactory(opts, sp),
-                Serilog.Log.Logger.ForContext<SqliteStoreAndForwardSink>(),
+                Serilog.Log.Logger.ForContext<LocalDbStoreAndForwardSink>(),
                 batchSize: opts.BatchSize,
                 capacity: opts.Capacity,
                 deadLetterRetention: TimeSpan.FromDays(opts.DeadLetterRetentionDays),
-                maxAttempts: opts.MaxAttempts);
+                maxAttempts: opts.MaxAttempts,
+                drainGate: () => roleView.ShouldServiceAsPrimary);
             sink.StartDrainLoop(TimeSpan.FromSeconds(opts.DrainIntervalSeconds));
             return sink;
         });
@@ -227,6 +247,11 @@ public static class ServiceCollectionExtensions
             // harnesses) rather than given a null-object, so DriverHostActor skips caching outright
             // instead of pretending to cache into a sink that drops everything.
             var deploymentArtifactCache = resolver.GetService<IDeploymentArtifactCache>();
+            // Where this actor publishes its Primary-gate verdict for the alarm store-and-forward
+            // drain, which runs on a timer and so cannot read RedundancyStateChanged itself.
+            // Registered by AddAlarmHistorian; absent when no durable sink is configured, in which
+            // case there is nothing downstream to inform.
+            var redundancyRoleView = resolver.GetService<IRedundancyRoleView>();
             // Root script logger backs the ScriptedAlarm host's engine + script logging. Registered in
             // Host DI inside the hasDriver block; may be absent in some role configs / test harnesses,
             // in which case the DriverHostActor gracefully skips spawning the ScriptedAlarm host.
@@ -345,7 +370,8 @@ public static class ServiceCollectionExtensions
                     loggerFactory: loggerFactory,
                     scriptRootLogger: scriptRootLogger,
                     invokerFactory: invokerFactory,
-                    deploymentArtifactCache: deploymentArtifactCache),
+                    deploymentArtifactCache: deploymentArtifactCache,
+                    redundancyRoleView: redundancyRoleView),
                 DriverHostActorName);
             registry.Register<DriverHostActorKey>(driverHost);