fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3)

- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is
  re-seeded for the referenced paths) so retired tags no longer accrete across
  deploys; OnUpstreamChange no longer writes the cache after Dispose.
- R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s
  (DisposeDrainTimeout), then cancels it through a shutdown token every
  DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives
  one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer
  hang host shutdown. A save that completes inside the bound is still drained
  (positive control test).
- R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at
  Warning (naming the alarm) instead of swallowing it.
- R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step
  with every dead-letter/revive/purge path and resyncs it with the queued
  counter on the same cadence; GetStatus no longer runs a COUNT(*) scan.
- R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one
  summary entry naming the dropped count when the window rolls) behind
  ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink
  (key = script name).

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs

@@ -170,6 +170,24 @@ public sealed class ScriptedAlarmEngine : IDisposable
         => _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null;
     private readonly ConcurrentDictionary<string, DataValueSnapshot> _valueCache
         = new(StringComparer.Ordinal);
+
+    /// <summary>Test hook — the number of paths currently held in the value cache.</summary>
+    internal int ValueCacheCountForTest => _valueCache.Count;
+
+    /// <summary>Test hook — whether <paramref name="path"/> is currently held in the value cache.</summary>
+    internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path);
+
+    /// <summary>
+    ///     How long <see cref="Dispose"/> waits for in-flight background work (a re-evaluation or shelving
+    ///     check mid-<c>SaveAsync</c>) before cancelling it through <see cref="_shutdownCts"/> and
+    ///     returning. 5 s, like <c>PollGroupEngine</c>'s drain; a hung store must not hang host shutdown.
+    /// </summary>
+    internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5);
+
+    /// <summary>Cancelled by <see cref="Dispose"/> once the drain bound expires — every background task
+    /// (<c>DrainDirtyAsync</c>, <c>ShelvingCheckAsync</c>) runs on its token, so a store call that honours
+    /// cancellation returns instead of outliving the engine.</summary>
+    private readonly CancellationTokenSource _shutdownCts = new();
     private readonly Dictionary<string, HashSet<string>> _alarmsReferencing
         = new(StringComparer.Ordinal); // tag path -> alarm ids
 
@@ -281,6 +299,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
             UnsubscribeFromUpstream();
             _alarms.Clear();
             _alarmsReferencing.Clear();
+            // The value cache is re-seeded below for exactly the paths this generation references. It
+            // used to survive reloads untouched, so every path a retired alarm had ever referenced stayed
+            // cached forever — a slow leak over many deploys with a churning tag set.
+            _valueCache.Clear();
             // Drop the prior generation's per-alarm scratch buffers — definitions may
             // have changed (different Inputs, different Logger), so any reuse would be
             // unsafe.
@@ -573,8 +595,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
     /// <param name="value">The new data value snapshot.</param>
     internal void OnUpstreamChange(string path, DataValueSnapshot value)
     {
+        if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
         _valueCache[path] = value;
-        if (_disposed) return; // don't queue new work against a disposing engine
         if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
 
         bool startDrain;
@@ -589,7 +611,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
 
         // Started outside the lock: DrainDirtyAsync runs synchronously up to its
         // first real await, and nothing it does should happen under _dirtyLock.
-        if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None));
+        if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token));
     }
 
     /// <summary>
@@ -732,6 +754,11 @@ public sealed class ScriptedAlarmEngine : IDisposable
             }
             finally { _evalGate.Release(); }
         }
+        catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
+        {
+            // Dispose cut the drain short after its bound — expected, not a failure.
+            return;
+        }
         catch (Exception ex)
         {
             _engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed");
@@ -1048,7 +1075,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
     {
         if (_disposed) return;
         var ids = _alarms.Keys.ToArray();
-        TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None));
+        TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token));
     }
 
     /// <summary>
@@ -1187,6 +1214,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
             }
             finally { _evalGate.Release(); }
         }
+        catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
+        {
+            return; // Dispose cut the check short after its bound
+        }
         catch (Exception ex)
         {
             _engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed");
@@ -1227,11 +1258,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
         // flight can outlive the engine and write to a (possibly disposed) store
         // after Dispose() has returned. The tasks re-check _disposed after
         // acquiring the gate and bail out, but the await still has to complete.
+        // BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store
+        // (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that
+        // calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a
+        // store that honours cancellation returns promptly), log, and give them one short grace period.
         Task[] toAwait;
         lock (_inFlightLock) { toAwait = [.. _inFlight]; }
         if (toAwait.Length > 0)
         {
-            try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); }
+            var all = Task.WhenAll(toAwait);
+            try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); }
+            catch (TimeoutException)
+            {
+                _engineLogger.Warning(
+                    "ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)",
+                    toAwait.Length, DisposeDrainTimeout);
+                _shutdownCts.Cancel();
+                try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); }
+                catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ }
+            }
             catch (Exception ex)
             {
                 // Background task failures already logged inside ReevaluateAsync /