fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3)

- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is re-seeded for the referenced paths) so retired tags no longer accrete across deploys; OnUpstreamChange no longer writes the cache after Dispose. - R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s (DisposeDrainTimeout), then cancels it through a shutdown token every DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer hang host shutdown. A save that completes inside the bound is still drained (positive control test). - R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at Warning (naming the alarm) instead of swallowing it. - R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step with every dead-letter/revive/purge path and resyncs it with the queued counter on the same cadence; GetStatus no longer runs a COUNT(*) scan. - R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one summary entry naming the dropped count when the window rolls) behind ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink (key = script name). Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:17:30 -04:00
parent 2e9160317f
commit 6cbf763719
10 changed files with 683 additions and 32 deletions
@@ -0,0 +1,123 @@
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+/// <summary>
+///     Per-source fixed-window rate limit for the script-log fan-out: the first
+///     <see cref="DefaultLimitPerWindow"/> entries a source emits in a <see cref="DefaultWindow"/> are
+///     admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first
+///     admission after the window rolls — so the caller can emit one summary line in the dropped entries'
+///     place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot
+///     shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the
+///     cluster bus / main log.
+/// </summary>
+/// <remarks>
+///     Used by <see cref="ScriptLogTopicSink"/> (key = script + virtual-tag + alarm identity),
+///     <see cref="ScriptLogCompanionSink"/> (key = script name) and the virtual-tag actor's per-failure
+///     log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources
+///     are swept every <see cref="SweepEvery"/> admissions so a churning tag set cannot grow the table
+///     without bound.
+/// </remarks>
+public sealed class ScriptLogRateLimiter
+{
+    /// <summary>Entries admitted per source per window by default: 20.</summary>
+    public const int DefaultLimitPerWindow = 20;
+
+    /// <summary>The default window: one minute.</summary>
+    public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1);
+
+    private const int SweepEvery = 256;
+
+    private readonly Dictionary<string, Source> _sources = new(StringComparer.Ordinal);
+    private readonly int _limit;
+    private readonly TimeSpan _window;
+    private readonly Func<DateTime> _clock;
+    private int _admitsSinceSweep;
+
+    /// <summary>Initializes a limiter.</summary>
+    /// <param name="limitPerWindow">Entries admitted per source per window; values below 1 are treated as 1.</param>
+    /// <param name="window">The fixed window; null ⇒ <see cref="DefaultWindow"/>.</param>
+    /// <param name="clock">UTC clock; null ⇒ <see cref="DateTime.UtcNow"/>.</param>
+    public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func<DateTime>? clock = null)
+    {
+        _limit = Math.Max(1, limitPerWindow);
+        _window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow;
+        _clock = clock ?? (static () => DateTime.UtcNow);
+    }
+
+    /// <summary>The window length.</summary>
+    public TimeSpan Window => _window;
+
+    /// <summary>
+    ///     Ask whether <paramref name="sourceKey"/>'s next entry may be emitted. When the window has rolled
+    ///     since the last call, <see cref="ScriptLogAdmission.SuppressedInPreviousWindow"/> carries the
+    ///     number of entries dropped in the window that just closed (zero otherwise) — the caller emits one
+    ///     summary for them BEFORE the admitted entry.
+    /// </summary>
+    /// <param name="sourceKey">Identity of the emitting script instance.</param>
+    /// <returns>The admission decision.</returns>
+    public ScriptLogAdmission Admit(string sourceKey)
+    {
+        ArgumentNullException.ThrowIfNull(sourceKey);
+        var now = _clock();
+        lock (_sources)
+        {
+            if (++_admitsSinceSweep >= SweepEvery)
+            {
+                _admitsSinceSweep = 0;
+                Sweep(now);
+            }
+
+            if (!_sources.TryGetValue(sourceKey, out var source))
+            {
+                source = new Source { WindowStart = now };
+                _sources[sourceKey] = source;
+            }
+
+            long summary = 0;
+            if (now - source.WindowStart >= _window)
+            {
+                summary = source.Suppressed;
+                source.WindowStart = now;
+                source.Admitted = 0;
+                source.Suppressed = 0;
+            }
+
+            source.LastSeen = now;
+            if (source.Admitted < _limit)
+            {
+                source.Admitted++;
+                return new ScriptLogAdmission(true, summary);
+            }
+
+            source.Suppressed++;
+            return new ScriptLogAdmission(false, summary);
+        }
+    }
+
+    private void Sweep(DateTime now)
+    {
+        var idle = now - (_window + _window);
+        List<string>? dead = null;
+        foreach (var (key, source) in _sources)
+        {
+            // A source that still has a pending summary is kept: its next entry must report the count.
+            if (source.LastSeen < idle && source.Suppressed == 0)
+                (dead ??= []).Add(key);
+        }
+        if (dead is null) return;
+        foreach (var key in dead) _sources.Remove(key);
+    }
+
+    private sealed class Source
+    {
+        public DateTime WindowStart;
+        public DateTime LastSeen;
+        public int Admitted;
+        public long Suppressed;
+    }
+}
+
+/// <summary>Result of <see cref="ScriptLogRateLimiter.Admit"/>.</summary>
+/// <param name="Admitted">Whether the entry may be emitted.</param>
+/// <param name="SuppressedInPreviousWindow">Entries dropped in the window that just closed — non-zero at
+/// most once per window roll; the caller emits one summary line for them.</param>
+public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow);