fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3)

- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is
  re-seeded for the referenced paths) so retired tags no longer accrete across
  deploys; OnUpstreamChange no longer writes the cache after Dispose.
- R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s
  (DisposeDrainTimeout), then cancels it through a shutdown token every
  DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives
  one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer
  hang host shutdown. A save that completes inside the bound is still drained
  (positive control test).
- R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at
  Warning (naming the alarm) instead of swallowing it.
- R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step
  with every dead-letter/revive/purge path and resyncs it with the queued
  counter on the same cadence; GetStatus no longer runs a COUNT(*) scan.
- R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one
  summary entry naming the dropped count when the window rolls) behind
  ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink
  (key = script name).

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 10:17:30 -04:00
parent 2e9160317f
commit 6cbf763719
10 changed files with 683 additions and 32 deletions
@@ -92,6 +92,12 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
private bool? _lastGateDecision;
private long _queuedRowCount;
// Cached dead-letter depth (R05:7). GetStatus used to run a synchronous COUNT(*) table scan per
// call — one per /healthz probe and per AdminUI poll on the shared LocalDb file. Kept in step by
// every path that flips dead_lettered (drain outcomes, corrupt rows, RetryDeadLettered, retention
// purge) and resynced from storage on the same cadence as _queuedRowCount, so a row the pair peer
// replicates in is visible after the next resync rather than never.
private long _deadLetterRowCount;
// Probe counter — incremented every time we actually issue a real COUNT(*) for
// capacity enforcement. Public for test instrumentation only.
private long _capacityProbeCount;
@@ -145,9 +151,16 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
_clock = clock ?? (() => DateTime.UtcNow);
_drainGate = drainGate ?? (static () => true);
_queuedRowCount = ProbeQueuedRowCount();
_queuedRowCount = ProbeRowCount(deadLettered: false);
_deadLetterRowCount = ProbeRowCount(deadLettered: true);
}
/// <summary>Test-only: re-sync both cached counters (queued + dead-lettered) from storage — the same
/// resync the enqueue cadence drives every <see cref="ResyncEnqueueInterval"/> enqueues.</summary>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that completes once both counters match storage.</returns>
public Task DebugResyncCountersAsync(CancellationToken ct) => ResyncRowCountsAsync(ct);
/// <summary>
/// Start the background drain worker. Not started automatically so tests can
/// drive <see cref="DrainOnceAsync"/> deterministically.
@@ -260,7 +273,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
// has a second way to fall behind that no local code path can observe.
if (enqueuesSinceResync >= ResyncEnqueueInterval)
{
await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false);
await ResyncRowCountsAsync(ct).ConfigureAwait(false);
cached = Interlocked.Read(ref _queuedRowCount);
Interlocked.Exchange(ref _enqueuesSinceResync, 0);
}
@@ -273,22 +286,26 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
await EnforceCapacityAsync(ct).ConfigureAwait(false);
}
/// <summary>Synchronously query <c>COUNT(*)</c> of non-dead-lettered rows. Used at startup.</summary>
private long ProbeQueuedRowCount()
/// <summary>Synchronously query <c>COUNT(*)</c> of queued or dead-lettered rows. Used at startup.</summary>
private long ProbeRowCount(bool deadLettered)
{
Interlocked.Increment(ref _capacityProbeCount);
using var conn = _db.CreateConnection();
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0";
cmd.CommandText = deadLettered
? "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"
: "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0";
return (long)(cmd.ExecuteScalar() ?? 0L);
}
/// <summary>Re-sync the in-memory counter from storage (async path).</summary>
private async Task ResyncQueuedRowCountAsync(CancellationToken ct)
/// <summary>Re-sync both in-memory counters from storage (async path).</summary>
private async Task ResyncRowCountsAsync(CancellationToken ct)
{
Interlocked.Increment(ref _capacityProbeCount);
var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false);
Interlocked.Exchange(ref _queuedRowCount, live);
var dead = await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false);
Interlocked.Exchange(ref _deadLetterRowCount, dead);
}
private async Task<long> CountAsync(string predicate, CancellationToken ct)
@@ -356,6 +373,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
await corruptTx.CommitAsync(ct).ConfigureAwait(false);
}
Interlocked.Add(ref _queuedRowCount, -corruptIds.Count);
Interlocked.Add(ref _deadLetterRowCount, corruptIds.Count);
_logger.Warning(
"Dead-lettered {Count} historian queue row(s) with un-deserializable payload",
corruptIds.Count);
@@ -405,6 +423,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
}
int rowsLeavingQueue = 0;
int rowsDeadLettered = 0;
await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false))
{
for (var i = 0; i < outcomes.Count; i++)
@@ -423,6 +442,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
case HistorianWriteOutcome.PermanentFail:
await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false);
rowsLeavingQueue++;
rowsDeadLettered++;
break;
case HistorianWriteOutcome.RetryPlease:
// finding 002: cap retries so a perpetually-RetryPlease (poison)
@@ -434,6 +454,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
{
await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false);
rowsLeavingQueue++;
rowsDeadLettered++;
}
else
{
@@ -449,6 +470,8 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
// cap (finding 002) — keep the counter aligned.
if (rowsLeavingQueue > 0)
Interlocked.Add(ref _queuedRowCount, -rowsLeavingQueue);
if (rowsDeadLettered > 0)
Interlocked.Add(ref _deadLetterRowCount, rowsDeadLettered);
var acks = outcomes.Count(o => o == HistorianWriteOutcome.Ack);
lock (_statusLock)
@@ -523,13 +546,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
var queued = Interlocked.Read(ref _queuedRowCount);
if (queued < 0) queued = 0;
long deadlettered;
using (var conn = _db.CreateConnection())
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1";
deadlettered = (long)(cmd.ExecuteScalar() ?? 0L);
}
// Cached, never a live scan (R05:7) — see the field's remarks for the resync cadence.
var deadlettered = Interlocked.Read(ref _deadLetterRowCount);
if (deadlettered < 0) deadlettered = 0;
DateTime? lastDrain, lastSuccess;
string? lastError;
@@ -568,7 +587,11 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
var revived = cmd.ExecuteNonQuery();
// Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory
// counter aligned.
if (revived > 0) Interlocked.Add(ref _queuedRowCount, revived);
if (revived > 0)
{
Interlocked.Add(ref _queuedRowCount, revived);
Interlocked.Add(ref _deadLetterRowCount, -revived);
}
return revived;
}
@@ -646,8 +669,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
Interlocked.Increment(ref _capacityProbeCount);
var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false);
// Resync the in-memory counter while we have a fresh number.
// Resync the in-memory counters while we have a fresh number.
Interlocked.Exchange(ref _queuedRowCount, count);
Interlocked.Exchange(ref _deadLetterRowCount, await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false));
if (count < _capacity) return;
var toEvict = count - _capacity + 1;
@@ -684,7 +708,10 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
ct).ConfigureAwait(false);
if (purged > 0)
{
Interlocked.Add(ref _deadLetterRowCount, -purged);
_logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged);
}
}
private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1);
@@ -170,6 +170,24 @@ public sealed class ScriptedAlarmEngine : IDisposable
=> _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null;
private readonly ConcurrentDictionary<string, DataValueSnapshot> _valueCache
= new(StringComparer.Ordinal);
/// <summary>Test hook — the number of paths currently held in the value cache.</summary>
internal int ValueCacheCountForTest => _valueCache.Count;
/// <summary>Test hook — whether <paramref name="path"/> is currently held in the value cache.</summary>
internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path);
/// <summary>
/// How long <see cref="Dispose"/> waits for in-flight background work (a re-evaluation or shelving
/// check mid-<c>SaveAsync</c>) before cancelling it through <see cref="_shutdownCts"/> and
/// returning. 5 s, like <c>PollGroupEngine</c>'s drain; a hung store must not hang host shutdown.
/// </summary>
internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>Cancelled by <see cref="Dispose"/> once the drain bound expires — every background task
/// (<c>DrainDirtyAsync</c>, <c>ShelvingCheckAsync</c>) runs on its token, so a store call that honours
/// cancellation returns instead of outliving the engine.</summary>
private readonly CancellationTokenSource _shutdownCts = new();
private readonly Dictionary<string, HashSet<string>> _alarmsReferencing
= new(StringComparer.Ordinal); // tag path -> alarm ids
@@ -281,6 +299,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
UnsubscribeFromUpstream();
_alarms.Clear();
_alarmsReferencing.Clear();
// The value cache is re-seeded below for exactly the paths this generation references. It
// used to survive reloads untouched, so every path a retired alarm had ever referenced stayed
// cached forever — a slow leak over many deploys with a churning tag set.
_valueCache.Clear();
// Drop the prior generation's per-alarm scratch buffers — definitions may
// have changed (different Inputs, different Logger), so any reuse would be
// unsafe.
@@ -573,8 +595,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// <param name="value">The new data value snapshot.</param>
internal void OnUpstreamChange(string path, DataValueSnapshot value)
{
if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again
_valueCache[path] = value;
if (_disposed) return; // don't queue new work against a disposing engine
if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return;
bool startDrain;
@@ -589,7 +611,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
// Started outside the lock: DrainDirtyAsync runs synchronously up to its
// first real await, and nothing it does should happen under _dirtyLock.
if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None));
if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token));
}
/// <summary>
@@ -732,6 +754,11 @@ public sealed class ScriptedAlarmEngine : IDisposable
}
finally { _evalGate.Release(); }
}
catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
{
// Dispose cut the drain short after its bound — expected, not a failure.
return;
}
catch (Exception ex)
{
_engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed");
@@ -1048,7 +1075,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
{
if (_disposed) return;
var ids = _alarms.Keys.ToArray();
TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None));
TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token));
}
/// <summary>
@@ -1187,6 +1214,10 @@ public sealed class ScriptedAlarmEngine : IDisposable
}
finally { _evalGate.Release(); }
}
catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
{
return; // Dispose cut the check short after its bound
}
catch (Exception ex)
{
_engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed");
@@ -1227,11 +1258,25 @@ public sealed class ScriptedAlarmEngine : IDisposable
// flight can outlive the engine and write to a (possibly disposed) store
// after Dispose() has returned. The tasks re-check _disposed after
// acquiring the gate and bail out, but the await still has to complete.
// BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store
// (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that
// calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a
// store that honours cancellation returns promptly), log, and give them one short grace period.
Task[] toAwait;
lock (_inFlightLock) { toAwait = [.. _inFlight]; }
if (toAwait.Length > 0)
{
try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); }
var all = Task.WhenAll(toAwait);
try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); }
catch (TimeoutException)
{
_engineLogger.Warning(
"ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)",
toAwait.Length, DisposeDrainTimeout);
_shutdownCts.Cancel();
try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); }
catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ }
}
catch (Exception ex)
{
// Background task failures already logged inside ReevaluateAsync /
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using Serilog;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
@@ -23,19 +24,27 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
/// adapter doesn't maintain per-subscription state beyond the filter set — it
/// checks each emission against every live subscription.
/// </para>
/// <para>
/// A throwing <see cref="OnAlarmEvent"/> subscriber is logged at Warning (naming the alarm) and
/// does not crash the adapter — the same posture as the engine's own <c>FireEvent</c>. It used to
/// be swallowed silently: a lost alarm event with no trace.
/// </para>
/// </remarks>
public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable
{
private readonly ScriptedAlarmEngine _engine;
private readonly ILogger _logger;
private readonly ConcurrentDictionary<string, Subscription> _subscriptions
= new(StringComparer.Ordinal);
private bool _disposed;
/// <summary>Initializes a new instance of the <see cref="ScriptedAlarmSource"/> class.</summary>
/// <param name="engine">The scripted alarm engine to expose.</param>
public ScriptedAlarmSource(ScriptedAlarmEngine engine)
/// <param name="logger">Where a throwing subscriber is reported; null ⇒ Serilog's global logger.</param>
public ScriptedAlarmSource(ScriptedAlarmEngine engine, ILogger? logger = null)
{
_engine = engine ?? throw new ArgumentNullException(nameof(engine));
_logger = logger ?? Log.Logger;
_engine.OnEvent += OnEngineEvent;
}
@@ -98,7 +107,13 @@ public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable
Severity: evt.Severity,
SourceTimestampUtc: evt.TimestampUtc);
try { OnAlarmEvent?.Invoke(this, payload); }
catch { /* subscriber exceptions don't crash the adapter */ }
catch (Exception ex)
{
// Subscriber exceptions don't crash the adapter — but they are not silent either: a lost
// alarm event must leave a trace the operator can find.
_logger.Warning(ex, "ScriptedAlarmSource OnAlarmEvent subscriber threw for {AlarmId} ({Emission})",
evt.AlarmId, evt.Emission);
}
}
}
@@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// Original exception (if any) is attached so the main log's diagnostics keep
/// the full stack trace.
/// </para>
/// <para>
/// <b>Rate-limited per script.</b> A predicate that throws on every push would otherwise mirror
/// one Warning per push into the main log; a <see cref="ScriptLogRateLimiter"/> keyed by script
/// name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The
/// scripts-*.log file sink is unaffected and keeps the full stream.
/// </para>
/// </remarks>
public sealed class ScriptLogCompanionSink : ILogEventSink
{
private readonly ILogger _mainLogger;
private readonly LogEventLevel _minMirrorLevel;
private readonly ScriptLogRateLimiter _limiter;
/// <summary>Initializes a new instance of the ScriptLogCompanionSink.</summary>
/// <param name="mainLogger">The companion logger to mirror events to.</param>
/// <param name="minMirrorLevel">The minimum log level to mirror (default: Error).</param>
public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error)
/// <param name="limiter">Per-script rate limiter; null ⇒ the default 20 per minute.</param>
public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error,
ScriptLogRateLimiter? limiter = null)
{
_mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger));
_minMirrorLevel = minMirrorLevel;
_limiter = limiter ?? new ScriptLogRateLimiter();
}
/// <summary>Emits a log event to the companion logger if it meets the minimum level.</summary>
/// <summary>Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget.</summary>
/// <param name="logEvent">The log event to emit.</param>
public void Emit(LogEvent logEvent)
{
@@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink
scriptName = s;
}
var admission = _limiter.Admit(scriptName);
if (admission.SuppressedInPreviousWindow > 0)
{
_mainLogger.Warning(
"[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log",
scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds);
}
if (!admission.Admitted) return;
var rendered = logEvent.RenderMessage();
if (logEvent.Exception is not null)
{
@@ -0,0 +1,123 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// <summary>
/// Per-source fixed-window rate limit for the script-log fan-out: the first
/// <see cref="DefaultLimitPerWindow"/> entries a source emits in a <see cref="DefaultWindow"/> are
/// admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first
/// admission after the window rolls — so the caller can emit one summary line in the dropped entries'
/// place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot
/// shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the
/// cluster bus / main log.
/// </summary>
/// <remarks>
/// Used by <see cref="ScriptLogTopicSink"/> (key = script + virtual-tag + alarm identity),
/// <see cref="ScriptLogCompanionSink"/> (key = script name) and the virtual-tag actor's per-failure
/// log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources
/// are swept every <see cref="SweepEvery"/> admissions so a churning tag set cannot grow the table
/// without bound.
/// </remarks>
public sealed class ScriptLogRateLimiter
{
/// <summary>Entries admitted per source per window by default: 20.</summary>
public const int DefaultLimitPerWindow = 20;
/// <summary>The default window: one minute.</summary>
public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1);
private const int SweepEvery = 256;
private readonly Dictionary<string, Source> _sources = new(StringComparer.Ordinal);
private readonly int _limit;
private readonly TimeSpan _window;
private readonly Func<DateTime> _clock;
private int _admitsSinceSweep;
/// <summary>Initializes a limiter.</summary>
/// <param name="limitPerWindow">Entries admitted per source per window; values below 1 are treated as 1.</param>
/// <param name="window">The fixed window; null ⇒ <see cref="DefaultWindow"/>.</param>
/// <param name="clock">UTC clock; null ⇒ <see cref="DateTime.UtcNow"/>.</param>
public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func<DateTime>? clock = null)
{
_limit = Math.Max(1, limitPerWindow);
_window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow;
_clock = clock ?? (static () => DateTime.UtcNow);
}
/// <summary>The window length.</summary>
public TimeSpan Window => _window;
/// <summary>
/// Ask whether <paramref name="sourceKey"/>'s next entry may be emitted. When the window has rolled
/// since the last call, <see cref="ScriptLogAdmission.SuppressedInPreviousWindow"/> carries the
/// number of entries dropped in the window that just closed (zero otherwise) — the caller emits one
/// summary for them BEFORE the admitted entry.
/// </summary>
/// <param name="sourceKey">Identity of the emitting script instance.</param>
/// <returns>The admission decision.</returns>
public ScriptLogAdmission Admit(string sourceKey)
{
ArgumentNullException.ThrowIfNull(sourceKey);
var now = _clock();
lock (_sources)
{
if (++_admitsSinceSweep >= SweepEvery)
{
_admitsSinceSweep = 0;
Sweep(now);
}
if (!_sources.TryGetValue(sourceKey, out var source))
{
source = new Source { WindowStart = now };
_sources[sourceKey] = source;
}
long summary = 0;
if (now - source.WindowStart >= _window)
{
summary = source.Suppressed;
source.WindowStart = now;
source.Admitted = 0;
source.Suppressed = 0;
}
source.LastSeen = now;
if (source.Admitted < _limit)
{
source.Admitted++;
return new ScriptLogAdmission(true, summary);
}
source.Suppressed++;
return new ScriptLogAdmission(false, summary);
}
}
private void Sweep(DateTime now)
{
var idle = now - (_window + _window);
List<string>? dead = null;
foreach (var (key, source) in _sources)
{
// A source that still has a pending summary is kept: its next entry must report the count.
if (source.LastSeen < idle && source.Suppressed == 0)
(dead ??= []).Add(key);
}
if (dead is null) return;
foreach (var key in dead) _sources.Remove(key);
}
private sealed class Source
{
public DateTime WindowStart;
public DateTime LastSeen;
public int Admitted;
public long Suppressed;
}
}
/// <summary>Result of <see cref="ScriptLogRateLimiter.Admit"/>.</summary>
/// <param name="Admitted">Whether the entry may be emitted.</param>
/// <param name="SuppressedInPreviousWindow">Entries dropped in the window that just closed — non-zero at
/// most once per window roll; the caller emits one summary line for them.</param>
public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow);
@@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// property so pipelines that pre-date this sink continue to work correctly. If
/// neither property is present <c>ScriptId</c> defaults to <c>"unknown"</c>.
/// </para>
/// <para>
/// <b>Rate-limited per source.</b> A script that logs at Information on every evaluation of a 1 Hz
/// input put one rendered string per second per script onto the cluster bus; 200 such scripts shed
/// everyone's entries out of the bounded telemetry channels. A <see cref="ScriptLogRateLimiter"/>
/// keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per
/// source and publishes one summary entry naming the dropped count when the window rolls.
/// </para>
/// </remarks>
public sealed class ScriptLogTopicSink : ILogEventSink
{
private readonly IScriptLogPublisher _publisher;
private readonly LogEventLevel _min;
private readonly ScriptLogRateLimiter _limiter;
/// <summary>Initializes a new instance of the <see cref="ScriptLogTopicSink"/> class.</summary>
/// <param name="publisher">The publisher that routes entries to the cluster bus. Must not be <c>null</c>.</param>
/// <param name="min">Minimum log level to forward. Events below this level are discarded (default: <see cref="LogEventLevel.Information"/>).</param>
/// <param name="limiter">Per-source rate limiter; null ⇒ the default 20 per minute.</param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="publisher"/> is <c>null</c>.</exception>
public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information)
public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information,
ScriptLogRateLimiter? limiter = null)
{
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_min = min;
_limiter = limiter ?? new ScriptLogRateLimiter();
}
/// <summary>
/// Converts the <paramref name="logEvent"/> to a <see cref="ScriptLogEntry"/> and
/// publishes it, unless the event is below the configured minimum level.
/// publishes it, unless the event is below the configured minimum level or the source is over its
/// per-minute budget.
/// </summary>
/// <param name="logEvent">The Serilog event to process. Silently ignored when <c>null</c>.</param>
public void Emit(LogEvent logEvent)
@@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink
? s
: null;
var scriptId = P(ScriptLoggerFactory.ScriptIdProperty)
?? P(ScriptLoggerFactory.ScriptNameProperty)
?? "unknown";
var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty);
var alarmId = P(ScriptLoggerFactory.AlarmIdProperty);
var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty);
var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId));
if (admission.SuppressedInPreviousWindow > 0)
{
_publisher.Publish(new ScriptLogEntry(
ScriptId: scriptId,
Level: nameof(LogEventLevel.Warning),
Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " +
$"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " +
$"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream",
TimestampUtc: logEvent.Timestamp.UtcDateTime,
VirtualTagId: virtualTagId,
AlarmId: alarmId,
EquipmentId: equipmentId));
}
if (!admission.Admitted) return;
_publisher.Publish(new ScriptLogEntry(
ScriptId: P(ScriptLoggerFactory.ScriptIdProperty)
?? P(ScriptLoggerFactory.ScriptNameProperty)
?? "unknown",
ScriptId: scriptId,
Level: logEvent.Level.ToString(),
Message: logEvent.RenderMessage(),
TimestampUtc: logEvent.Timestamp.UtcDateTime,
VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty),
AlarmId: P(ScriptLoggerFactory.AlarmIdProperty),
EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty)));
VirtualTagId: virtualTagId,
AlarmId: alarmId,
EquipmentId: equipmentId));
}
}