dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3)

Browse Source 
- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is
  re-seeded for the referenced paths) so retired tags no longer accrete across
  deploys; OnUpstreamChange no longer writes the cache after Dispose.
- R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s
  (DisposeDrainTimeout), then cancels it through a shutdown token every
  DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives
  one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer
  hang host shutdown. A save that completes inside the bound is still drained
  (positive control test).
- R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at
  Warning (naming the alarm) instead of swallowing it.
- R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step
  with every dead-letter/revive/purge path and resyncs it with the queued
  counter on the same cadence; GetStatus no longer runs a COUNT(*) scan.
- R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one
  summary entry naming the dropped count when the window rolls) behind
  ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink
  (key = script name).

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 10:17:30 -04:00
parent 2e9160317f
commit 6cbf763719
10 changed files with 683 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs
+21 -2
View File
@@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// Original exception (if any) is attached so the main log's diagnostics keep
/// the full stack trace.
/// </para>
/// <para>
/// <b>Rate-limited per script.</b> A predicate that throws on every push would otherwise mirror
/// one Warning per push into the main log; a <see cref="ScriptLogRateLimiter"/> keyed by script
/// name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The
/// scripts-*.log file sink is unaffected and keeps the full stream.
/// </para>
/// </remarks>
public sealed class ScriptLogCompanionSink : ILogEventSink
{
private readonly ILogger _mainLogger;
private readonly LogEventLevel _minMirrorLevel;
private readonly ScriptLogRateLimiter _limiter;
/// <summary>Initializes a new instance of the ScriptLogCompanionSink.</summary>
/// <param name="mainLogger">The companion logger to mirror events to.</param>
/// <param name="minMirrorLevel">The minimum log level to mirror (default: Error).</param>
public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error)
/// <param name="limiter">Per-script rate limiter; null ⇒ the default 20 per minute.</param>
public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error,
ScriptLogRateLimiter? limiter = null)
{
_mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger));
_minMirrorLevel = minMirrorLevel;
_limiter = limiter ?? new ScriptLogRateLimiter();
}
/// <summary>Emits a log event to the companion logger if it meets the minimum level.</summary>
/// <summary>Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget.</summary>
/// <param name="logEvent">The log event to emit.</param>
public void Emit(LogEvent logEvent)
{
@@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink
scriptName = s;
}
var admission = _limiter.Admit(scriptName);
if (admission.SuppressedInPreviousWindow > 0)
{
_mainLogger.Warning(
"[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log",
scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds);
}
if (!admission.Admitted) return;
var rendered = logEvent.RenderMessage();
if (logEvent.Exception is not null)
{
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs
+123
View File
@@ -0,0 +1,123 @@
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// <summary>
/// Per-source fixed-window rate limit for the script-log fan-out: the first
/// <see cref="DefaultLimitPerWindow"/> entries a source emits in a <see cref="DefaultWindow"/> are
/// admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first
/// admission after the window rolls — so the caller can emit one summary line in the dropped entries'
/// place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot
/// shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the
/// cluster bus / main log.
/// </summary>
/// <remarks>
/// Used by <see cref="ScriptLogTopicSink"/> (key = script + virtual-tag + alarm identity),
/// <see cref="ScriptLogCompanionSink"/> (key = script name) and the virtual-tag actor's per-failure
/// log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources
/// are swept every <see cref="SweepEvery"/> admissions so a churning tag set cannot grow the table
/// without bound.
/// </remarks>
public sealed class ScriptLogRateLimiter
{
/// <summary>Entries admitted per source per window by default: 20.</summary>
public const int DefaultLimitPerWindow = 20;
/// <summary>The default window: one minute.</summary>
public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1);
private const int SweepEvery = 256;
private readonly Dictionary<string, Source> _sources = new(StringComparer.Ordinal);
private readonly int _limit;
private readonly TimeSpan _window;
private readonly Func<DateTime> _clock;
private int _admitsSinceSweep;
/// <summary>Initializes a limiter.</summary>
/// <param name="limitPerWindow">Entries admitted per source per window; values below 1 are treated as 1.</param>
/// <param name="window">The fixed window; null ⇒ <see cref="DefaultWindow"/>.</param>
/// <param name="clock">UTC clock; null ⇒ <see cref="DateTime.UtcNow"/>.</param>
public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func<DateTime>? clock = null)
{
_limit = Math.Max(1, limitPerWindow);
_window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow;
_clock = clock ?? (static () => DateTime.UtcNow);
}
/// <summary>The window length.</summary>
public TimeSpan Window => _window;
/// <summary>
/// Ask whether <paramref name="sourceKey"/>'s next entry may be emitted. When the window has rolled
/// since the last call, <see cref="ScriptLogAdmission.SuppressedInPreviousWindow"/> carries the
/// number of entries dropped in the window that just closed (zero otherwise) — the caller emits one
/// summary for them BEFORE the admitted entry.
/// </summary>
/// <param name="sourceKey">Identity of the emitting script instance.</param>
/// <returns>The admission decision.</returns>
public ScriptLogAdmission Admit(string sourceKey)
{
ArgumentNullException.ThrowIfNull(sourceKey);
var now = _clock();
lock (_sources)
{
if (++_admitsSinceSweep >= SweepEvery)
{
_admitsSinceSweep = 0;
Sweep(now);
}
if (!_sources.TryGetValue(sourceKey, out var source))
{
source = new Source { WindowStart = now };
_sources[sourceKey] = source;
}
long summary = 0;
if (now - source.WindowStart >= _window)
{
summary = source.Suppressed;
source.WindowStart = now;
source.Admitted = 0;
source.Suppressed = 0;
}
source.LastSeen = now;
if (source.Admitted < _limit)
{
source.Admitted++;
return new ScriptLogAdmission(true, summary);
}
source.Suppressed++;
return new ScriptLogAdmission(false, summary);
}
}
private void Sweep(DateTime now)
{
var idle = now - (_window + _window);
List<string>? dead = null;
foreach (var (key, source) in _sources)
{
// A source that still has a pending summary is kept: its next entry must report the count.
if (source.LastSeen < idle && source.Suppressed == 0)
(dead ??= []).Add(key);
}
if (dead is null) return;
foreach (var key in dead) _sources.Remove(key);
}
private sealed class Source
{
public DateTime WindowStart;
public DateTime LastSeen;
public int Admitted;
public long Suppressed;
}
}
/// <summary>Result of <see cref="ScriptLogRateLimiter.Admit"/>.</summary>
/// <param name="Admitted">Whether the entry may be emitted.</param>
/// <param name="SuppressedInPreviousWindow">Entries dropped in the window that just closed — non-zero at
/// most once per window roll; the caller emits one summary line for them.</param>
public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow);
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs
+41 -8
View File
@@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// property so pipelines that pre-date this sink continue to work correctly. If
/// neither property is present <c>ScriptId</c> defaults to <c>"unknown"</c>.
/// </para>
/// <para>
/// <b>Rate-limited per source.</b> A script that logs at Information on every evaluation of a 1 Hz
/// input put one rendered string per second per script onto the cluster bus; 200 such scripts shed
/// everyone's entries out of the bounded telemetry channels. A <see cref="ScriptLogRateLimiter"/>
/// keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per
/// source and publishes one summary entry naming the dropped count when the window rolls.
/// </para>
/// </remarks>
public sealed class ScriptLogTopicSink : ILogEventSink
{
private readonly IScriptLogPublisher _publisher;
private readonly LogEventLevel _min;
private readonly ScriptLogRateLimiter _limiter;
/// <summary>Initializes a new instance of the <see cref="ScriptLogTopicSink"/> class.</summary>
/// <param name="publisher">The publisher that routes entries to the cluster bus. Must not be <c>null</c>.</param>
/// <param name="min">Minimum log level to forward. Events below this level are discarded (default: <see cref="LogEventLevel.Information"/>).</param>
/// <param name="limiter">Per-source rate limiter; null ⇒ the default 20 per minute.</param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="publisher"/> is <c>null</c>.</exception>
public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information)
public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information,
ScriptLogRateLimiter? limiter = null)
{
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_min = min;
_limiter = limiter ?? new ScriptLogRateLimiter();
}
/// <summary>
/// Converts the <paramref name="logEvent"/> to a <see cref="ScriptLogEntry"/> and
/// publishes it, unless the event is below the configured minimum level.
/// publishes it, unless the event is below the configured minimum level or the source is over its
/// per-minute budget.
/// </summary>
/// <param name="logEvent">The Serilog event to process. Silently ignored when <c>null</c>.</param>
public void Emit(LogEvent logEvent)
@@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink
? s
: null;
var scriptId = P(ScriptLoggerFactory.ScriptIdProperty)
?? P(ScriptLoggerFactory.ScriptNameProperty)
?? "unknown";
var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty);
var alarmId = P(ScriptLoggerFactory.AlarmIdProperty);
var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty);
var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId));
if (admission.SuppressedInPreviousWindow > 0)
{
_publisher.Publish(new ScriptLogEntry(
ScriptId: scriptId,
Level: nameof(LogEventLevel.Warning),
Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " +
$"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " +
$"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream",
TimestampUtc: logEvent.Timestamp.UtcDateTime,
VirtualTagId: virtualTagId,
AlarmId: alarmId,
EquipmentId: equipmentId));
}
if (!admission.Admitted) return;
_publisher.Publish(new ScriptLogEntry(
ScriptId: P(ScriptLoggerFactory.ScriptIdProperty)
?? P(ScriptLoggerFactory.ScriptNameProperty)
?? "unknown",
ScriptId: scriptId,
Level: logEvent.Level.ToString(),
Message: logEvent.RenderMessage(),
TimestampUtc: logEvent.Timestamp.UtcDateTime,
VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty),
AlarmId: P(ScriptLoggerFactory.AlarmIdProperty),
EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty)));
VirtualTagId: virtualTagId,
AlarmId: alarmId,
EquipmentId: equipmentId));
}
}