fix(runtime): review findings — recursion-safe run cap, atomic detach counter, summary edge cases, per-row event-log fallback

This commit is contained in:
Joseph Doherty
2026-08-14 23:42:29 -04:00
parent b1de9dfdd4
commit 950c54c5fc
10 changed files with 626 additions and 42 deletions
@@ -289,11 +289,18 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
if (Volatile.Read(ref worker.RunStamp) != observedRunStamp) return WorkerDetachOutcome.NotRunning;
if (Volatile.Read(ref worker.Detached) != 0) return WorkerDetachOutcome.NotRunning;
// Bound: never hold more than 2x threads (N wedged + N live).
if (_detachedLive >= _configuredCount) return WorkerDetachOutcome.AtCap;
// Bound: never hold more than 2x threads (N wedged + N live). Volatile read —
// the decrement side runs outside _growLock (see below).
if (Volatile.Read(ref _detachedLive) >= _configuredCount) return WorkerDetachOutcome.AtCap;
Volatile.Write(ref worker.Detached, 1);
_detachedLive++;
// Interlocked, NOT ++: the matching decrement in WorkerLoop is lock-free (the
// exiting worker never touches _growLock), so a plain read-modify-write here can
// lose that concurrent decrement. The drift is upward-only and permanent — the
// gauge over-reports and TryDetachWorker starts returning AtCap while real
// capacity is available. The surrounding sequence keeps the lock; only this one
// field is shared with a lock-free writer.
Interlocked.Increment(ref _detachedLive);
_configuredCount--; // the detached worker no longer counts towards the pool …
GrowTo(1); // … and GrowTo puts the count back by starting its replacement.
return WorkerDetachOutcome.Detached;
@@ -16,10 +16,18 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
///
/// <para>Lock-free: each script's counters are a small class of <c>long</c> fields mutated
/// via <see cref="Interlocked"/>, reached through a <see cref="ConcurrentDictionary{TKey,TValue}"/>
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>) so a
/// concurrent increment either lands in the snapshot being flushed or the fresh one that
/// replaces it — never lost, never double-counted, and the flush never blocks a recording
/// call (or vice versa).</para>
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>), so
/// the flush never blocks a recording call (or vice versa).</para>
///
/// <para>Accounting is BEST-EFFORT, not exact. A recorder that has already resolved its
/// <see cref="Counters"/> object from the pre-swap dictionary can increment it after the swap
/// has snapshotted it, so that increment lands in a window that has already been summarised
/// and is never reported. These are operational volume counters for one interval row, not a
/// ledger: an occasional count landing in neither window is acceptable, and nothing
/// downstream reconciles totals across intervals. What IS guaranteed is that a count is never
/// double-reported (each <see cref="Counters"/> instance is summarised by exactly one flush)
/// and that runs spanning a flush boundary — started in window N, completed in N+1 — still
/// emit their completion, because the flush filter admits a completion-only entry.</para>
/// </summary>
public sealed class ScriptRunSummaryRecorder
{
@@ -99,8 +107,8 @@ public sealed class ScriptRunSummaryRecorder
}
/// <summary>
/// Snapshots and resets the accumulated counters, then — if any script ran, failed, or
/// timed out during the interval — emits ONE aggregate "script" Info site event
/// Snapshots and resets the accumulated counters, then — if any script started, completed,
/// failed, or timed out during the interval — emits ONE aggregate "script" Info site event
/// summarizing the window. An interval with zero activity emits no row (the standby,
/// which never runs scripts, therefore never produces a summary row naturally).
/// </summary>
@@ -119,8 +127,13 @@ public sealed class ScriptRunSummaryRecorder
ref _counters,
new ConcurrentDictionary<(string InstanceName, string ScriptName), Counters>());
// Completed > 0 is admitted on its own: a run that started in window N and finished in
// N+1 records its completion against a FRESH counters object whose Started is 0, so
// filtering on Started/Failed/TimedOut alone would silently drop that completion (and,
// with it, the run's duration) from every summary row that could ever carry it.
var entries = snapshot
.Where(kvp => kvp.Value.Started > 0 || kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
.Where(kvp => kvp.Value.Started > 0 || kvp.Value.Completed > 0 ||
kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
.ToList();
if (entries.Count == 0)