dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime): review findings — recursion-safe run cap, atomic detach counter, summary edge cases, per-row event-log fallback

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 23:42:29 -04:00
parent b1de9dfdd4
commit 950c54c5fc
10 changed files with 626 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs
+17 -3
View File
@@ -526,6 +526,18 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// but the script body runs on the bounded set of dedicated threads, so blocking script
/// I/O is contained there and cannot starve the shared .NET thread pool. No per-run child
/// actor is created.</para>
///
/// <para>The cap governs NEW work only (<paramref name="callDepth"/> 0 — trigger-driven
/// runs and depth-0 Ask calls). A nested <c>CallScript</c> is exempt: the calling run is
/// itself still counted in <see cref="_runsInFlight"/> while it awaits its callee (the
/// slot is released only by <see cref="ScriptExecutionCompleted"/>, sent after the body
/// returns), and a script calling ITSELF routes back to this same actor — so counting the
/// nested launch against the cap would refuse legitimate self-recursion at depth
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> with a misleading "shed",
/// making <see cref="SiteRuntimeOptions.MaxScriptCallDepth"/> — the limit that actually
/// owns this path, enforced in <c>ScriptRuntimeContext.CallScript</c> — unreachable.
/// Nested depth is bounded by MaxScriptCallDepth instead, which is what the cap would
/// otherwise be doing badly.</para>
/// </summary>
private void SpawnExecution(
IReadOnlyDictionary<string, object?>? parameters,
@@ -534,7 +546,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
string correlationId,
Guid? parentExecutionId = null)
{
if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
if (callDepth == 0 && _runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedRun(replyTo, correlationId);
return;
@@ -604,8 +616,10 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> runs are already in flight.
/// The four already queued/running are kept — they are closest to their own deadlines and
/// already charged against them — so nothing is ever reordered. A trigger-driven run is
/// simply not launched; an Ask-based <c>CallScript</c> gets an explicit error so a nested
/// call or inbound-API route fails fast rather than hanging to its Ask timeout.
/// simply not launched; a depth-0 Ask (an inbound-API route, or a <c>CallScript</c> from
/// an unrelated script's run) gets an explicit error so the caller fails fast rather than
/// hanging to its Ask timeout. Nested (<c>callDepth &gt; 0</c>) launches never reach here —
/// see <see cref="SpawnExecution"/>.
/// </summary>
private void ShedRun(IActorRef replyTo, string correlationId)
{
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs
+10 -3
View File
@@ -289,11 +289,18 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
if (Volatile.Read(ref worker.RunStamp) != observedRunStamp) return WorkerDetachOutcome.NotRunning;
if (Volatile.Read(ref worker.Detached) != 0) return WorkerDetachOutcome.NotRunning;
// Bound: never hold more than 2x threads (N wedged + N live).
if (_detachedLive >= _configuredCount) return WorkerDetachOutcome.AtCap;
// Bound: never hold more than 2x threads (N wedged + N live). Volatile read —
// the decrement side runs outside _growLock (see below).
if (Volatile.Read(ref _detachedLive) >= _configuredCount) return WorkerDetachOutcome.AtCap;
Volatile.Write(ref worker.Detached, 1);
_detachedLive++;
// Interlocked, NOT ++: the matching decrement in WorkerLoop is lock-free (the
// exiting worker never touches _growLock), so a plain read-modify-write here can
// lose that concurrent decrement. The drift is upward-only and permanent — the
// gauge over-reports and TryDetachWorker starts returning AtCap while real
// capacity is available. The surrounding sequence keeps the lock; only this one
// field is shared with a lock-free writer.
Interlocked.Increment(ref _detachedLive);
_configuredCount--; // the detached worker no longer counts towards the pool …
GrowTo(1); // … and GrowTo puts the count back by starting its replacement.
return WorkerDetachOutcome.Detached;
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRunSummaryRecorder.cs
+20 -7
View File
@@ -16,10 +16,18 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
///
/// <para>Lock-free: each script's counters are a small class of <c>long</c> fields mutated
/// via <see cref="Interlocked"/>, reached through a <see cref="ConcurrentDictionary{TKey,TValue}"/>
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>) so a
/// concurrent increment either lands in the snapshot being flushed or the fresh one that
/// replaces it — never lost, never double-counted, and the flush never blocks a recording
/// call (or vice versa).</para>
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>), so
/// the flush never blocks a recording call (or vice versa).</para>
///
/// <para>Accounting is BEST-EFFORT, not exact. A recorder that has already resolved its
/// <see cref="Counters"/> object from the pre-swap dictionary can increment it after the swap
/// has snapshotted it, so that increment lands in a window that has already been summarised
/// and is never reported. These are operational volume counters for one interval row, not a
/// ledger: an occasional count landing in neither window is acceptable, and nothing
/// downstream reconciles totals across intervals. What IS guaranteed is that a count is never
/// double-reported (each <see cref="Counters"/> instance is summarised by exactly one flush)
/// and that runs spanning a flush boundary — started in window N, completed in N+1 — still
/// emit their completion, because the flush filter admits a completion-only entry.</para>
/// </summary>
public sealed class ScriptRunSummaryRecorder
{
@@ -99,8 +107,8 @@ public sealed class ScriptRunSummaryRecorder
}
/// <summary>
/// Snapshots and resets the accumulated counters, then — if any script ran, failed, or
/// timed out during the interval — emits ONE aggregate "script" Info site event
/// Snapshots and resets the accumulated counters, then — if any script started, completed,
/// failed, or timed out during the interval — emits ONE aggregate "script" Info site event
/// summarizing the window. An interval with zero activity emits no row (the standby,
/// which never runs scripts, therefore never produces a summary row naturally).
/// </summary>
@@ -119,8 +127,13 @@ public sealed class ScriptRunSummaryRecorder
ref _counters,
new ConcurrentDictionary<(string InstanceName, string ScriptName), Counters>());
// Completed > 0 is admitted on its own: a run that started in window N and finished in
// N+1 records its completion against a FRESH counters object whose Started is 0, so
// filtering on Started/Failed/TimedOut alone would silently drop that completion (and,
// with it, the run's duration) from every summary row that could ever carry it.
var entries = snapshot
.Where(kvp => kvp.Value.Started > 0 || kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
.Where(kvp => kvp.Value.Started > 0 || kvp.Value.Completed > 0 ||
kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
.ToList();
if (entries.Count == 0)