perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution
This commit is contained in:
Joseph Doherty
@@ -3,30 +3,106 @@ using System.Collections.Concurrent;
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// A dedicated, bounded <see cref="TaskScheduler"/> for running script
|
||||
/// The outcome of a <see cref="ScriptExecutionScheduler.TryDetachWorker"/> attempt.
|
||||
/// </summary>
|
||||
public enum WorkerDetachOutcome
|
||||
{
|
||||
/// <summary>
|
||||
/// The recorded slot/run pair no longer identifies a running task (the script finished,
|
||||
/// or it never held a worker because it hopped threads on an <c>await</c>). Nothing was
|
||||
/// detached and nothing needed to be — the worker is not lost.
|
||||
/// </summary>
|
||||
NotRunning,
|
||||
|
||||
/// <summary>The worker was marked detached and a replacement thread was started.</summary>
|
||||
Detached,
|
||||
|
||||
/// <summary>
|
||||
/// The number of live detached threads already equals the pool size, so no replacement was
|
||||
/// started. Bounded starvation is preferable to unbounded thread growth when scripts wedge
|
||||
/// en masse; the caller is expected to log this loudly.
|
||||
/// </summary>
|
||||
AtCap
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A dedicated, grow-only <see cref="TaskScheduler"/> for running script
|
||||
/// and alarm on-trigger bodies.
|
||||
///
|
||||
/// Script bodies may perform synchronous blocking I/O (a database connection, a
|
||||
/// synchronous external-system call). Running them on the shared .NET
|
||||
/// <see cref="ThreadPool"/> lets a burst of blocking scripts starve the pool and stall
|
||||
/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a fixed set
|
||||
/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a set
|
||||
/// of dedicated threads, so script blocking is contained to those threads and cannot
|
||||
/// exhaust the global pool.
|
||||
///
|
||||
/// <para>WP3.1 changed three things about that pool:</para>
|
||||
/// <list type="number">
|
||||
/// <item>It is no longer fixed-size. <see cref="EnsureCapacity"/> grows it towards
|
||||
/// <see cref="ComputeTargetThreads"/> (instance-scaled, clamped between the configured
|
||||
/// floor and ceiling). It is deliberately <em>grow-only</em>: undeploying instances
|
||||
/// leaves idle threads, which cost nothing measurable and avoid drain/steal complexity.</item>
|
||||
/// <item>A worker whose task has wedged in uninterruptible blocking I/O can be
|
||||
/// <see cref="TryDetachWorker">detached</see> and replaced, so a stuck script no longer
|
||||
/// permanently costs the pool a thread. Live detached threads are capped at the pool size.</item>
|
||||
/// <item>Trigger-expression evaluations no longer run here at all — they are non-blocking
|
||||
/// by construction and now run on the shared thread pool behind
|
||||
/// <see cref="TriggerEvalGate"/>, so an alarm's Expression trigger can never queue behind
|
||||
/// eight blocking script bodies (finding #4).</item>
|
||||
/// </list>
|
||||
///
|
||||
/// The scheduler is process-wide (one set of threads for all instances) and is sized
|
||||
/// from <see cref="SiteRuntimeOptions"/> the first time it is configured.
|
||||
/// </summary>
|
||||
public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Deployed instances per script-execution thread used by <see cref="ComputeTargetThreads"/>.
|
||||
/// A named constant rather than an option: no deployment needs to tune the ratio
|
||||
/// independently of the floor (<see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>)
|
||||
/// and the ceiling (<see cref="SiteRuntimeOptions.ScriptExecutionMaxThreadCount"/>).
|
||||
/// </summary>
|
||||
internal const int InstancesPerScriptThread = 8;
|
||||
|
||||
/// <summary>Thread-name prefix; also the inlining guard in <see cref="TryExecuteTaskInline"/>.</summary>
|
||||
private const string ThreadNamePrefix = "script-execution-";
|
||||
|
||||
private readonly BlockingCollection<Task> _queue = new();
|
||||
private readonly List<Thread> _threads;
|
||||
|
||||
/// <summary>
|
||||
/// Immutable-on-read snapshot array of worker slots. Growth (EnsureCapacity, detach
|
||||
/// replacement) publishes a NEW longer array under <see cref="_growLock"/>; existing
|
||||
/// slot objects are carried over by reference so a worker's index stays stable for its
|
||||
/// whole life. Readers take one volatile read and then work on that snapshot.
|
||||
/// </summary>
|
||||
private volatile WorkerSlot[] _slots;
|
||||
|
||||
/// <summary>Guards every mutation of <see cref="_slots"/>, <see cref="_configuredCount"/>, and the detach bookkeeping.</summary>
|
||||
private readonly object _growLock = new();
|
||||
|
||||
/// <summary>
|
||||
/// Number of NON-detached workers — i.e. the pool's nominal size. Unchanged by a
|
||||
/// detach (each detach starts a replacement), grown only by <see cref="EnsureCapacity"/>.
|
||||
/// </summary>
|
||||
private int _configuredCount;
|
||||
|
||||
/// <summary>Detached workers that have not yet finished their wedged task and exited.</summary>
|
||||
private int _detachedLive;
|
||||
|
||||
/// <summary>Monotonic run identity; see <see cref="WorkerSlot.RunStamp"/>.</summary>
|
||||
private long _runStampSeed;
|
||||
|
||||
private int _disposed;
|
||||
|
||||
// Per-worker "busy since" timestamp (Environment.TickCount64 ms) while a task
|
||||
// is executing on that worker, 0 when idle. Written by the worker thread,
|
||||
// read (lock-free) by the observability gauges below. S2/UA5: makes a
|
||||
// saturated or stuck script-execution pool visible on the site health report.
|
||||
private readonly long[] _busySinceTicks;
|
||||
/// <summary>
|
||||
/// The slot index of the script-execution worker running the current thread, or null on
|
||||
/// any other thread. Captured by the first synchronous segment of a script body so the
|
||||
/// stuck-script watchdog can identify — and replace — the exact worker a wedged script
|
||||
/// is holding. A script that hopped threads on an <c>await</c> is no longer on its worker,
|
||||
/// which is correct: it holds no thread and must not cause a detach.
|
||||
/// </summary>
|
||||
[ThreadStatic]
|
||||
internal static int? CurrentWorkerSlot;
|
||||
|
||||
private static volatile ScriptExecutionScheduler? _shared;
|
||||
private static readonly object SharedLock = new();
|
||||
@@ -34,8 +110,9 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
/// <summary>
|
||||
/// The process-wide script-execution scheduler, used as the default when no scheduler
|
||||
/// is injected. Lazily created on first use with the thread count from
|
||||
/// <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>; the first caller wins,
|
||||
/// subsequent calls reuse the existing instance.
|
||||
/// <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/> (the floor — the Deployment
|
||||
/// Manager grows it from there as instances deploy); the first caller wins, subsequent
|
||||
/// calls reuse the existing instance.
|
||||
///
|
||||
/// If the cached instance has been disposed it is recreated rather than handed back:
|
||||
/// a disposed scheduler can execute no work, so returning it would silently poison
|
||||
@@ -59,27 +136,40 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure sizing function for the blocking script-execution pool: the configured floor,
|
||||
/// raised to one thread per <see cref="InstancesPerScriptThread"/> enabled instances,
|
||||
/// clamped to the configured ceiling. Static and side-effect-free so the policy is
|
||||
/// unit-testable without starting a single thread.
|
||||
/// </summary>
|
||||
/// <param name="enabledInstances">Number of currently-running (enabled) Instance Actors.</param>
|
||||
/// <param name="options">Site runtime options supplying the floor and ceiling.</param>
|
||||
/// <returns>The target worker-thread count, always at least 1.</returns>
|
||||
public static int ComputeTargetThreads(int enabledInstances, SiteRuntimeOptions options)
|
||||
{
|
||||
var floor = Math.Max(1, options.ScriptExecutionThreadCount);
|
||||
// A ceiling below the floor is rejected by SiteRuntimeOptionsValidator; clamp here
|
||||
// too so a directly-constructed options object can never invert the range.
|
||||
var ceiling = Math.Max(floor, options.ScriptExecutionMaxThreadCount);
|
||||
var scaled = enabledInstances <= 0
|
||||
? 0
|
||||
: (int)Math.Ceiling(enabledInstances / (double)InstancesPerScriptThread);
|
||||
return Math.Clamp(Math.Max(floor, scaled), 1, ceiling);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a scheduler backed by <paramref name="threadCount"/> dedicated threads.
|
||||
/// </summary>
|
||||
/// <param name="threadCount">Number of dedicated worker threads to create.</param>
|
||||
/// <param name="threadCount">Initial number of dedicated worker threads to create.</param>
|
||||
public ScriptExecutionScheduler(int threadCount)
|
||||
{
|
||||
if (threadCount < 1)
|
||||
threadCount = 1;
|
||||
|
||||
_busySinceTicks = new long[threadCount];
|
||||
_threads = new List<Thread>(threadCount);
|
||||
for (var i = 0; i < threadCount; i++)
|
||||
_slots = [];
|
||||
lock (_growLock)
|
||||
{
|
||||
var index = i; // capture per-worker slot index
|
||||
var thread = new Thread(() => WorkerLoop(index))
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = $"script-execution-{i}"
|
||||
};
|
||||
_threads.Add(thread);
|
||||
thread.Start();
|
||||
GrowTo(threadCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,19 +178,28 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
public bool IsDisposed => Volatile.Read(ref _disposed) != 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int MaximumConcurrencyLevel => _threads.Count;
|
||||
public override int MaximumConcurrencyLevel => Volatile.Read(ref _configuredCount);
|
||||
|
||||
/// <summary>Number of tasks waiting in the queue (not counting those currently executing).</summary>
|
||||
public int QueueDepth => _queue.Count;
|
||||
|
||||
/// <summary>Number of worker threads currently executing a task.</summary>
|
||||
/// <summary>
|
||||
/// Workers whose task wedged past its deadline plus the stuck-script grace, which have been
|
||||
/// detached and replaced but have not yet returned. Surfaced on the site health report as
|
||||
/// <c>DetachedScriptThreads</c>: a non-zero, non-draining value means script bodies are
|
||||
/// permanently blocking threads.
|
||||
/// </summary>
|
||||
public int DetachedThreadCount => Volatile.Read(ref _detachedLive);
|
||||
|
||||
/// <summary>Number of worker threads currently executing a task (detached workers included — they really are busy).</summary>
|
||||
public int BusyThreadCount
|
||||
{
|
||||
get
|
||||
{
|
||||
var slots = _slots;
|
||||
var count = 0;
|
||||
for (var i = 0; i < _busySinceTicks.Length; i++)
|
||||
if (Volatile.Read(ref _busySinceTicks[i]) != 0) count++;
|
||||
foreach (var slot in slots)
|
||||
if (Volatile.Read(ref slot.BusySinceTicks) != 0) count++;
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -115,10 +214,11 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
get
|
||||
{
|
||||
var now = Environment.TickCount64;
|
||||
var slots = _slots;
|
||||
long oldestSince = 0;
|
||||
for (var i = 0; i < _busySinceTicks.Length; i++)
|
||||
foreach (var slot in slots)
|
||||
{
|
||||
var since = Volatile.Read(ref _busySinceTicks[i]);
|
||||
var since = Volatile.Read(ref slot.BusySinceTicks);
|
||||
if (since != 0 && (oldestSince == 0 || since < oldestSince))
|
||||
oldestSince = since;
|
||||
}
|
||||
@@ -126,20 +226,138 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Grows the pool to <paramref name="target"/> non-detached workers. Idempotent and
|
||||
/// never shrinking: a target at or below the current size is a no-op. Called from
|
||||
/// <c>DeploymentManagerActor.UpdateInstanceCounts</c> on every deploy/undeploy/enable/
|
||||
/// disable and per staggered startup batch.
|
||||
/// </summary>
|
||||
/// <param name="target">Desired number of non-detached worker threads.</param>
|
||||
/// <returns>The pool's non-detached worker count after the call.</returns>
|
||||
public int EnsureCapacity(int target)
|
||||
{
|
||||
if (IsDisposed) return Volatile.Read(ref _configuredCount);
|
||||
if (target <= Volatile.Read(ref _configuredCount)) return Volatile.Read(ref _configuredCount);
|
||||
|
||||
lock (_growLock)
|
||||
{
|
||||
if (IsDisposed || target <= _configuredCount) return _configuredCount;
|
||||
GrowTo(target - _configuredCount);
|
||||
return _configuredCount;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The identity stamp of the task currently running on <paramref name="slot"/>, or 0 when
|
||||
/// that worker is idle. Captured alongside the slot index at script-body start and handed
|
||||
/// back to <see cref="TryDetachWorker"/>, which detaches only if the SAME task is still
|
||||
/// running. A monotonic counter rather than a timestamp: two runs on one worker inside the
|
||||
/// same <see cref="Environment.TickCount64"/> tick would otherwise be indistinguishable.
|
||||
/// </summary>
|
||||
/// <param name="slot">The worker slot index.</param>
|
||||
/// <returns>The current run stamp, or 0 when the slot is idle or out of range.</returns>
|
||||
internal long CurrentRunStamp(int slot)
|
||||
{
|
||||
var slots = _slots;
|
||||
return slot >= 0 && slot < slots.Length ? Volatile.Read(ref slots[slot].RunStamp) : 0L;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detaches the worker at <paramref name="slot"/> — if it is still running the task
|
||||
/// identified by <paramref name="observedRunStamp"/> — and starts a replacement thread, so
|
||||
/// a script wedged in uninterruptible blocking I/O no longer costs the pool a thread
|
||||
/// permanently. The detached worker exits (instead of pulling more work) as soon as its
|
||||
/// wedged task finally returns, so capacity never silently doubles-and-drains.
|
||||
/// </summary>
|
||||
/// <param name="slot">The worker slot recorded when the script body started.</param>
|
||||
/// <param name="observedRunStamp">The run stamp recorded at the same moment.</param>
|
||||
/// <returns>What was done; see <see cref="WorkerDetachOutcome"/>.</returns>
|
||||
internal WorkerDetachOutcome TryDetachWorker(int slot, long observedRunStamp)
|
||||
{
|
||||
if (observedRunStamp == 0 || IsDisposed) return WorkerDetachOutcome.NotRunning;
|
||||
|
||||
lock (_growLock)
|
||||
{
|
||||
if (IsDisposed) return WorkerDetachOutcome.NotRunning;
|
||||
|
||||
var slots = _slots;
|
||||
if (slot < 0 || slot >= slots.Length) return WorkerDetachOutcome.NotRunning;
|
||||
|
||||
var worker = slots[slot];
|
||||
// Same task still running on that exact worker? If the stamp moved (or went to 0)
|
||||
// the script finished or never held this thread — there is nothing lost to replace.
|
||||
if (Volatile.Read(ref worker.RunStamp) != observedRunStamp) return WorkerDetachOutcome.NotRunning;
|
||||
if (Volatile.Read(ref worker.Detached) != 0) return WorkerDetachOutcome.NotRunning;
|
||||
|
||||
// Bound: never hold more than 2x threads (N wedged + N live).
|
||||
if (_detachedLive >= _configuredCount) return WorkerDetachOutcome.AtCap;
|
||||
|
||||
Volatile.Write(ref worker.Detached, 1);
|
||||
_detachedLive++;
|
||||
_configuredCount--; // the detached worker no longer counts towards the pool …
|
||||
GrowTo(1); // … and GrowTo puts the count back by starting its replacement.
|
||||
return WorkerDetachOutcome.Detached;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Appends <paramref name="count"/> new worker slots + threads. Caller holds <see cref="_growLock"/>.</summary>
|
||||
private void GrowTo(int count)
|
||||
{
|
||||
var existing = _slots;
|
||||
var grown = new WorkerSlot[existing.Length + count];
|
||||
Array.Copy(existing, grown, existing.Length);
|
||||
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var index = existing.Length + i;
|
||||
grown[index] = new WorkerSlot();
|
||||
}
|
||||
|
||||
// Publish the array BEFORE starting the threads: a worker's very first action is to
|
||||
// index its own slot, which must already be visible on the published snapshot.
|
||||
_slots = grown;
|
||||
_configuredCount += count;
|
||||
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var index = existing.Length + i;
|
||||
var thread = new Thread(() => WorkerLoop(index))
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = ThreadNamePrefix + index
|
||||
};
|
||||
grown[index].Thread = thread;
|
||||
thread.Start();
|
||||
}
|
||||
}
|
||||
|
||||
private void WorkerLoop(int index)
|
||||
{
|
||||
CurrentWorkerSlot = index;
|
||||
var slot = _slots[index];
|
||||
try
|
||||
{
|
||||
foreach (var task in _queue.GetConsumingEnumerable())
|
||||
{
|
||||
Volatile.Write(ref _busySinceTicks[index], Environment.TickCount64);
|
||||
Volatile.Write(ref slot.RunStamp, Interlocked.Increment(ref _runStampSeed));
|
||||
Volatile.Write(ref slot.BusySinceTicks, Environment.TickCount64);
|
||||
try
|
||||
{
|
||||
TryExecuteTask(task);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Volatile.Write(ref _busySinceTicks[index], 0);
|
||||
Volatile.Write(ref slot.BusySinceTicks, 0);
|
||||
Volatile.Write(ref slot.RunStamp, 0);
|
||||
}
|
||||
|
||||
// Detached while this task ran: a replacement worker is already live, so
|
||||
// exit rather than pulling more work — otherwise capacity would silently
|
||||
// double once the wedged script finally returned.
|
||||
if (Volatile.Read(ref slot.Detached) != 0)
|
||||
{
|
||||
Interlocked.Decrement(ref _detachedLive);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -147,6 +365,10 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
{
|
||||
// Scheduler disposed — worker exits.
|
||||
}
|
||||
finally
|
||||
{
|
||||
CurrentWorkerSlot = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -157,7 +379,7 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
{
|
||||
// Only inline if we are already on one of this scheduler's worker threads,
|
||||
// so script work never escapes onto a thread-pool thread.
|
||||
if (Thread.CurrentThread.Name?.StartsWith("script-execution-", StringComparison.Ordinal) != true)
|
||||
if (Thread.CurrentThread.Name?.StartsWith(ThreadNamePrefix, StringComparison.Ordinal) != true)
|
||||
return false;
|
||||
|
||||
return TryExecuteTask(task);
|
||||
@@ -173,8 +395,35 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
return;
|
||||
|
||||
_queue.CompleteAdding();
|
||||
foreach (var thread in _threads)
|
||||
thread.Join(TimeSpan.FromSeconds(5));
|
||||
foreach (var slot in _slots)
|
||||
slot.Thread?.Join(TimeSpan.FromSeconds(5));
|
||||
_queue.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-worker bookkeeping. One instance per slot index, created once and carried by
|
||||
/// reference across every <see cref="_slots"/> growth so a worker's state survives
|
||||
/// the array being replaced.
|
||||
/// </summary>
|
||||
private sealed class WorkerSlot
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="Environment.TickCount64"/> at which the worker picked up its current
|
||||
/// task, 0 when idle. Written by the worker, read lock-free by the gauges.
|
||||
/// </summary>
|
||||
public long BusySinceTicks;
|
||||
|
||||
/// <summary>
|
||||
/// Monotonic identity of the task currently running on this worker, 0 when idle.
|
||||
/// Distinguishes two runs that start inside the same clock tick, which
|
||||
/// <see cref="BusySinceTicks"/> alone cannot.
|
||||
/// </summary>
|
||||
public long RunStamp;
|
||||
|
||||
/// <summary>Non-zero once the worker has been detached; it exits after its current task.</summary>
|
||||
public int Detached;
|
||||
|
||||
/// <summary>The worker thread, for <see cref="Dispose"/> to join.</summary>
|
||||
public Thread? Thread;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1: launches one script or alarm on-trigger run directly from its owning coordinator
|
||||
/// (<see cref="ScriptActor"/> / <see cref="AlarmActor"/>), replacing the short-lived
|
||||
/// <c>ScriptExecutionActor</c> and <c>AlarmExecutionActor</c>.
|
||||
///
|
||||
/// <para>Those actors were already inert shells: neither declared a single <c>Receive</c>
|
||||
/// handler (they executed from their constructor), neither had a <c>PostStop</c>, state, or
|
||||
/// stash, and their <c>IActorRef</c> was never a message target — the whole lifecycle lived
|
||||
/// inside a detached <see cref="Task"/> the actor never observed. What they cost was a real
|
||||
/// actor cell, mailbox, and name registration per run, plus a per-spawn expression-tree
|
||||
/// <c>Props.Create</c>. Removing them changes no semantics; the run body below is the former
|
||||
/// <c>ExecuteScript</c>/<c>ExecuteAlarmScript</c> body, unified.</para>
|
||||
///
|
||||
/// <para>Two behaviours DID change, both deliberately:</para>
|
||||
/// <list type="number">
|
||||
/// <item>The deadline <see cref="CancellationTokenSource"/> is now armed by the CALLER,
|
||||
/// before the body is queued to the <see cref="ScriptExecutionScheduler"/>, so queue wait
|
||||
/// consumes the script's own budget. A body that dequeues past its deadline skips execution
|
||||
/// entirely and takes the timeout path — a saturated pool sheds stale work instead of
|
||||
/// running it late.</item>
|
||||
/// <item>The stuck-script watchdog now DETACHES and replaces the worker thread a wedged
|
||||
/// script is holding (<see cref="ScriptExecutionScheduler.TryDetachWorker"/>) instead of
|
||||
/// only naming it, so the pool recovers its capacity.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>Ordering, supervision, telemetry, DI scoping, Ask replies, completion messages, and
|
||||
/// the audit <c>ExecutionId</c>/<c>ParentExecutionId</c> threading are all unchanged. The
|
||||
/// launch call itself can throw only if the target scheduler is unusable (e.g. disposed); the
|
||||
/// caller wraps it so that failure replies to the Ask caller and decrements the in-flight
|
||||
/// counter rather than escalating to the coordinator's supervisor.</para>
|
||||
/// </summary>
|
||||
internal static class ScriptRunLauncher
|
||||
{
|
||||
/// <summary>
|
||||
/// Launches an instance script run. Returns as soon as the body is queued; the run reports
|
||||
/// completion to <paramref name="completionTarget"/>.
|
||||
/// </summary>
|
||||
/// <param name="scriptName">Name of the script being executed.</param>
|
||||
/// <param name="instanceName">Name of the instance that owns the script.</param>
|
||||
/// <param name="compiledScript">Compiled Roslyn script to execute.</param>
|
||||
/// <param name="parameters">Optional named parameter values for the script.</param>
|
||||
/// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
|
||||
/// <param name="instanceActor">Instance actor reference for attribute access.</param>
|
||||
/// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
|
||||
/// <param name="options">Site runtime options applied during execution.</param>
|
||||
/// <param name="replyTo">Actor reference that receives the script result; <c>Nobody</c> for fire-and-forget.</param>
|
||||
/// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
|
||||
/// <param name="completionTarget">The owning <see cref="ScriptActor"/>, which receives the completion message.</param>
|
||||
/// <param name="siteCommunicationActor">Site communication actor (resolved on the actor thread) for <c>Notify.Status</c>.</param>
|
||||
/// <param name="logger">Logger for script execution events.</param>
|
||||
/// <param name="scope">Script scope controlling which APIs are available.</param>
|
||||
/// <param name="runId">Per-script run counter, used only in log messages (replaces the former per-run actor name).</param>
|
||||
/// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
|
||||
/// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
|
||||
/// <param name="parentExecutionId">ExecutionId of the spawning execution for audit correlation; null for root runs.</param>
|
||||
/// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global value.</param>
|
||||
/// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
|
||||
public static void LaunchScript(
|
||||
string scriptName,
|
||||
string instanceName,
|
||||
Script<object?> compiledScript,
|
||||
IReadOnlyDictionary<string, object?>? parameters,
|
||||
int callDepth,
|
||||
IActorRef instanceActor,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef replyTo,
|
||||
string correlationId,
|
||||
IActorRef completionTarget,
|
||||
ICanTell? siteCommunicationActor,
|
||||
ILogger logger,
|
||||
ScriptScope scope,
|
||||
long runId,
|
||||
ISiteHealthCollector? healthCollector,
|
||||
IServiceProvider? serviceProvider,
|
||||
Guid? parentExecutionId,
|
||||
int? executionTimeoutSeconds,
|
||||
ScriptExecutionScheduler? scheduler)
|
||||
{
|
||||
var timeout = ResolveTimeout(executionTimeoutSeconds, options);
|
||||
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
|
||||
|
||||
// Armed HERE, on the caller's (actor) thread, not inside the queued body: queue wait
|
||||
// must consume the script's own budget, otherwise a saturated pool silently grants
|
||||
// every backlogged run a fresh full timeout.
|
||||
var cts = new CancellationTokenSource(timeout);
|
||||
try
|
||||
{
|
||||
_ = Task.Factory.StartNew(
|
||||
() => RunScriptAsync(
|
||||
scriptName, instanceName, compiledScript, parameters, callDepth,
|
||||
instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
|
||||
completionTarget, siteCommunicationActor, logger, scope, runId,
|
||||
healthCollector, serviceProvider, parentExecutionId, timeout,
|
||||
executionScheduler, cts),
|
||||
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
|
||||
.Unwrap();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The body never got queued, so nothing will ever dispose the CTS.
|
||||
cts.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Launches an alarm on-trigger run. Same contract as <see cref="LaunchScript"/>, with the
|
||||
/// firing alarm's level/priority/message exposed to the body through the <c>Alarm</c> global.
|
||||
/// </summary>
|
||||
/// <param name="alarmName">The canonical name of the alarm that triggered.</param>
|
||||
/// <param name="instanceName">The name of the owning instance.</param>
|
||||
/// <param name="level">The alarm severity level at the time of triggering.</param>
|
||||
/// <param name="priority">The alarm priority value.</param>
|
||||
/// <param name="message">The alarm message to pass to the script.</param>
|
||||
/// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
|
||||
/// <param name="instanceActor">Reference to the instance actor for attribute/script calls.</param>
|
||||
/// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
|
||||
/// <param name="options">Site runtime configuration options, including the execution timeout.</param>
|
||||
/// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
|
||||
/// <param name="logger">Logger for execution diagnostics.</param>
|
||||
/// <param name="runId">Per-alarm run counter, used only in log messages.</param>
|
||||
/// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
|
||||
/// <param name="parentExecutionId">
|
||||
/// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of the execution whose attribute
|
||||
/// write fired this alarm. Null when the firing value came from the Data Connection Layer
|
||||
/// (external data has no spawning execution) — that on-trigger run is a tree ROOT.
|
||||
/// </param>
|
||||
/// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
|
||||
public static void LaunchAlarmScript(
|
||||
string alarmName,
|
||||
string instanceName,
|
||||
AlarmLevel level,
|
||||
int priority,
|
||||
string message,
|
||||
Script<object?> compiledScript,
|
||||
IActorRef instanceActor,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef completionTarget,
|
||||
ILogger logger,
|
||||
long runId,
|
||||
int? executionTimeoutSeconds,
|
||||
Guid? parentExecutionId,
|
||||
ScriptExecutionScheduler? scheduler)
|
||||
{
|
||||
var timeout = ResolveTimeout(executionTimeoutSeconds, options);
|
||||
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
|
||||
|
||||
// Enqueue-anchored deadline; see LaunchScript.
|
||||
var cts = new CancellationTokenSource(timeout);
|
||||
try
|
||||
{
|
||||
_ = Task.Factory.StartNew(
|
||||
() => RunAlarmScriptAsync(
|
||||
alarmName, instanceName, level, priority, message, compiledScript,
|
||||
instanceActor, sharedScriptLibrary, options, completionTarget, logger,
|
||||
runId, parentExecutionId, timeout, cts),
|
||||
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
|
||||
.Unwrap();
|
||||
}
|
||||
catch
|
||||
{
|
||||
cts.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-script timeout overrides the global default. A null or non-positive per-script
|
||||
/// value (≤ 0) falls back to the global.
|
||||
/// </summary>
|
||||
private static TimeSpan ResolveTimeout(int? executionTimeoutSeconds, SiteRuntimeOptions options)
|
||||
=> TimeSpan.FromSeconds(
|
||||
executionTimeoutSeconds is { } perScript && perScript > 0
|
||||
? perScript
|
||||
: options.ScriptExecutionTimeoutSeconds);
|
||||
|
||||
private static async Task RunScriptAsync(
|
||||
string scriptName,
|
||||
string instanceName,
|
||||
Script<object?> compiledScript,
|
||||
IReadOnlyDictionary<string, object?>? parameters,
|
||||
int callDepth,
|
||||
IActorRef instanceActor,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef replyTo,
|
||||
string correlationId,
|
||||
IActorRef completionTarget,
|
||||
ICanTell? siteCommunicationActor,
|
||||
ILogger logger,
|
||||
ScriptScope scope,
|
||||
long runId,
|
||||
ISiteHealthCollector? healthCollector,
|
||||
IServiceProvider? serviceProvider,
|
||||
Guid? parentExecutionId,
|
||||
TimeSpan timeout,
|
||||
ScriptExecutionScheduler executionScheduler,
|
||||
CancellationTokenSource cts)
|
||||
{
|
||||
IServiceScope? serviceScope = null;
|
||||
// ISiteEventLogger is a singleton; resolve from the root provider so
|
||||
// it is available to the catch blocks regardless of scope state.
|
||||
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
|
||||
|
||||
// WP3.1: identify the worker this body is holding, so the watchdog can detach and
|
||||
// replace it if the body wedges. Read in the FIRST synchronous segment — a script
|
||||
// that later hops threads on an await no longer holds this worker, and the run-stamp
|
||||
// guard in TryDetachWorker will (correctly) refuse to detach it.
|
||||
var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
|
||||
var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
|
||||
|
||||
var completed = 0;
|
||||
ArmStuckScriptWatchdog(
|
||||
cts, options, logger, siteEventLogger, executionScheduler,
|
||||
workerSlot, workerRunStamp, timeout,
|
||||
() => Volatile.Read(ref completed) != 0,
|
||||
$"Script '{scriptName}' on instance '{instanceName}'",
|
||||
instanceName, $"ScriptActor:{scriptName}");
|
||||
|
||||
try
|
||||
{
|
||||
// WP3.1 shed-at-dequeue: the deadline was armed at ENQUEUE, so an already-cancelled
|
||||
// token here means this run spent its entire budget queueing. Running it now would
|
||||
// burn a scarce script thread on work that is already stale — throw straight into
|
||||
// the existing timeout path instead, BEFORE the "started" event and before the body.
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
// Resolve integration services from DI (scoped lifetime)
|
||||
IExternalSystemClient? externalSystemClient = null;
|
||||
IDatabaseGateway? databaseGateway = null;
|
||||
// Notification Outbox: the S&F engine is a singleton; the site identity
|
||||
// provider supplies the site id stamped on enqueued notifications.
|
||||
StoreAndForwardService? storeAndForward = null;
|
||||
var siteId = string.Empty;
|
||||
// The writer is a singleton (FallbackAuditWriter
|
||||
// composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
|
||||
// that haven't called AddAuditLog, which the helper handles as a no-op.
|
||||
IAuditWriter? auditWriter = null;
|
||||
// Site-local tracking store
|
||||
// backing Tracking.Status(id). Singleton; null in tests / hosts
|
||||
// that haven't wired the store, which the helper handles by
|
||||
// throwing on access.
|
||||
IOperationTrackingStore? operationTrackingStore = null;
|
||||
// Site-side cached-call
|
||||
// telemetry forwarder. Singleton bound to the AuditLog
|
||||
// composition root; null in tests / hosts that haven't called
|
||||
// AddAuditLog, in which case the cached-call helpers degrade
|
||||
// to the no-emission path (the underlying S&F handoff still
|
||||
// happens and a TrackedOperationId is still returned).
|
||||
ICachedCallTelemetryForwarder? cachedForwarder = null;
|
||||
// SourceNode-stamping: the local node name
|
||||
// resolved from INodeIdentityProvider — node-a/node-b on site
|
||||
// hosts. Null in tests / hosts that haven't registered the
|
||||
// provider, in which case NotificationSubmit.SourceNode and
|
||||
// SiteCallOperational.SourceNode stay null and central
|
||||
// persists the rows with SourceNode NULL.
|
||||
string? sourceNode = null;
|
||||
|
||||
if (serviceProvider != null)
|
||||
{
|
||||
serviceScope = serviceProvider.CreateScope();
|
||||
externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
|
||||
databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
|
||||
storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
|
||||
siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
|
||||
?? string.Empty;
|
||||
auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
|
||||
operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
|
||||
cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
|
||||
sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
|
||||
}
|
||||
|
||||
var context = new ScriptRuntimeContext(
|
||||
instanceActor,
|
||||
sharedScriptLibrary,
|
||||
callDepth,
|
||||
options.MaxScriptCallDepth,
|
||||
timeout,
|
||||
instanceName,
|
||||
logger,
|
||||
externalSystemClient,
|
||||
databaseGateway,
|
||||
storeAndForward,
|
||||
siteCommunicationActor,
|
||||
siteId,
|
||||
// Notification Outbox (FU3): stamp the executing script onto outbound
|
||||
// notifications using the Site Event Logging "Source" convention.
|
||||
sourceScript: $"ScriptActor:{scriptName}",
|
||||
// Emit one ApiOutbound/ApiCall row per
|
||||
// ExternalSystem.Call. Writer is best-effort; failures are logged
|
||||
// and swallowed inside the helper so the script's call path is
|
||||
// never aborted by an audit failure.
|
||||
auditWriter: auditWriter,
|
||||
// Site-local tracking store
|
||||
// backing Tracking.Status(id). Authoritative source of truth for
|
||||
// cached-call status — read directly by the script API.
|
||||
operationTrackingStore: operationTrackingStore,
|
||||
// Cached-call telemetry
|
||||
// forwarder for ExternalSystem.CachedCall / Database.CachedWrite
|
||||
// CachedSubmit emission + the immediate-success terminal-row
|
||||
// emission. Best-effort: null degrades the helpers to a
|
||||
// no-emission path; the S&F handoff and TrackedOperationId
|
||||
// return are unaffected.
|
||||
cachedForwarder: cachedForwarder,
|
||||
// The spawning execution's
|
||||
// id for an inbound-API-routed call. The routed script still
|
||||
// mints its own fresh ExecutionId — this records the spawner.
|
||||
// Null for normal (tag-change / timer) runs.
|
||||
parentExecutionId: parentExecutionId,
|
||||
// SourceNode-stamping: the local node name
|
||||
// (node-a/node-b on a site) — threaded down so Notify.Send
|
||||
// and the four cached-call telemetry constructors can stamp
|
||||
// it onto NotificationSubmit.SourceNode and
|
||||
// SiteCallOperational.SourceNode respectively.
|
||||
sourceNode: sourceNode,
|
||||
// Thread the singleton site event logger so
|
||||
// recursion-limit violations at CallScript/CallShared emit a
|
||||
// script Error site event in addition to ILogger.LogError.
|
||||
siteEventLogger: siteEventLogger,
|
||||
// WaitForAttribute (spec §4.3/§4.4): thread the per-script
|
||||
// execution-timeout token so Attributes.WaitAsync's Ask is
|
||||
// bounded by the script's own ExecutionTimeoutSeconds — a
|
||||
// shorter script deadline wins over the wait's own timeout.
|
||||
scriptTimeoutToken: cts.Token);
|
||||
|
||||
var globals = new ScriptGlobals
|
||||
{
|
||||
Instance = context,
|
||||
Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
|
||||
CancellationToken = cts.Token,
|
||||
Scope = scope
|
||||
};
|
||||
|
||||
// Operational `script` event — execution started. Fire-and-forget
|
||||
// (the `_ =` discards the task) so the event log can never block or
|
||||
// fault the script's own run; mirrors the existing Error-path emit.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' started");
|
||||
|
||||
var state = await compiledScript.RunAsync(globals, cts.Token);
|
||||
|
||||
// Send result to requester if this was an Ask-based call
|
||||
if (!replyTo.IsNobody())
|
||||
{
|
||||
replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
|
||||
}
|
||||
|
||||
// Operational `script` event — execution completed successfully.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' completed");
|
||||
|
||||
// Notify the owning ScriptActor of completion (also releases its in-flight slot).
|
||||
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
|
||||
logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);
|
||||
|
||||
// Failures recorded to site event log; script NOT disabled after failure.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
|
||||
|
||||
if (!replyTo.IsNobody())
|
||||
{
|
||||
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
|
||||
}
|
||||
|
||||
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
// Failures recorded to site event log; script NOT disabled after failure.
|
||||
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
|
||||
logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
|
||||
scriptName, instanceName, runId);
|
||||
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
|
||||
|
||||
if (!replyTo.IsNobody())
|
||||
{
|
||||
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
|
||||
}
|
||||
|
||||
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Mark the body finished so the stuck-script watchdog (registered on
|
||||
// cts.Token above) treats a timely cancellation as NOT stuck.
|
||||
Interlocked.Exchange(ref completed, 1);
|
||||
// Dispose the DI scope (and scoped services) after script execution completes
|
||||
serviceScope?.Dispose();
|
||||
cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task RunAlarmScriptAsync(
|
||||
string alarmName,
|
||||
string instanceName,
|
||||
AlarmLevel level,
|
||||
int priority,
|
||||
string message,
|
||||
Script<object?> compiledScript,
|
||||
IActorRef instanceActor,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef completionTarget,
|
||||
ILogger logger,
|
||||
long runId,
|
||||
Guid? parentExecutionId,
|
||||
TimeSpan timeout,
|
||||
CancellationTokenSource cts)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
|
||||
// spent its whole budget queueing behind other script bodies. Shed it rather than
|
||||
// run it late — the alarm it would react to is already stale.
|
||||
cts.Token.ThrowIfCancellationRequested();
|
||||
|
||||
// An alarm on-trigger run can call Instance.CallScript()
|
||||
// via the ScriptRuntimeContext injected into globals
|
||||
var context = new ScriptRuntimeContext(
|
||||
instanceActor,
|
||||
sharedScriptLibrary,
|
||||
currentCallDepth: 0,
|
||||
options.MaxScriptCallDepth,
|
||||
timeout,
|
||||
instanceName,
|
||||
logger,
|
||||
// ParentExecutionId tag-cascade: the
|
||||
// alarm on-trigger run mints its own fresh ExecutionId (the
|
||||
// ctor's `?? NewGuid()` fallback) and records the firing
|
||||
// execution's id as its ParentExecutionId — null (a root)
|
||||
// only when the firing value came from the DCL.
|
||||
parentExecutionId: parentExecutionId,
|
||||
// WaitForAttribute (spec §4.4): thread the alarm on-trigger
|
||||
// script's per-script execution-timeout token so a
|
||||
// Attributes.WaitAsync inside an on-trigger script is bounded
|
||||
// by the same script deadline.
|
||||
scriptTimeoutToken: cts.Token);
|
||||
|
||||
var globals = new ScriptGlobals
|
||||
{
|
||||
Instance = context,
|
||||
Parameters = new ScriptParameters(),
|
||||
CancellationToken = cts.Token,
|
||||
Alarm = new AlarmContext
|
||||
{
|
||||
Name = alarmName,
|
||||
Level = level,
|
||||
Priority = priority,
|
||||
Message = message
|
||||
}
|
||||
};
|
||||
|
||||
await compiledScript.RunAsync(globals, cts.Token);
|
||||
|
||||
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
|
||||
alarmName, instanceName, runId);
|
||||
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Failures logged, alarm continues
|
||||
logger.LogError(ex,
|
||||
"Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
|
||||
alarmName, instanceName, runId);
|
||||
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
|
||||
}
|
||||
finally
|
||||
{
|
||||
cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the stuck-script watchdog (S2, extended by WP3.1).
|
||||
///
|
||||
/// <para>The CTS firing only REQUESTS cooperative cancellation; it does NOT free a thread
|
||||
/// blocked in synchronous I/O. When the deadline elapses, this waits
|
||||
/// <see cref="SiteRuntimeOptions.StuckScriptGraceMs"/> and, if the body still has not
|
||||
/// returned, (a) names the script loudly on the site event log and (b) DETACHES the worker
|
||||
/// thread it is holding and starts a replacement — so a wedged script costs the pool one
|
||||
/// thread only until the watchdog fires, not forever. At the detach cap it logs an Error
|
||||
/// instead of growing threads without bound.</para>
|
||||
///
|
||||
/// <para>The <c>Task.Run</c>/<c>Task.Delay</c> run on the shared thread pool, never on the
|
||||
/// (possibly saturated) script scheduler — deliberate, and the whole point of the watchdog.
|
||||
/// <c>Register</c> fires on cancellation only; a normally-completing run disposes its CTS
|
||||
/// with no callback. A run cancelled while still QUEUED flips its completed flag in the
|
||||
/// body's <c>finally</c> long before the grace elapses, so it is correctly not reported.</para>
|
||||
/// </summary>
|
||||
private static void ArmStuckScriptWatchdog(
|
||||
CancellationTokenSource cts,
|
||||
SiteRuntimeOptions options,
|
||||
ILogger logger,
|
||||
ISiteEventLogger? siteEventLogger,
|
||||
ScriptExecutionScheduler scheduler,
|
||||
int? workerSlot,
|
||||
long workerRunStamp,
|
||||
TimeSpan timeout,
|
||||
Func<bool> isCompleted,
|
||||
string subject,
|
||||
string instanceName,
|
||||
string source)
|
||||
{
|
||||
var graceMs = options.StuckScriptGraceMs;
|
||||
cts.Token.Register(() => _ = Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(graceMs);
|
||||
if (isCompleted()) return;
|
||||
|
||||
var stuckMsg = $"{subject} exceeded its {timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — " +
|
||||
"its dedicated script-execution thread is blocked (cooperative cancellation not observed).";
|
||||
|
||||
var outcome = workerSlot is { } slot
|
||||
? scheduler.TryDetachWorker(slot, workerRunStamp)
|
||||
: WorkerDetachOutcome.NotRunning;
|
||||
|
||||
switch (outcome)
|
||||
{
|
||||
case WorkerDetachOutcome.Detached:
|
||||
stuckMsg += " The worker thread has been DETACHED and replaced; it will exit when the body returns.";
|
||||
break;
|
||||
case WorkerDetachOutcome.AtCap:
|
||||
stuckMsg += $" The script-execution pool already holds {scheduler.DetachedThreadCount} detached " +
|
||||
"stuck threads (at cap); NOT replacing this one.";
|
||||
break;
|
||||
case WorkerDetachOutcome.NotRunning:
|
||||
// The body is not on a worker thread (it hopped threads on an await), so no
|
||||
// dedicated thread is lost — report it, but there is nothing to replace.
|
||||
break;
|
||||
}
|
||||
|
||||
logger.LogError(stuckMsg);
|
||||
_ = siteEventLogger?.LogEventAsync("script", "Error", instanceName, source, stuckMsg);
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,6 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
public class ScriptRuntimeContext
|
||||
{
|
||||
private readonly IActorRef _instanceActor;
|
||||
private readonly IActorRef _self;
|
||||
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
||||
private readonly int _currentCallDepth;
|
||||
private readonly int _maxCallDepth;
|
||||
@@ -50,7 +49,7 @@ public class ScriptRuntimeContext
|
||||
|
||||
/// <summary>
|
||||
/// WaitForAttribute (spec §4.3): the per-script execution-timeout token from
|
||||
/// the owning <c>ScriptExecutionActor</c>/<c>AlarmExecutionActor</c>
|
||||
/// the owning script or alarm on-trigger run
|
||||
/// (<c>cts.Token</c>). Bounds the <c>Attributes.WaitAsync</c> Ask so a script
|
||||
/// that hits its own <c>ExecutionTimeoutSeconds</c> abandons the wait. Defaults
|
||||
/// to <see cref="CancellationToken.None"/> for contexts that do not thread one
|
||||
@@ -170,7 +169,6 @@ public class ScriptRuntimeContext
|
||||
/// execution, external system calls, database access, and notification delivery.
|
||||
/// </summary>
|
||||
/// <param name="instanceActor">Reference to the Instance Actor managing this instance's state.</param>
|
||||
/// <param name="self">Reference to the executing script actor.</param>
|
||||
/// <param name="sharedScriptLibrary">Library containing shared scripts available to all instances.</param>
|
||||
/// <param name="currentCallDepth">Current recursion depth of script calls.</param>
|
||||
/// <param name="maxCallDepth">Maximum allowed recursion depth before an error is thrown.</param>
|
||||
@@ -217,7 +215,6 @@ public class ScriptRuntimeContext
|
||||
/// </param>
|
||||
public ScriptRuntimeContext(
|
||||
IActorRef instanceActor,
|
||||
IActorRef self,
|
||||
SharedScriptLibrary sharedScriptLibrary,
|
||||
int currentCallDepth,
|
||||
int maxCallDepth,
|
||||
@@ -240,7 +237,6 @@ public class ScriptRuntimeContext
|
||||
CancellationToken scriptTimeoutToken = default)
|
||||
{
|
||||
_instanceActor = instanceActor;
|
||||
_self = self;
|
||||
_sharedScriptLibrary = sharedScriptLibrary;
|
||||
_currentCallDepth = currentCallDepth;
|
||||
_maxCallDepth = maxCallDepth;
|
||||
@@ -257,7 +253,7 @@ public class ScriptRuntimeContext
|
||||
_operationTrackingStore = operationTrackingStore;
|
||||
_cachedForwarder = cachedForwarder;
|
||||
// SourceNode-stamping: the local node name read from
|
||||
// INodeIdentityProvider at the ScriptExecutionActor; null when no
|
||||
// INodeIdentityProvider at the launching script run; null when no
|
||||
// provider was wired so the downstream callsites pass null through
|
||||
// verbatim — leaving central SourceNode as NULL.
|
||||
_sourceNode = sourceNode;
|
||||
@@ -265,7 +261,7 @@ public class ScriptRuntimeContext
|
||||
// (ParentExecutionId): stored verbatim — no `?? NewGuid()`
|
||||
// fallback. A non-routed run legitimately has no parent and stays null.
|
||||
_parentExecutionId = parentExecutionId;
|
||||
// Optional — null when not wired (tests / AlarmExecutionActor).
|
||||
// Optional — null when not wired (tests / alarm on-trigger runs).
|
||||
_siteEventLogger = siteEventLogger;
|
||||
// WaitForAttribute (spec §4.3): default(CancellationToken) == None when
|
||||
// not threaded in — the WaitAsync Ask is then bounded only by its own timeout.
|
||||
@@ -302,7 +298,6 @@ public class ScriptRuntimeContext
|
||||
{
|
||||
return new ScriptRuntimeContext(
|
||||
_instanceActor,
|
||||
_self,
|
||||
_sharedScriptLibrary,
|
||||
childCallDepth,
|
||||
_maxCallDepth,
|
||||
@@ -333,7 +328,7 @@ public class ScriptRuntimeContext
|
||||
/// <summary>
|
||||
/// Fire-and-forget emission of a <c>script</c> Error site event
|
||||
/// for a recursion-limit violation. Mirrors the call shape used by
|
||||
/// <c>ScriptExecutionActor</c>'s catch blocks. A fault from
|
||||
/// the run's own catch blocks. A fault from
|
||||
/// the site-event logger is observed-and-dropped (best-effort) via
|
||||
/// <c>ContinueWith(OnlyOnFaulted)</c> — it never blocks or faults the
|
||||
/// <c>_logger.LogError</c> + throw path that follows. A null logger is a no-op.
|
||||
@@ -1135,7 +1130,7 @@ public class ScriptRuntimeContext
|
||||
SourceSite: _siteId,
|
||||
// SourceNode-stamping: the local node name
|
||||
// (node-a/node-b) — threaded through INodeIdentityProvider
|
||||
// at the ScriptExecutionActor; null when no provider was
|
||||
// at the launching script run; null when no provider was
|
||||
// wired so central persists SiteCalls.SourceNode as NULL.
|
||||
SourceNode: _sourceNode,
|
||||
Status: "Submitted",
|
||||
@@ -1254,7 +1249,7 @@ public class ScriptRuntimeContext
|
||||
SourceSite: _siteId,
|
||||
// SourceNode-stamping: the local node name
|
||||
// (node-a/node-b) — threaded through INodeIdentityProvider
|
||||
// at the ScriptExecutionActor; null when no provider was
|
||||
// at the launching script run; null when no provider was
|
||||
// wired so central persists SiteCalls.SourceNode as NULL.
|
||||
SourceNode: _sourceNode,
|
||||
Status: "Attempted",
|
||||
@@ -1322,7 +1317,7 @@ public class ScriptRuntimeContext
|
||||
SourceSite: _siteId,
|
||||
// SourceNode-stamping: the local node name
|
||||
// (node-a/node-b) — threaded through INodeIdentityProvider
|
||||
// at the ScriptExecutionActor; null when no provider was
|
||||
// at the launching script run; null when no provider was
|
||||
// wired so central persists SiteCalls.SourceNode as NULL.
|
||||
SourceNode: _sourceNode,
|
||||
Status: operationalTerminalStatus,
|
||||
@@ -1913,7 +1908,7 @@ public class ScriptRuntimeContext
|
||||
SourceSite: _siteId,
|
||||
// SourceNode-stamping: the local node name
|
||||
// (node-a/node-b) — threaded through INodeIdentityProvider
|
||||
// at the ScriptExecutionActor; null when no provider was
|
||||
// at the launching script run; null when no provider was
|
||||
// wired so central persists SiteCalls.SourceNode as NULL.
|
||||
SourceNode: _sourceNode,
|
||||
Status: "Submitted",
|
||||
@@ -2260,7 +2255,7 @@ public class ScriptRuntimeContext
|
||||
OriginParentExecutionId: _parentExecutionId,
|
||||
// SourceNode-stamping: the cluster node name on which this
|
||||
// notification was emitted (node-a/node-b). Stamped from the local
|
||||
// INodeIdentityProvider via ScriptExecutionActor. Rides inside the
|
||||
// INodeIdentityProvider via the launching script run. Rides inside the
|
||||
// serialized payload through the S&F buffer to central, where
|
||||
// NotificationOutboxActor persists it on the Notifications row.
|
||||
SourceNode: _sourceNode);
|
||||
|
||||
@@ -80,7 +80,9 @@ public sealed class ScriptSchedulerStatsReporter : BackgroundService
|
||||
_collector.SetScriptSchedulerStats(
|
||||
scheduler.QueueDepth,
|
||||
scheduler.BusyThreadCount,
|
||||
scheduler.OldestBusyAge?.TotalSeconds);
|
||||
scheduler.OldestBusyAge?.TotalSeconds,
|
||||
// WP3.1: workers detached and replaced by the stuck-script watchdog.
|
||||
scheduler.DetachedThreadCount);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -26,19 +26,41 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
///
|
||||
/// <para>
|
||||
/// Bounded at <see cref="MaxEntries"/> entries — deliberately smaller than the verdict cache's
|
||||
/// 4096 because entries pin compiled assemblies, not just verdict strings. On overflow the cache
|
||||
/// is cleared wholesale (results are recomputable, so a coarse reset avoids eviction bookkeeping).
|
||||
/// <see cref="Hits"/>/<see cref="Count"/>/<see cref="Clear"/> are exposed for tests and diagnostics.
|
||||
/// 4096 because entries pin compiled assemblies, not just verdict strings.
|
||||
/// </para>
|
||||
///
|
||||
/// <para><b>WP3.1 — approximate LRU replaced the overflow cliff.</b> The cache used to
|
||||
/// <c>Clear()</c> wholesale on overflow. Instance scripts AND trigger expressions share this
|
||||
/// cache, so on a site large enough to cross 1024 entries every overflow discarded up to 1023
|
||||
/// live compiled scripts and the next deploy or Instance-Actor start paid a full recompile
|
||||
/// storm — on actor threads. Now an insert at the bound evicts only the oldest
|
||||
/// <see cref="EvictionBatchDivisor">⅛</see> of entries by last-access stamp: overflow costs one
|
||||
/// 1024-element scan instead of 1023 future recompiles, and hot entries survive.</para>
|
||||
///
|
||||
/// <para>Recency is an <see cref="Interlocked"/> access sequence, not a clock: it is
|
||||
/// deterministic for tests and immune to clock steps. Hits update the stamp lock-free; the
|
||||
/// only lock is the (rare) eviction sweep, double-checked so concurrent inserts do not
|
||||
/// stampede it.</para>
|
||||
/// </summary>
|
||||
internal static class SiteScriptCompileCache
|
||||
{
|
||||
/// <summary>Upper bound on cached entries; the cache is cleared wholesale on overflow.</summary>
|
||||
/// <summary>Upper bound on cached entries; crossing it evicts the oldest batch.</summary>
|
||||
internal const int MaxEntries = 1024;
|
||||
|
||||
private static readonly ConcurrentDictionary<string, ScriptCompilationResult> Cache = new();
|
||||
/// <summary>
|
||||
/// Fraction of the cache evicted in one sweep (⅛ = 128 entries at
|
||||
/// <see cref="MaxEntries"/>). Batching amortises the O(n) scan across many inserts, so
|
||||
/// steady-state churn does not re-scan on every single miss.
|
||||
/// </summary>
|
||||
private const int EvictionBatchDivisor = 8;
|
||||
|
||||
private static readonly ConcurrentDictionary<string, CacheEntry> Cache = new();
|
||||
private static readonly object EvictionLock = new();
|
||||
private static long _hits;
|
||||
|
||||
/// <summary>Monotonic access sequence; the recency stamp written onto entries.</summary>
|
||||
private static long _accessSequence;
|
||||
|
||||
/// <summary>Number of cache hits observed since the last <see cref="Clear"/>.</summary>
|
||||
public static long Hits => Interlocked.Read(ref _hits);
|
||||
|
||||
@@ -48,8 +70,8 @@ internal static class SiteScriptCompileCache
|
||||
/// <summary>
|
||||
/// Returns the cached compile result for <paramref name="code"/> against
|
||||
/// <paramref name="globalsType"/>, or computes it via <paramref name="factory"/> and caches
|
||||
/// it on a miss. A hit increments <see cref="Hits"/>. Both success and failure results are
|
||||
/// cached — the error text is name-free by construction.
|
||||
/// it on a miss. A hit increments <see cref="Hits"/> and refreshes the entry's recency.
|
||||
/// Both success and failure results are cached — the error text is name-free by construction.
|
||||
/// </summary>
|
||||
/// <param name="code">The script source code to look up (hashed to form the cache key).</param>
|
||||
/// <param name="globalsType">The Roslyn globals surface the script compiles against; part of the key so identical source under different globals stays distinct.</param>
|
||||
@@ -59,19 +81,19 @@ internal static class SiteScriptCompileCache
|
||||
{
|
||||
var key = globalsType.FullName + ":" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
|
||||
|
||||
if (Cache.TryGetValue(key, out var result))
|
||||
if (Cache.TryGetValue(key, out var entry))
|
||||
{
|
||||
Interlocked.Increment(ref _hits);
|
||||
return result;
|
||||
Touch(entry);
|
||||
return entry.Result;
|
||||
}
|
||||
|
||||
result = factory();
|
||||
var result = factory();
|
||||
|
||||
// Coarse bound: on overflow drop everything rather than track evictions.
|
||||
if (Cache.Count >= MaxEntries)
|
||||
Cache.Clear();
|
||||
EvictOldestBatch();
|
||||
|
||||
Cache[key] = result;
|
||||
Cache[key] = new CacheEntry(result) { LastAccess = NextStamp() };
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -81,4 +103,53 @@ internal static class SiteScriptCompileCache
|
||||
Cache.Clear();
|
||||
Interlocked.Exchange(ref _hits, 0);
|
||||
}
|
||||
|
||||
/// <summary>Refreshes an entry's recency stamp. Lock-free — a lost race only costs accuracy, never correctness.</summary>
|
||||
private static void Touch(CacheEntry entry) => Volatile.Write(ref entry.LastAccess, NextStamp());
|
||||
|
||||
private static long NextStamp() => Interlocked.Increment(ref _accessSequence);
|
||||
|
||||
/// <summary>
|
||||
/// Evicts the oldest <see cref="MaxEntries"/> / <see cref="EvictionBatchDivisor"/> entries
|
||||
/// by recency stamp. Double-checked under <see cref="EvictionLock"/> so several concurrent
|
||||
/// inserts crossing the bound together perform ONE sweep rather than one each.
|
||||
/// </summary>
|
||||
private static void EvictOldestBatch()
|
||||
{
|
||||
lock (EvictionLock)
|
||||
{
|
||||
if (Cache.Count < MaxEntries) return; // another thread already swept
|
||||
|
||||
var batch = Math.Max(1, MaxEntries / EvictionBatchDivisor);
|
||||
|
||||
// ConcurrentDictionary.ToArray() takes an ATOMIC snapshot. Enumerating (or
|
||||
// LINQ-ing) the dictionary directly does not: LINQ's ToArray picks the
|
||||
// ICollection<KeyValuePair<,>>.CopyTo fast path, which reads Count and then
|
||||
// copies, and throws ArgumentException when a concurrent insert lands between
|
||||
// the two. The eviction lock only excludes other EVICTORS — inserts run
|
||||
// lock-free by design — so the snapshot must be the thread-safe one.
|
||||
var victims = Cache.ToArray()
|
||||
.OrderBy(kvp => Volatile.Read(ref kvp.Value.LastAccess))
|
||||
.Take(batch)
|
||||
.Select(kvp => kvp.Key)
|
||||
.ToArray();
|
||||
|
||||
foreach (var victim in victims)
|
||||
Cache.TryRemove(victim, out _);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A cached compile result plus its recency stamp. A class (not a struct) so
|
||||
/// <see cref="Touch"/> can update recency in place without replacing the dictionary
|
||||
/// value — the hot path stays a single volatile write.
|
||||
/// </summary>
|
||||
private sealed class CacheEntry(ScriptCompilationResult result)
|
||||
{
|
||||
/// <summary>The memoised compile result (success or failure).</summary>
|
||||
public ScriptCompilationResult Result { get; } = result;
|
||||
|
||||
/// <summary>Access sequence number of the most recent hit; older = evicted first.</summary>
|
||||
public long LastAccess;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 (finding #4): the process-wide concurrency gate for trigger-expression evaluation.
|
||||
///
|
||||
/// <para>Before WP3.1, <c>ScriptActor</c> and <c>AlarmActor</c> evaluated their compiled
|
||||
/// Expression triggers on the <see cref="ScriptExecutionScheduler"/> — the same bounded pool
|
||||
/// of dedicated threads that runs script bodies. Eight scripts blocked in synchronous I/O
|
||||
/// therefore stalled EVERY Expression trigger on the node behind them, for an unbounded time,
|
||||
/// and the evaluation's own 2 s timeout did not even start until it was dequeued. An alarm
|
||||
/// that should have raised in milliseconds simply never raised.</para>
|
||||
///
|
||||
/// <para>Trigger expressions are non-blocking <em>by construction</em>:
|
||||
/// <see cref="TriggerExpressionGlobals"/> exposes only reads over an in-memory snapshot
|
||||
/// dictionary, and the script trust gate has already denied I/O, network, threading, and
|
||||
/// reflection long before the expression can deploy. They are short CPU-bound work — exactly
|
||||
/// what the shared .NET thread pool is for. So they run there, and this semaphore is the only
|
||||
/// thing bounding their fan-out. A second dedicated pool was considered and rejected: it would
|
||||
/// add threads, gauges, and a second starvation surface for no isolation gain.</para>
|
||||
///
|
||||
/// <para>Per-trigger coalescing in the actors (one evaluation in flight, one pending) already
|
||||
/// caps waiters at one per Expression trigger, so this gate's queue is bounded by trigger
|
||||
/// count.</para>
|
||||
///
|
||||
/// Mirrors <see cref="ScriptExecutionScheduler.Shared"/>'s lazy-singleton plus injectable-seam
|
||||
/// shape so tests can drive a gate of size 1 deterministically.
|
||||
/// </summary>
|
||||
public sealed class TriggerEvalGate : IDisposable
|
||||
{
|
||||
private readonly SemaphoreSlim _gate;
|
||||
|
||||
private static volatile TriggerEvalGate? _shared;
|
||||
private static readonly object SharedLock = new();
|
||||
|
||||
/// <summary>Creates a gate admitting <paramref name="maxConcurrency"/> concurrent evaluations.</summary>
|
||||
/// <param name="maxConcurrency">Maximum concurrent trigger-expression evaluations; values below 1 are clamped to 1.</param>
|
||||
public TriggerEvalGate(int maxConcurrency)
|
||||
{
|
||||
MaxConcurrency = Math.Max(1, maxConcurrency);
|
||||
_gate = new SemaphoreSlim(MaxConcurrency, MaxConcurrency);
|
||||
}
|
||||
|
||||
/// <summary>The configured concurrency limit.</summary>
|
||||
public int MaxConcurrency { get; }
|
||||
|
||||
/// <summary>Free permits right now; 0 means the gate is saturated and new evaluations will queue.</summary>
|
||||
public int AvailablePermits => _gate.CurrentCount;
|
||||
|
||||
/// <summary>
|
||||
/// The process-wide gate, used when no gate is injected. Lazily created from
|
||||
/// <see cref="SiteRuntimeOptions.TriggerEvalMaxConcurrency"/>; the first caller wins.
|
||||
/// </summary>
|
||||
/// <param name="options">Site runtime options supplying the concurrency limit.</param>
|
||||
/// <returns>The shared gate instance.</returns>
|
||||
public static TriggerEvalGate Shared(SiteRuntimeOptions options)
|
||||
{
|
||||
var existing = _shared;
|
||||
if (existing is not null) return existing;
|
||||
|
||||
lock (SharedLock)
|
||||
{
|
||||
return _shared ??= new TriggerEvalGate(options.TriggerEvalMaxConcurrency);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits for a permit. The caller passes the evaluation's own deadline token, which was
|
||||
/// armed at ENQUEUE — so time spent queueing here burns the same budget the evaluation
|
||||
/// itself would, and a saturated gate produces a timely cancellation (treated as
|
||||
/// <see langword="false"/> by both actors) instead of an unbounded stall.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The evaluation's deadline token.</param>
|
||||
/// <returns>A task that completes when a permit is acquired.</returns>
|
||||
public Task WaitAsync(CancellationToken cancellationToken) => _gate.WaitAsync(cancellationToken);
|
||||
|
||||
/// <summary>Returns a permit. Must be called exactly once per successful <see cref="WaitAsync"/>.</summary>
|
||||
public void Release() => _gate.Release();
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => _gate.Dispose();
|
||||
}
|
||||
Reference in New Issue
Block a user