perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

This commit is contained in:
Joseph Doherty
2026-08-14 22:36:15 -04:00
parent 312216ff2b
commit c4fc1f8ecd
42 changed files with 3673 additions and 1251 deletions
@@ -38,13 +38,64 @@ public class SiteRuntimeOptions
public int StreamBufferSize { get; set; } = 1000;
/// <summary>
/// Number of dedicated threads in the script-execution scheduler.
/// FLOOR for the number of dedicated threads in the script-execution scheduler.
/// Script and alarm on-trigger bodies run on these threads instead of the shared
/// .NET thread pool, so blocking script I/O cannot starve the global pool.
///
/// <para>WP3.1: this was a fixed size and is now the lower bound of a grow-only,
/// instance-scaled pool — see
/// <see cref="Scripts.ScriptExecutionScheduler.ComputeTargetThreads"/>. Existing
/// configurations keep exactly the previous behaviour at or below
/// <c>ScriptExecutionThreadCount * InstancesPerScriptThread</c> deployed instances.</para>
///
/// Default: 8.
/// </summary>
public int ScriptExecutionThreadCount { get; set; } = 8;
/// <summary>
/// WP3.1: CEILING for the instance-scaled script-execution pool. The pool grows
/// towards <c>ceil(enabledInstances / 8)</c> threads but never past this value; beyond
/// it, <see cref="MaxConcurrentRunsPerScript"/> is the real regulator. A 1 MB-stack
/// dedicated thread is cheap, so 32 (≈ 256 instances at the /8 ratio) is a generous
/// default. Must be greater than or equal to <see cref="ScriptExecutionThreadCount"/>.
/// Default: 32.
/// </summary>
public int ScriptExecutionMaxThreadCount { get; set; } = 32;
/// <summary>
/// WP3.1 (finding #4): maximum number of trigger-expression evaluations allowed to run
/// concurrently across the whole process. Trigger expressions are non-blocking by
/// construction (<see cref="Scripts.TriggerExpressionGlobals"/> exposes only reads over an
/// in-memory snapshot, and the script trust gate has already denied I/O, network, and
/// threading), so they run as plain async work on the shared .NET thread pool behind this
/// gate rather than on the blocking script-execution pool. That is what keeps an alarm's
/// Expression trigger from queueing behind blocking script bodies.
/// Default: <c>max(2, Environment.ProcessorCount)</c>.
/// </summary>
public int TriggerEvalMaxConcurrency { get; set; } = Math.Max(2, Environment.ProcessorCount);
/// <summary>
/// WP3.1: timeout (seconds) for a single trigger-expression evaluation. Previously
/// hardcoded at 2 s in both <c>ScriptActor</c> and <c>AlarmActor</c>. The deadline is now
/// armed when the evaluation is ENQUEUED, not when it is dequeued, so time spent waiting
/// on <see cref="TriggerEvalMaxConcurrency"/> burns the same budget — a saturated gate
/// produces a timely "false" instead of an unbounded stall.
/// Default: 2.
/// </summary>
public int TriggerEvalTimeoutSeconds { get; set; } = 2;
/// <summary>
/// WP3.1: maximum number of concurrent runs (queued or executing) for any single script
/// or alarm on-trigger script. A trigger that arrives while the cap is reached is SHED —
/// the newest run is refused, the four already in flight (which are closest to their own
/// deadlines) are kept, and no reordering occurs. A shed increments
/// <c>ISiteHealthCollector.IncrementScriptRunShed</c>, emits a rate-limited Warning site
/// event, and — for an Ask-based <c>CallScript</c> — replies with an explicit error rather
/// than letting the caller hang to its Ask timeout.
/// Default: 4.
/// </summary>
public int MaxConcurrentRunsPerScript { get; set; } = 4;
/// <summary>
/// Max mirrored native alarms retained per source binding before older entries are dropped (logged).
/// Default: 1000.