feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)

A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:39:43 -04:00
parent ad2b5c4044
commit eb9269c337
17 changed files with 1313 additions and 74 deletions
@@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus
    // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync,
    // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse.
-    private readonly object _evalGate = new();
+    // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async —
+    // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's
+    // budget clock starts only once the pool has picked the script up, so pool starvation is never
+    // reported as a script timeout).
+    private readonly SemaphoreSlim _evalGate = new(1, 1);
+
+    // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight
+    // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false).
+    private readonly CancellationTokenSource _evalCts = new();

    // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single
    // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an
@@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        // _logger (ILogger<CalculationDriver>) implements the non-generic ILogger, so it is passed straight
        // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution
        // throws) land on the SAME logger the driver itself was constructed with, not a separate one.
-        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout);
+        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting);

        RebuildTagTable(_options);
    }
@@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                    {
                        RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
                        RunTimeout = _options.RunTimeout,
+                        Scripting = _options.Scripting,
                    };
                }
            }
@@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                    _driverInstanceId);
            }
        }
-        return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout };
+        return new CalculationDriverOptions
+        {
+            RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting,
+        };
    }

    /// <summary>Re-bind the tag table from <paramref name="driverConfigJson"/> — shared by
@@ -491,13 +503,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    {
        if (_pumpScheduled) return;
        _pumpScheduled = true;
-        _ = Task.Run(RunEvaluationPump);
+        _ = Task.Run(RunEvaluationPumpAsync);
    }

-    /// <summary>Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most
-    /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the
-    /// change path evaluated inline under the state lock.</summary>
-    private void RunEvaluationPump()
+    /// <summary>Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a
+    /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated
+    /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread
+    /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured
+    /// false timeouts under pool starvation).</summary>
+    private async Task RunEvaluationPumpAsync()
    {
        while (true)
        {
@@ -515,39 +529,50 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba

            // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still
            // evaluate one at a time while no host thread can block behind a script.
-            lock (_evalGate)
+            await _evalGate.WaitAsync().ConfigureAwait(false);
+            try
            {
-                try
-                {
-                    RunEvaluation(next);
-                }
-                catch (Exception ex)
-                {
-                    // The evaluator never throws for script faults, so this is a defect path only — it
-                    // must not kill the pump and strand every queued evaluation behind it.
-                    _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
-                        _driverInstanceId, next.Definition.RawPath);
-                }
+                await RunEvaluationAsync(next).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                // The evaluator never throws for script faults, so this is a defect path only — it
+                // must not kill the pump and strand every queued evaluation behind it.
+                _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
+                    _driverInstanceId, next.Definition.RawPath);
+            }
+            finally
+            {
+                _evalGate.Release();
            }
        }
    }

-    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's
-    /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work).</summary>
-    private void EvaluateTagFromTimer(string rawPath)
+    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
+    /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
+    /// completes).</summary>
+    private async Task EvaluateTagFromTimerAsync(string rawPath)
    {
        if (!_running) return;
        PendingEvaluation? prepared;
        lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow);
        if (prepared is null) return;
-        lock (_evalGate) RunEvaluation(prepared);
+        await _evalGate.WaitAsync().ConfigureAwait(false);
+        try
+        {
+            await RunEvaluationAsync(prepared).ConfigureAwait(false);
+        }
+        finally
+        {
+            _evalGate.Release();
+        }
    }

    /// <summary>Run one prepared evaluation and publish the outcome. MUST be called under
    /// <see cref="_evalGate"/> and NOT under <see cref="_lock"/>: the script run is the whole point of the
    /// split, and the script log + <see cref="OnDataChange"/> fan-out both invoke subscriber code.</summary>
    /// <param name="pending">The prepared evaluation.</param>
-    private void RunEvaluation(PendingEvaluation pending)
+    private async Task RunEvaluationAsync(PendingEvaluation pending)
    {
        var def = pending.Definition;

@@ -572,7 +597,8 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba

        // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are
        // then the input's real ones rather than a fabricated Good stamped at evaluation time.
-        var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies);
+        var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
+            .ConfigureAwait(false);

        string? failureReason = null;
        (object? Value, uint StatusCode)? toPublish;
@@ -688,7 +714,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    {
        var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray();
        if (timed.Length == 0) return;
-        _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger);
+        _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger);
        _timers.Start(timed);
    }

@@ -723,6 +749,9 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        _running = false;
        StopTimers();
        lock (_lock) _pending.Clear();
+        // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop
+        // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token.
+        try { _evalCts.Cancel(); } catch (ObjectDisposedException) { }
        _evaluator.Dispose();
    }

@@ -71,11 +71,22 @@ public static class CalculationDriverFactoryExtensions
            ?? throw new InvalidOperationException(
                $"Calculation driver config for '{driverInstanceId}' deserialised to null");

+        // Run-guard knobs are optional and default to the node-wide ScriptingOptions defaults; only an
+        // explicitly authored key overrides one (so a config carrying neither binds Scripting = null).
+        ScriptingOptions? scripting = null;
+        if (dto.MaxAllocBytesPerRun is not null || dto.QuarantineAfterConsecutiveTimeouts is not null)
+        {
+            scripting = new ScriptingOptions();
+            if (dto.MaxAllocBytesPerRun is { } alloc) scripting.MaxAllocBytesPerRun = alloc;
+            if (dto.QuarantineAfterConsecutiveTimeouts is { } n) scripting.QuarantineAfterConsecutiveTimeouts = n;
+        }
+
        var options = new CalculationDriverOptions
        {
            RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
            RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0
                ? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2),
+            Scripting = scripting,
        };

        return new CalculationDriver(
@@ -92,5 +103,11 @@ public static class CalculationDriverFactoryExtensions

        /// <summary>Optional per-script evaluation wall-clock budget override, in milliseconds (default 2000).</summary>
        public int? RunTimeoutMs { get; init; }
+
+        /// <summary>Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables).</summary>
+        public long? MaxAllocBytesPerRun { get; init; }
+
+        /// <summary>Optional run-guard quarantine threshold: consecutive non-completions before a tag's script stops being evaluated (default 3).</summary>
+        public int? QuarantineAfterConsecutiveTimeouts { get; init; }
    }
 }
@@ -1,20 +1,21 @@
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;

 namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;

-/// <summary>
-///     Bound options for a <see cref="CalculationDriver"/> instance. The Calculation pseudo-driver has
-///     no backend/endpoint config — its only input is the authored raw calc tags the deploy artifact
-///     injects as the <see cref="RawTags"/> list (each carrying the tag's <c>scriptId</c> + resolved
-///     <c>scriptSource</c> + trigger config inside its <see cref="RawTagEntry.TagConfig"/> blob).
-/// </summary>
+/// <summary>Bound options for one <see cref="CalculationDriver"/> instance.</summary>
 public sealed class CalculationDriverOptions
 {
-    /// <summary>The driver's authored raw calc tags (RawPath + TagConfig blob + WriteIdempotent), from
-    /// the deploy artifact. Each is mapped to a <see cref="CalculationTagDefinition"/> at construction.</summary>
+    /// <summary>The authored raw calc tags (RawPath + TagConfig JSON) this driver computes.</summary>
    public IReadOnlyList<RawTagEntry> RawTags { get; init; } = Array.Empty<RawTagEntry>();

-    /// <summary>Per-script wall-clock evaluation budget handed to the <see cref="CalculationEvaluator"/>.
-    /// Defaults to 2 seconds (parity with the VirtualTag evaluator).</summary>
+    /// <summary>Wall-clock budget per script run (driver-config <c>runTimeoutMs</c>); default 2 s.</summary>
    public TimeSpan RunTimeout { get; init; } = TimeSpan.FromSeconds(2);
+
+    /// <summary>
+    ///     Run-guard budget + quarantine threshold for this driver's scripts (driver-config
+    ///     <c>maxAllocBytesPerRun</c> / <c>quarantineAfterConsecutiveTimeouts</c>). Null ⇒ the
+    ///     <see cref="ScriptingOptions"/> defaults (64 MB, 3).
+    /// </summary>
+    public ScriptingOptions? Scripting { get; init; }
 }
@@ -37,8 +37,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
 ///     apply-boundary clear, which is what stops collectible <c>AssemblyLoadContext</c>s accreting across
 ///     script edits.
 /// </para>
+/// <para>
+///     <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
+///     calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous
+///     (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
+///     non-completions or an allocation-budget breach — both surface as a <see cref="VirtualTagEvalResult.Failure"/>
+///     the driver maps to Bad. The driver's evaluation pump awaits <see cref="EvaluateAsync"/> rather than
+///     blocking a pool thread on the sync <see cref="Evaluate(string, string, IReadOnlyDictionary{string, DataValueSnapshot})"/>,
+///     and the guard starts the budget clock only once the pool has picked the script up, so pool starvation
+///     is never reported as a script timeout.
+/// </para>
 /// </summary>
-public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
+public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
 {
    // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose()
    // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the
@@ -47,6 +57,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
    private readonly ILogger _logger;
    private readonly SerilogLogger _scriptRoot;
    private readonly TimeSpan _runTimeout;
+    private readonly ScriptRunGuard _guard;
    private bool _disposed;

    /// <summary>Initializes a new <see cref="CalculationEvaluator"/>.</summary>
@@ -56,16 +67,22 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
    /// implements this — instead of needing a second, separately-obtained logger instance.</param>
    /// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
    /// <param name="runTimeout">Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator).</param>
+    /// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
    public CalculationEvaluator(
        ILogger logger,
        ScriptRootLogger scriptRoot,
-        TimeSpan? runTimeout = null)
+        TimeSpan? runTimeout = null,
+        ScriptingOptions? scripting = null)
    {
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
        _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
+        _guard = new ScriptRunGuard(scripting, _scriptRoot);
    }

+    /// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
+    public ScriptRunGuard RunGuard => _guard;
+
    /// <summary>
    ///     Evaluate <paramref name="scriptSource"/> for the calc tag <paramref name="calcTagId"/> against
    ///     the snapshot in <paramref name="dependencies"/> (RawPath → last value). Never throws — script
@@ -112,8 +129,25 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
    /// <param name="scriptSource">The C# script source producing the tag's value.</param>
    /// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
    /// <returns>The evaluation outcome.</returns>
+    /// <remarks>Blocks the calling thread for the run. The driver's pump uses <see cref="EvaluateAsync"/>;
+    /// this overload remains for callers with no async context — chiefly tests.</remarks>
    public VirtualTagEvalResult Evaluate(
        string calcTagId, string scriptSource, IReadOnlyDictionary<string, DataValueSnapshot> dependencies)
+        => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult();
+
+    /// <summary>
+    ///     The async form of <see cref="Evaluate(string, string, IReadOnlyDictionary{string, DataValueSnapshot})"/>:
+    ///     the compiled script runs on the pool under the <see cref="ScriptRunGuard"/> and the caller awaits
+    ///     it instead of blocking a second pool thread for the duration. Never throws for script faults.
+    /// </summary>
+    /// <param name="calcTagId">Identity of the calc tag (bound to the script log).</param>
+    /// <param name="scriptSource">The C# script source producing the tag's value.</param>
+    /// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
+    /// <param name="ct">Caller cancellation (driver shutdown); wins over the run timeout.</param>
+    /// <returns>The evaluation outcome.</returns>
+    public async Task<VirtualTagEvalResult> EvaluateAsync(
+        string calcTagId, string scriptSource, IReadOnlyDictionary<string, DataValueSnapshot> dependencies,
+        CancellationToken ct = default)
    {
        if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed");
        if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression");
@@ -180,11 +214,12 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable

            try
            {
-                // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
-                // interrupt a CPU-bound/infinite-loop Roslyn script, so the timed wrapper is what bounds a
-                // runaway script to the wall-clock budget instead of wedging the driver's dispatch.
-                var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
-                var raw = timed.RunAsync(context).GetAwaiter().GetResult();
+                // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw
+                // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock
+                // budget is what bounds a runaway script — and the guard is what keeps the next trigger from
+                // stacking another orphan behind it (one per tag, then quarantine).
+                var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct)
+                    .ConfigureAwait(false);
                return VirtualTagEvalResult.Ok(raw);
            }
            catch (ScriptTimeoutException)
@@ -192,6 +227,18 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
                return VirtualTagEvalResult.Failure(
                    $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
            }
+            catch (ScriptRunRefusedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
+            catch (ScriptQuarantinedException ex)
+            {
+                return VirtualTagEvalResult.Failure(ex.Message);
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)");
+            }
            catch (ObjectDisposedException) when (!_disposed && attempt == 0)
            {
                // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the
@@ -212,7 +259,19 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
    }

    /// <inheritdoc />
-    public void ClearCompiledScripts() => _cache.Clear();
+    public void ClearCompiledScripts()
+    {
+        _cache.Clear();
+        // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine
+        // (an edited source already lifts its own through the guard's source identity).
+        _guard.ResetAll();
+    }
+
+    /// <inheritdoc />
+    public void ResetQuarantines() => _guard.ResetAll();
+
+    /// <inheritdoc />
+    public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;

    /// <summary>Disposes the evaluator, draining + disposing every cached compile's collectible ALC.</summary>
    public void Dispose()
@@ -7,11 +7,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
 ///     <see cref="System.Threading.Timer"/> per distinct interval group (so the wire count stays low
 ///     regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for
 ///     the same group is still running. Each tick invokes the driver's own evaluate
-///     callback per tag in the group.
+///     callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script
+///     run, not just across the synchronous part of the callback, so a slow script still bounds its group
+///     to one outstanding tick.
 /// </summary>
 internal sealed class CalculationTimerScheduler : IDisposable
 {
-    private readonly Action<string> _evaluate;
+    private readonly Func<string, Task> _evaluate;
    private readonly ILogger _logger;
    private readonly List<Timer> _timers = [];
    private readonly List<TickGroup> _groups = [];
@@ -20,9 +22,9 @@ internal sealed class CalculationTimerScheduler : IDisposable
    private bool _disposed;

    /// <summary>Initializes a new instance of the <see cref="CalculationTimerScheduler"/> class.</summary>
-    /// <param name="evaluate">Callback invoked with a calc tag's RawPath when its timer group ticks.</param>
+    /// <param name="evaluate">Callback invoked with a calc tag's RawPath when its timer group ticks; awaited.</param>
    /// <param name="logger">Logger for diagnostics.</param>
-    public CalculationTimerScheduler(Action<string> evaluate, ILogger logger)
+    public CalculationTimerScheduler(Func<string, Task> evaluate, ILogger logger)
    {
        _evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -69,12 +71,20 @@ internal sealed class CalculationTimerScheduler : IDisposable
            return;
        }

+        // The Timer callback is void; the tick itself is async so the awaited script runs do not pin this
+        // pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the
+        // Timer (which would be an unobserved fault on a timer thread).
+        _ = TickAsync(ctx);
+    }
+
+    private async Task TickAsync(TickGroup ctx)
+    {
        try
        {
            foreach (var path in ctx.Paths)
            {
                if (_cts.IsCancellationRequested) return;
-                try { _evaluate(path); }
+                try { await _evaluate(path).ConfigureAwait(false); }
                catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); }
            }
        }