eb9269c337
A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
120 lines
5.0 KiB
C#
120 lines
5.0 KiB
C#
using Microsoft.Extensions.Logging;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;
|
|
|
|
/// <summary>
|
|
/// Periodic re-evaluation scheduler for calc tags carrying a <c>timerIntervalMs</c>. One
|
|
/// <see cref="System.Threading.Timer"/> per distinct interval group (so the wire count stays low
|
|
/// regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for
|
|
/// the same group is still running. Each tick invokes the driver's own evaluate
|
|
/// callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script
|
|
/// run, not just across the synchronous part of the callback, so a slow script still bounds its group
|
|
/// to one outstanding tick.
|
|
/// </summary>
|
|
internal sealed class CalculationTimerScheduler : IDisposable
|
|
{
|
|
private readonly Func<string, Task> _evaluate;
|
|
private readonly ILogger _logger;
|
|
private readonly List<Timer> _timers = [];
|
|
private readonly List<TickGroup> _groups = [];
|
|
private readonly CancellationTokenSource _cts = new();
|
|
private long _skippedTickCount;
|
|
private bool _disposed;
|
|
|
|
/// <summary>Initializes a new instance of the <see cref="CalculationTimerScheduler"/> class.</summary>
|
|
/// <param name="evaluate">Callback invoked with a calc tag's RawPath when its timer group ticks; awaited.</param>
|
|
/// <param name="logger">Logger for diagnostics.</param>
|
|
public CalculationTimerScheduler(Func<string, Task> evaluate, ILogger logger)
|
|
{
|
|
_evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
}
|
|
|
|
/// <summary>Number of timer callbacks that skipped their work because the prior tick for the same
|
|
/// group was still running. Monotonic; exposed for tests + operational metrics.</summary>
|
|
public long SkippedTickCount => Interlocked.Read(ref _skippedTickCount);
|
|
|
|
/// <summary>Stand up one <see cref="Timer"/> per distinct interval. All tags sharing an interval share
|
|
/// a timer; each tick re-evaluates every tag in the group.</summary>
|
|
/// <param name="tags">The calc tags to schedule (only those with a positive <see cref="CalculationTagDefinition.TimerInterval"/> are scheduled).</param>
|
|
public void Start(IEnumerable<CalculationTagDefinition> tags)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(tags);
|
|
if (_disposed) throw new ObjectDisposedException(nameof(CalculationTimerScheduler));
|
|
|
|
var byInterval = tags
|
|
.Where(t => t.TimerInterval is { } iv && iv > TimeSpan.Zero)
|
|
.GroupBy(t => t.TimerInterval!.Value);
|
|
|
|
foreach (var group in byInterval)
|
|
{
|
|
var paths = group.Select(t => t.RawPath).ToArray();
|
|
var interval = group.Key;
|
|
var ctx = new TickGroup(paths);
|
|
_groups.Add(ctx);
|
|
var timer = new Timer(_ => OnTimer(ctx), null, interval, interval);
|
|
_timers.Add(timer);
|
|
_logger.LogInformation("CalculationTimerScheduler: {TagCount} tag(s) on {Interval} cadence",
|
|
paths.Length, interval);
|
|
}
|
|
}
|
|
|
|
private void OnTimer(TickGroup ctx)
|
|
{
|
|
if (_cts.IsCancellationRequested) return;
|
|
|
|
// Skip the tick when the prior one for this group is still running — bounds the outstanding
|
|
// work to one tick per group regardless of how long an evaluation takes.
|
|
if (Interlocked.CompareExchange(ref ctx.InFlight, 1, 0) != 0)
|
|
{
|
|
Interlocked.Increment(ref _skippedTickCount);
|
|
return;
|
|
}
|
|
|
|
// The Timer callback is void; the tick itself is async so the awaited script runs do not pin this
|
|
// pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the
|
|
// Timer (which would be an unobserved fault on a timer thread).
|
|
_ = TickAsync(ctx);
|
|
}
|
|
|
|
private async Task TickAsync(TickGroup ctx)
|
|
{
|
|
try
|
|
{
|
|
foreach (var path in ctx.Paths)
|
|
{
|
|
if (_cts.IsCancellationRequested) return;
|
|
try { await _evaluate(path).ConfigureAwait(false); }
|
|
catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); }
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Exchange(ref ctx.InFlight, 0);
|
|
}
|
|
}
|
|
|
|
/// <summary>Releases all timers and disposes the scheduler's resources.</summary>
|
|
public void Dispose()
|
|
{
|
|
if (_disposed) return;
|
|
_disposed = true;
|
|
_cts.Cancel();
|
|
foreach (var t in _timers)
|
|
{
|
|
try { t.Dispose(); } catch { /* best-effort teardown */ }
|
|
}
|
|
_timers.Clear();
|
|
_groups.Clear();
|
|
_cts.Dispose();
|
|
}
|
|
|
|
private sealed class TickGroup(IReadOnlyList<string> paths)
|
|
{
|
|
// 0 = idle, 1 = a tick is currently running for this group.
|
|
public int InFlight;
|
|
/// <summary>The paths evaluated on each tick of this group.</summary>
|
|
public IReadOnlyList<string> Paths { get; } = paths;
|
|
}
|
|
}
|