dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scripting): quarantine a refusal behind a stuck orphan (fix round 2)

Browse Source 
Ruling refinement on I1. Only timeouts are strikes, so a never-returning
while(true), which times out once and is then only refused, was never
quarantined, and that defeated E1. A refusal now quarantines immediately
when the orphan it was refused behind has been executing for longer than
ScriptingOptions.StuckOrphanQuarantineFactor x the script's timeout
(default 10, so 20 s at the 2 s default; 0 or less disables it). It uses
the same quarantine path: one Error, otopcua.script.quarantined with
reason=stuck, Bad, lifted on source change or redeploy. A young orphan
(a short overrun) is still a plain refusal.

The guard stamps the run's start (Stopwatch timestamp) when the pool
picks it up and clears it when the run really finishes.

Tests: the 30-trigger while(true) regression (paced) now ends
quarantined with reason "stuck", still with exactly one orphan and a
healthy sibling. Positive control: a 2x overrun is refused, not
quarantined, and evaluates again once the orphan completes. A factor of
0 disables the rule. Docs (ScriptEditor "Runtime safety", VirtualTags)
and the seam remarks are updated.

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 13:59:10 -04:00
parent ef2eebe6b6
commit a85d1c069a
8 changed files with 133 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs
+32 -4
View File
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using Serilog;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
@@ -27,7 +28,10 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// consecutive timeouts</b> (only a timeout is a strike; a refusal is a Bad publish plus its counter
/// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a
/// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw —
/// resets the streak): no further dispatch, every attempt throws
/// resets the streak), <b>or on a refusal behind a stuck orphan</b> — one that has been executing for
/// longer than <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> × the timeout (20 s at the
/// 2 s default), which is how a never-returning <c>while(true)</c> (one timeout, then only refusals)
/// gets quarantined: no further dispatch, every attempt throws
/// <see cref="ScriptQuarantinedException"/>, one Error log, one <c>otopcua.script.quarantined</c>
/// increment, one <see cref="Quarantined"/> event. Lifted by a source change (the operator fixed the
/// script — a new source identity under the same key starts clean and the old orphan no longer blocks
@@ -63,11 +67,11 @@ public sealed class ScriptRunGuard
/// <summary>
/// Counter <c>otopcua.script.quarantined</c> (Prometheus: <c>otopcua_script_quarantined</c>): one
/// increment per script quarantined, tagged <c>reason</c> = <c>timeouts</c> | <c>allocation</c>.
/// increment per script quarantined, tagged <c>reason</c> = <c>timeouts</c> | <c>stuck</c> | <c>allocation</c>.
/// </summary>
public static readonly Counter<long> QuarantinedCounter =
OtOpcUaTelemetry.Meter.CreateCounter<long>("otopcua.script.quarantined", unit: "{script}",
description: "Scripts quarantined by the run guard (reason: timeouts | allocation).");
description: "Scripts quarantined by the run guard (reason: timeouts | stuck | allocation).");
/// <summary>
/// Counter <c>otopcua.script.run_refused</c>: runs refused because the script's previous run was
@@ -80,6 +84,7 @@ public sealed class ScriptRunGuard
private readonly ConcurrentDictionary<string, KeyState> _keys = new(StringComparer.Ordinal);
private readonly long _maxAllocBytesPerRun;
private readonly int _quarantineThreshold;
private readonly double _stuckFactor;
private readonly ILogger _logger;
private int _inFlight;
private long _started;
@@ -94,6 +99,7 @@ public sealed class ScriptRunGuard
options ??= new ScriptingOptions();
_maxAllocBytesPerRun = options.MaxAllocBytesPerRun;
_quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts);
_stuckFactor = options.StuckOrphanQuarantineFactor;
_logger = logger ?? Log.Logger;
}
@@ -185,6 +191,24 @@ public sealed class ScriptRunGuard
// ran long once. The refusal also leaves the timeout streak untouched.
Interlocked.Increment(ref _refused);
RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
// …unless the orphan is STUCK: a never-returning script times out once and is then only ever
// refused, so the timeout streak alone would never quarantine it. Past the stuck threshold the
// refusal itself quarantines (same path: Error log, metric, Bad, lifted on redeploy/source change).
if (_stuckFactor > 0 && Volatile.Read(ref state.RunStartedTimestamp) is var startedAt and not 0)
{
var running = Stopwatch.GetElapsedTime(startedAt);
var threshold = timeout * _stuckFactor;
if (running > threshold)
{
Quarantine(key, state,
$"its previous evaluation has been running for {running.TotalSeconds:F1} s, over " +
$"{_stuckFactor:0.##}x its {timeout.TotalMilliseconds:F0} ms budget " +
"(Scripting:StuckOrphanQuarantineFactor) — it is stuck; the script is not evaluated again " +
"until its source changes or it is redeployed",
"stuck");
throw new ScriptQuarantinedException(key, state.QuarantineReason!);
}
}
throw new ScriptRunRefusedException(key);
}
@@ -200,6 +224,7 @@ public sealed class ScriptRunGuard
// never complete, and the slot would never be released. The evaluator observes the token instead.
var runTask = Task.Run(async () =>
{
Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp());
started.TrySetResult();
var before = GC.GetAllocatedBytesForCurrentThread();
try
@@ -223,6 +248,7 @@ public sealed class ScriptRunGuard
// still quarantines its key here.
_ = runTask.ContinueWith(t =>
{
Volatile.Write(ref state.RunStartedTimestamp, 0);
Interlocked.Exchange(ref state.InFlight, 0);
Interlocked.Decrement(ref _inFlight);
runCts.Dispose();
@@ -339,6 +365,8 @@ public sealed class ScriptRunGuard
public string? SourceIdentity = sourceIdentity;
public int InFlight;
public int ConsecutiveTimeouts;
/// <summary><see cref="Stopwatch.GetTimestamp"/> when the pool started the in-flight run; 0 when idle.</summary>
public long RunStartedTimestamp;
public string? QuarantineReason;
}
@@ -373,7 +401,7 @@ public sealed class ScriptRunRefusedException : Exception
/// <summary>
/// Thrown by <see cref="ScriptRunGuard.RunAsync{TContext, TResult}"/> for a quarantined script: repeated
/// timeouts or an allocation-budget breach. Nothing is dispatched while quarantined; the seam
/// timeouts, a stuck orphan or an allocation-budget breach. Nothing is dispatched while quarantined; the seam
/// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or
/// redeploying.
/// </summary>
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs
+13
View File
@@ -25,6 +25,9 @@ public sealed class ScriptingOptions
/// <summary>Default <see cref="QuarantineAfterConsecutiveTimeouts"/>: 3.</summary>
public const int DefaultQuarantineAfterConsecutiveTimeouts = 3;
/// <summary>Default <see cref="StuckOrphanQuarantineFactor"/>: 10.</summary>
public const double DefaultStuckOrphanQuarantineFactor = 10;
/// <summary>
/// Managed bytes one script run may allocate on its own thread (sampled with
/// <see cref="GC.GetAllocatedBytesForCurrentThread"/> around the run) before the script is
@@ -40,4 +43,14 @@ public sealed class ScriptingOptions
/// throw) resets it. Default 3; values below 1 are treated as 1.
/// </summary>
public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts;
/// <summary>
/// A refusal quarantines the script immediately when the abandoned run it was refused behind has been
/// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is
/// what quarantines a permanently stuck script (<c>while(true)</c>): it times out once and is then
/// only ever refused, so the timeout streak alone would never reach
/// <see cref="QuarantineAfterConsecutiveTimeouts"/>. A young orphan (a short overrun) is still just a
/// refusal. Zero or negative disables the rule. Default 10.
/// </summary>
public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor;
}