dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scripting): out-of-process soft deadline measures the script's own run, not the pipe or a starved worker (#827)

Browse Source 
The soft deadline T was armed on the host at submit, so the pipe round trip
and the time a starved worker waits to be scheduled counted against the
script: under load a trivial script reported 'script timed out', in Shadow
mode that was a script mismatch, and at T + G the host killed the worker and
struck the key.

- Protocol (additive, negotiated): Hello/Ready.worker_soft_clock and
  EvalResults.soft_expired_eval_ids. protocol_version stays 1; a worker that
  does not echo the flag keeps the pre-#827 host clock.
- Worker: the watchdog runs each run's soft clock from the moment an
  evaluation thread starts it and reports the crossing (woken early for
  budgets shorter than its 50 ms poll). It still never cancels by itself;
  its fail-fast is anchored at the soft notice + G + margin.
- Host (PendingEval): the soft notice answers the caller and arms the kill G
  later. The host's own timer becomes a transport deadline, T + G from submit
  (restarted at a deferred run's start); passing it with no result and no
  notice yields TransportTimedOut -> Unavailable 'script worker unresponsive'
  (BadResourceUnavailable, recovery = the run's completion), no strike, no
  kill, no Cancel. A re-trigger refused by that still-unanswered run is
  Unavailable too, not 'previous run still running'. Counted as
  otopcua.script.ipc.timeouts{kind}.
- Frame writer: coalescing now carries soft_expired_eval_ids (it silently
  dropped them); the host processes soft notices before results in a frame.
- Shadow: unresponsive is shadow_unavailable (Unavailable), never a mismatch.

Tests: WorkerSoftClockTests suspends the real worker (SIGSTOP via libc kill /
NtSuspendProcess) - a stall past T still returns 7; a stall past T + G is a
runner fault with nothing struck; a late-starting runaway is still timed out
at T from its start and killed G after; Shadow stays match /
shadow_unavailable; both clocks enforce T and G; PendingEval state machine.
This commit is contained in:
Joseph Doherty
2026-11-11 01:38:46 -05:00
parent dea5b84af0
commit f7ca2c5826
11 changed files with 649 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs
+100 -7
View File
@@ -29,6 +29,9 @@ internal sealed class WorkerHost
/// <summary>The watchdog fails fast this long after the host's own hard deadline, so the host normally wins.</summary>
private static readonly TimeSpan WatchdogMargin = TimeSpan.FromMilliseconds(500);
/// <summary>The watchdog's longest sleep between passes.</summary>
private static readonly TimeSpan WatchdogPoll = TimeSpan.FromMilliseconds(50);
private readonly NamedPipeClientStream _pipe;
private readonly string _nonce;
private readonly int _parentPid;
@@ -75,6 +78,19 @@ internal sealed class WorkerHost
private TimeSpan _hardGrace = TimeSpan.FromSeconds(1);
private TimeSpan _heartbeatInterval = TimeSpan.FromSeconds(1);
private ScriptPeCacheStore? _peCache;
/// <summary>
/// lmxopcua#827: this worker runs each evaluation's soft clock from the run's start and reports the crossing
/// (<c>EvalResults.soft_expired_eval_ids</c>); the host asked for it in <c>Hello</c> and we echoed it in
/// <c>Ready</c>. Off (an older host): the host's own clock, started at submit, is the soft deadline.
/// </summary>
private bool _workerSoftClock;
/// <summary>Wakes the watchdog early when a run's soft deadline is sooner than its next planned pass.</summary>
private readonly AutoResetEvent _watchdogWake = new(false);
/// <summary><see cref="Stopwatch.GetTimestamp"/> of the watchdog's next planned pass.</summary>
private long _watchdogNextPass;
private long _seq;
public WorkerHost(NamedPipeClientStream pipe, string nonce, int parentPid)
@@ -109,6 +125,7 @@ internal sealed class WorkerHost
Nonce = _nonce,
Pid = Environment.ProcessId,
RuntimeVersion = Environment.Version.ToString(),
WorkerSoftClock = _workerSoftClock,
},
});
if (ScriptWorkerFrameCodec.UseBlockingIo) ScriptWorkerFrameCodec.Write(_pipe, ready, _maxFrameBytes);
@@ -140,6 +157,7 @@ internal sealed class WorkerHost
if (hello.HardGraceMs > 0) _hardGrace = TimeSpan.FromMilliseconds(hello.HardGraceMs);
if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs);
if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64);
_workerSoftClock = hello.WorkerSoftClock;
if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength)
{
// Design §7: the host owns the key; it reaches us only here, never through the environment or a file.
@@ -425,11 +443,17 @@ internal sealed class WorkerHost
var logger = _scriptLogger.ForContext(ForwardingLogSink.KeyProperty, request.Key);
var soft = TimeSpan.FromMilliseconds(Math.Max(1, request.SoftTimeoutMs));
// Cancelled by the host's Cancel frame at the soft deadline, never by our own clock (see the proto).
// Cancelled by the host's Cancel frame at the soft deadline, never by our own clock (see the proto): with the
// worker soft clock (lmxopcua#827) the watchdog only REPORTS the crossing; the host answers its caller, then
// cancels.
using var cts = new CancellationTokenSource();
var grace = request.HardGraceMs > 0 ? TimeSpan.FromMilliseconds(request.HardGraceMs) : _hardGrace;
var running = new RunningEval(request.Key, Stopwatch.GetTimestamp(), soft + grace + WatchdogMargin, cts);
var started = Stopwatch.GetTimestamp();
var running = new RunningEval(request.EvalId, request.Key, started, soft, grace, cts);
_running[(long)request.EvalId] = running;
// The watchdog plans its next pass at most WatchdogPoll ahead; a budget shorter than that wakes it now, so the
// soft notice is not late by up to a poll interval.
if (_workerSoftClock && running.SoftDue < Interlocked.Read(ref _watchdogNextPass)) _watchdogWake.Set();
var before = GC.GetAllocatedBytesForCurrentThread();
var sw = Stopwatch.StartNew();
try
@@ -465,22 +489,68 @@ internal sealed class WorkerHost
return result;
}
/// <summary>
/// The watchdog: reports soft-deadline crossings (worker soft clock, lmxopcua#827) and fails the process fast for
/// a run past its hard deadline that the host has not killed.
/// </summary>
/// <remarks>
/// The hard deadline it enforces is the host's plus <see cref="WatchdogMargin"/>, from the same anchor the host
/// uses: the run's start (T + G) on the host clock, or — with the worker soft clock — the soft notice it sent
/// (G), which is where the host arms its kill. So the host normally wins and the exit names the culprit.
/// </remarks>
private void WatchdogLoop()
{
while (true)
{
Thread.Sleep(50);
var now = Stopwatch.GetTimestamp();
var nextPass = now + Ticks(WatchdogPoll);
List<ulong>? expired = null;
foreach (var (_, run) in _running)
{
if (Stopwatch.GetElapsedTime(run.StartedTimestamp) > run.FailFastAfter)
if (_workerSoftClock)
{
// The host should have killed us already; this names the culprit in the exit/dump.
Environment.FailFast($"ScriptWorker: script '{run.Key}' exceeded its hard deadline ({run.FailFastAfter.TotalMilliseconds:F0} ms)");
if (run.SoftReportedAt == 0)
{
if (now >= run.SoftDue)
{
run.SoftReportedAt = now;
(expired ??= []).Add(run.EvalId);
}
else if (run.SoftDue < nextPass)
{
nextPass = run.SoftDue;
}
}
else if (now - run.SoftReportedAt > Ticks(run.Grace + WatchdogMargin))
{
FailFast(run, run.Grace + WatchdogMargin, "its soft deadline");
}
}
else if (now - run.StartedTimestamp > Ticks(run.Soft + run.Grace + WatchdogMargin))
{
FailFast(run, run.Soft + run.Grace + WatchdogMargin, "its start");
}
}
if (expired is not null)
{
var notice = new EvalResults();
notice.SoftExpiredEvalIds.AddRange(expired);
Send(new Envelope { Results = notice });
}
Interlocked.Exchange(ref _watchdogNextPass, nextPass);
var wait = Stopwatch.GetElapsedTime(Stopwatch.GetTimestamp(), nextPass);
if (wait > TimeSpan.Zero) _watchdogWake.WaitOne(wait);
}
static void FailFast(RunningEval run, TimeSpan after, string anchor)
{
// The host should have killed us already; this names the culprit in the exit/dump.
Environment.FailFast($"ScriptWorker: script '{run.Key}' exceeded its hard deadline ({after.TotalMilliseconds:F0} ms after {anchor})");
}
}
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
private async Task HeartbeatLoopAsync()
{
using var timer = new PeriodicTimer(_heartbeatInterval);
@@ -609,7 +679,30 @@ internal sealed class WorkerHost
Environment.Exit(ScriptWorkerEnvironment.ExitForbiddenAssembly);
}
private sealed record RunningEval(string Key, long StartedTimestamp, TimeSpan FailFastAfter, CancellationTokenSource Cancellation);
/// <summary>One evaluation on an evaluation thread, as the watchdog sees it.</summary>
private sealed class RunningEval(ulong evalId, string key, long startedTimestamp, TimeSpan soft, TimeSpan grace,
CancellationTokenSource cancellation)
{
public ulong EvalId { get; } = evalId;
public string Key { get; } = key;
/// <summary><see cref="Stopwatch.GetTimestamp"/> when the evaluation thread started the run.</summary>
public long StartedTimestamp { get; } = startedTimestamp;
/// <summary>T.</summary>
public TimeSpan Soft { get; } = soft;
/// <summary>The run's start + T.</summary>
public long SoftDue { get; } = startedTimestamp + Ticks(soft);
public TimeSpan Grace { get; } = grace;
public CancellationTokenSource Cancellation { get; } = cancellation;
/// <summary>When the watchdog reported the soft crossing to the host; 0 = not yet. Watchdog thread only.</summary>
public long SoftReportedAt { get; set; }
}
/// <summary>One loaded script: its compiled evaluator and its input slot order.</summary>
private sealed class LoadedScript : IDisposable