dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scripting): out-of-process soft deadline measures the script's own run, not the pipe or a starved worker (#827)

Browse Source 
The soft deadline T was armed on the host at submit, so the pipe round trip
and the time a starved worker waits to be scheduled counted against the
script: under load a trivial script reported 'script timed out', in Shadow
mode that was a script mismatch, and at T + G the host killed the worker and
struck the key.

- Protocol (additive, negotiated): Hello/Ready.worker_soft_clock and
  EvalResults.soft_expired_eval_ids. protocol_version stays 1; a worker that
  does not echo the flag keeps the pre-#827 host clock.
- Worker: the watchdog runs each run's soft clock from the moment an
  evaluation thread starts it and reports the crossing (woken early for
  budgets shorter than its 50 ms poll). It still never cancels by itself;
  its fail-fast is anchored at the soft notice + G + margin.
- Host (PendingEval): the soft notice answers the caller and arms the kill G
  later. The host's own timer becomes a transport deadline, T + G from submit
  (restarted at a deferred run's start); passing it with no result and no
  notice yields TransportTimedOut -> Unavailable 'script worker unresponsive'
  (BadResourceUnavailable, recovery = the run's completion), no strike, no
  kill, no Cancel. A re-trigger refused by that still-unanswered run is
  Unavailable too, not 'previous run still running'. Counted as
  otopcua.script.ipc.timeouts{kind}.
- Frame writer: coalescing now carries soft_expired_eval_ids (it silently
  dropped them); the host processes soft notices before results in a frame.
- Shadow: unresponsive is shadow_unavailable (Unavailable), never a mismatch.

Tests: WorkerSoftClockTests suspends the real worker (SIGSTOP via libc kill /
NtSuspendProcess) - a stall past T still returns 7; a stall past T + G is a
runner fault with nothing struck; a late-starting runaway is still timed out
at T from its start and killed G after; Shadow stays match /
shadow_unavailable; both clocks enforce T and G; PendingEval state machine.
This commit is contained in:
Joseph Doherty
2026-11-11 01:38:46 -05:00
parent dea5b84af0
commit f7ca2c5826
11 changed files with 649 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs
+42 -7
View File
@@ -100,6 +100,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
"otopcua.script.eval.duration", unit: "us",
description: "Worker-measured run time of one out-of-process evaluation (tag: kind).");
private static readonly Counter<long> TransportTimeoutCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
"otopcua.script.ipc.timeouts", unit: "{eval}",
description: "Out-of-process evaluations answered 'script worker unresponsive': no result and no soft-deadline notice from the worker within T + G of submit — a starved or stalled worker, not a slow script (tag: kind; lmxopcua#827).");
static OutOfProcessScriptRunner()
{
// Gauges observe every live runner (one per node in production; tests make several).
@@ -372,11 +376,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
return ScriptRunOutcome.Failed(knownFailure);
}
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
{
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
return ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
}
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) return Refused(key, state);
var handedOff = false;
SemaphoreSlim? gate = null;
@@ -533,8 +533,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
{
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
answered = Refused(key, state);
return BatchPreparation.Answered;
}
state.ScriptLogger = invocation.ScriptLogger;
@@ -559,6 +558,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
{
if (Interlocked.Exchange(ref released, 1) != 0) return;
SlotReleasingForTest?.Invoke(key);
Volatile.Write(ref state.StalledRun, null);
Interlocked.Exchange(ref state.InFlight, 0);
heldGate?.Release();
};
@@ -583,6 +583,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
return ScriptRunOutcome.Failed($"{ScriptEvalFailureReasons.TimedOutPrefix}{invocation.Timeout.TotalSeconds:F1}s");
}
if (ReferenceEquals(result, PendingEval.TransportTimedOut))
{
// lmxopcua#827: T + G passed with neither a result nor a soft-deadline notice. The worker runs the soft
// clock from the run's start, so this is the pipe or a starved / stalled worker — the runner's fault,
// never the script's: no timeout strike, no Cancel (the run may not even have started), and the slot
// stays with the run until the worker answers. The seam re-evaluates when it does (or when the worker
// is replaced, if the heartbeat monitor kills it).
TransportTimeoutCounter.Add(1, new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind)));
// Read by a refusal only while it is incomplete, so a write that loses a race with the slot release
// is harmless.
Volatile.Write(ref state.StalledRun, pending.Completion);
if (ct.IsCancellationRequested) return ScriptRunOutcome.Failed("evaluation cancelled");
return ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerUnresponsiveReason) with { Recovery = pending.Completion };
}
if (result is null) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
var kindTag = new KeyValuePair<string, object?>("kind", KindTag(invocation.Kind));
RoundTripHistogram.Record(Stopwatch.GetElapsedTime(submitted).TotalMicroseconds, kindTag);
@@ -717,6 +732,20 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
private static ScriptRunOutcome Unavailable(string reason, WorkerShard shard, WorkerConnection? failed)
=> ScriptRunOutcome.Unavailable(reason) with { Recovery = shard.WhenRecoveredAsync(failed) };
/// <summary>
/// The answer to an evaluation whose key's previous run still holds its single-flight slot: the usual
/// script-attributable refusal — unless that run was answered "worker unresponsive" (lmxopcua#827) and the
/// worker still has not answered it, in which case nothing says the script is slow, and the refusal is the
/// runner's fault too (re-evaluated when the stalled run completes).
/// </summary>
private static ScriptRunOutcome Refused(string key, KeyState state)
{
if (Volatile.Read(ref state.StalledRun) is { IsCompleted: false } stalled)
return ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerUnresponsiveReason) with { Recovery = stalled };
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
return ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
}
private void NoteLoad(WorkerConnection.LoadEntry entry)
{
// Counted once per compile, by the caller that waited for it (an already-loaded key is not a load).
@@ -986,5 +1015,11 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
public volatile bool Suspect;
public string? QuarantineReason;
public ILogger? ScriptLogger;
/// <summary>
/// The completion of this key's in-flight run while it is answered "worker unresponsive" and not yet answered
/// by the worker (lmxopcua#827); null otherwise. Cleared by the slot release.
/// </summary>
public Task? StalledRun;
}
}