fix(scripting): soft notice resumes the caller on the dispatcher; a late notice still cancels and counts (#827 review F1, F2)

F1: PendingEval.SoftExpired runs on the pipe reader and completed the
caller's outcome inline (production outcomes have no async continuations, by
design, because results resume on the ResultDispatcher). The caller's code
(the runner's Cancel write, the seam's publish) therefore ran on the reader,
which could hold up heartbeats (a false silence kill) and, with blocking I/O,
write to the pipe from the reader. The soft outcome is now posted to the
dispatcher like a result. Test: a 1.5 s continuation on a soft timeout with a
300 ms silence limit resumes on ScriptWorker[t].Results and the worker lives
(before: it resumed on the Reader).

F2: after a transport timeout ("worker unresponsive"), a late soft notice
armed G but nobody sent Cancel, so a polling script could not end itself and
the whole worker was killed at notice + G with a kill strike. PendingEval now
exposes LateSoftExpiry (asynchronous continuations, cancelled when the run
answers); the runner sends the Cancel and counts the timeout
(NoteTimeout) from it. A genuinely slow script whose notice is delayed past
T + G is thus still a timeout. Test: stall past T + G, thaw, a polling
script overruns T: the worker survives G and the key is quarantined for
timeouts, not a hard deadline (before: the worker was killed).
This commit is contained in:
Joseph Doherty
2026-11-11 02:21:45 -05:00
parent 30c6c3a0bf
commit c5dffbd8b8
3 changed files with 122 additions and 7 deletions
@@ -1128,6 +1128,7 @@ internal sealed class PendingEval
private bool _started;
private int _phase;
private int _done;
private TaskCompletionSource? _lateSoftExpiry;
public PendingEval(string key, ulong evalId, Action? onFinished = null, ResultDispatcher? dispatcher = null)
{
@@ -1160,6 +1161,16 @@ internal sealed class PendingEval
/// <summary>The worker's result, or null when the worker ended first.</summary>
public Task<EvalResult?> Completion => _completion.Task;
/// <summary>
/// After <see cref="TransportTimedOut"/> (null before): completes when the worker then reports the script itself
/// over T (slow AND late: it still gets its Cancel and counts as a timeout, #827 review F2), and is cancelled
/// when the worker answers or ends first. Continuations run asynchronously, never on the reader.
/// </summary>
internal Task? LateSoftExpiry
{
get { lock (_clock) return _lateSoftExpiry?.Task; }
}
/// <summary>
/// What the caller waits on: the worker's result, null when the worker ended first, <see cref="SoftTimedOut"/> at
/// the soft deadline, or <see cref="TransportTimedOut"/> at the transport deadline (the run itself is still
@@ -1246,7 +1257,7 @@ internal sealed class PendingEval
/// </summary>
internal void SoftExpired()
{
bool signal;
bool signal, late;
lock (_clock)
{
if (_timer is null || !_workerClock || Volatile.Read(ref _done) != 0) return;
@@ -1255,12 +1266,23 @@ internal sealed class PendingEval
_started = true;
_deferred = false;
signal = phase == Waiting;
late = phase == TransportReported;
Interlocked.Exchange(ref _phase, SoftReported);
_hardDue = Stopwatch.GetTimestamp() + Ticks(_grace);
TryChange(_grace);
}
// Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
if (signal) _outcome.TrySetResult(SoftTimedOut);
// This runs on the pipe READER. The caller's resumption is the caller's code (the seam's publish, the runner's
// Cancel write): it goes through the dispatcher exactly like a result (ResultDispatcher remarks), never inline
// here, or a slow consumer would hold up heartbeats and results (#827 review F1).
if (signal)
{
if (_dispatcher is { } dispatcher) dispatcher.Post(this, SoftTimedOut);
else _outcome.TrySetResult(SoftTimedOut);
}
// The caller was already answered "worker unresponsive"; the script has now itself run for T. Its watcher
// (the runner) cancels the run's token and counts the timeout, off this thread: the completion source runs its
// continuations asynchronously (#827 review F2).
if (late) _lateSoftExpiry?.TrySetResult();
}
private void OnTimer()
@@ -1277,6 +1299,7 @@ internal sealed class PendingEval
// A tick armed before a deferral/start can land early: re-arm for what is left.
if (now < _transportDue) { TryChange(Stopwatch.GetElapsedTime(now, _transportDue)); return; }
Interlocked.Exchange(ref _phase, TransportReported);
_lateSoftExpiry = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
transport = true;
}
else if (phase == Waiting)
@@ -1316,6 +1339,7 @@ internal sealed class PendingEval
{
if (Interlocked.Exchange(ref _done, 1) != 0) return;
_timer?.Dispose();
lock (_clock) _lateSoftExpiry?.TrySetCanceled();
try { _onFinished?.Invoke(); }
finally
{