fix(scripting): #789 review — reader faults end the worker, write stalls kill it, Cancel off the worker reader
- Host reader: the frame read and its handling are caught separately. Only the read's I/O exceptions mean the pipe ended; anything HandleFrame's inline completion work throws now kills that worker as ProtocolFault (outstanding runs -> worker restarted / BadResourceUnavailable) instead of escaping a dedicated thread (process crash) or being swallowed as a clean end (an InvalidOperationException used to be, leaving the worker unread until a Crash 2 s later). - Worker reader: the same catch-all around HandleFrame (exit ProtocolFault). - Worker: Cancel() runs on the pool, so a script's blocking ctx.CancellationToken callback cannot stop the worker reading its pipe. - Host write-stall check: ScriptWorkerFrameWriter.WriteInProgressFor; the heartbeat monitor kills a worker whose pipe write has been blocked longer than max(10 s, silence limit) as KilledUnresponsive, which fails the write and releases the producer (seam submit, deploy load, Cancel). Fixes the wrong 'heartbeat monitor ends it' comment on the pipe buffer quota. - StopAsync sends Shutdown off the caller so its 1 s budget holds on a full pipe. - WindowsBlockingPipeStream refuses the async surface (Stream's base versions serialise a write behind a waiting read). - Tests: PipeFaultTests (throwing completion callback -> ProtocolFault and a BadResourceUnavailable sibling, shard serves again; a worker that stops reading but heartbeats is killed within the bound and the blocked producer returns, via the Hello.test_stall_reader_on_diag seam); BlockingPipeStreamTests dispose with a write blocked on a full pipe, and with a read and a write both pending; Category=ScriptWorker on the protocol, handshake and pipe test classes. Mutation checks: no stall kill -> stall test red; catch-all without the fault -> callback test red (Crash). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -94,6 +94,9 @@ internal sealed class WorkerHost
|
||||
private long _watchdogNextPass;
|
||||
private long _seq;
|
||||
|
||||
/// <summary>Test seam (<c>Hello.test_stall_reader_on_diag</c>).</summary>
|
||||
private bool _testStallReaderOnDiag;
|
||||
|
||||
public WorkerHost(Stream pipe, string nonce, int parentPid)
|
||||
{
|
||||
_pipe = pipe;
|
||||
@@ -157,6 +160,7 @@ internal sealed class WorkerHost
|
||||
if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs);
|
||||
if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64);
|
||||
_workerSoftClock = hello.WorkerSoftClock;
|
||||
_testStallReaderOnDiag = hello.TestStallReaderOnDiag;
|
||||
if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength)
|
||||
{
|
||||
// Design §7: the host owns the key; it reaches us only here, never through the environment or a file.
|
||||
@@ -205,9 +209,21 @@ internal sealed class WorkerHost
|
||||
done.TrySetResult(ScriptWorkerEnvironment.ExitPipeClosed);
|
||||
return;
|
||||
}
|
||||
if (HandleFrame(frame) is { } exit)
|
||||
int? exit;
|
||||
try
|
||||
{
|
||||
done.TrySetResult(exit);
|
||||
exit = HandleFrame(frame);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Never an unhandled exception on this thread: the worker ends as a fault the host can attribute.
|
||||
Console.Error.WriteLine($"ScriptWorker: reader failed: {ex}");
|
||||
done.TrySetResult(ScriptWorkerEnvironment.ExitProtocolFault);
|
||||
return;
|
||||
}
|
||||
if (exit is { } code)
|
||||
{
|
||||
done.TrySetResult(code);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -250,10 +266,18 @@ internal sealed class WorkerHost
|
||||
case Envelope.BodyOneofCase.Cancel:
|
||||
if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
|
||||
{
|
||||
try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ }
|
||||
// Off the reader (lmxopcua#789): Cancel() runs the script's own ctx.CancellationToken callbacks, and a
|
||||
// callback that blocks would stop this worker reading its pipe — the host's writes would then block.
|
||||
ThreadPool.UnsafeQueueUserWorkItem(static cts =>
|
||||
{
|
||||
try { cts.Cancel(); }
|
||||
catch (ObjectDisposedException) { /* just finished */ }
|
||||
catch (Exception ex) { Console.Error.WriteLine($"ScriptWorker: a cancellation callback threw: {ex.Message}"); }
|
||||
}, toCancel.Cancellation, preferLocal: false);
|
||||
}
|
||||
return null;
|
||||
case Envelope.BodyOneofCase.DiagRequest:
|
||||
if (_testStallReaderOnDiag) Thread.Sleep(Timeout.Infinite); // test seam: talk but never listen again
|
||||
Send(new Envelope { DiagReply = Diagnostics() });
|
||||
return null;
|
||||
case Envelope.BodyOneofCase.Shutdown:
|
||||
|
||||
Reference in New Issue
Block a user