fix(scripting): #789 review — reader faults end the worker, write stalls kill it, Cancel off the worker reader

- Host reader: the frame read and its handling are caught separately. Only the
  read's I/O exceptions mean the pipe ended; anything HandleFrame's inline
  completion work throws now kills that worker as ProtocolFault (outstanding
  runs -> worker restarted / BadResourceUnavailable) instead of escaping a
  dedicated thread (process crash) or being swallowed as a clean end (an
  InvalidOperationException used to be, leaving the worker unread until a Crash
  2 s later).
- Worker reader: the same catch-all around HandleFrame (exit ProtocolFault).
- Worker: Cancel() runs on the pool, so a script's blocking
  ctx.CancellationToken callback cannot stop the worker reading its pipe.
- Host write-stall check: ScriptWorkerFrameWriter.WriteInProgressFor; the
  heartbeat monitor kills a worker whose pipe write has been blocked longer than
  max(10 s, silence limit) as KilledUnresponsive, which fails the write and
  releases the producer (seam submit, deploy load, Cancel). Fixes the wrong
  'heartbeat monitor ends it' comment on the pipe buffer quota.
- StopAsync sends Shutdown off the caller so its 1 s budget holds on a full pipe.
- WindowsBlockingPipeStream refuses the async surface (Stream's base versions
  serialise a write behind a waiting read).
- Tests: PipeFaultTests (throwing completion callback -> ProtocolFault and a
  BadResourceUnavailable sibling, shard serves again; a worker that stops reading
  but heartbeats is killed within the bound and the blocked producer returns,
  via the Hello.test_stall_reader_on_diag seam); BlockingPipeStreamTests dispose
  with a write blocked on a full pipe, and with a read and a write both pending;
  Category=ScriptWorker on the protocol, handshake and pipe test classes.
  Mutation checks: no stall kill -> stall test red; catch-all without the fault
  -> callback test red (Crash).

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 20:26:02 -05:00
parent d4a2aa6142
commit 03ac57d5d6
10 changed files with 345 additions and 19 deletions
@@ -12,6 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// <see cref="WindowsBlockingPipeStream"/> that keeps worker pipe I/O off the completion port. Exercised on a real
/// pipe pair; on Unix the same tests cover the plain pipe it returns there.
/// </summary>
[Trait("Category", "ScriptWorker")]
public sealed class BlockingPipeStreamTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
@@ -144,4 +145,85 @@ public sealed class BlockingPipeStreamTests
ended.ShouldBe(0);
}
}
/// <summary>
/// Writes until the pipe is full (the peer never reads), then returns the writing task, which is blocked in the
/// kernel at that point; it ends with the exception the write saw.
/// </summary>
private static async Task<Task> BlockedWriterAsync(Stream end)
{
long written = 0;
var writer = Task.Factory.StartNew(() =>
{
var chunk = new byte[64 * 1024];
while (true)
{
end.Write(chunk, 0, chunk.Length);
Interlocked.Add(ref written, chunk.Length);
}
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
// Full once no progress is made for a while (a zero-quota Windows pipe blocks at once; a socket after its buffer).
long last;
do
{
last = Interlocked.Read(ref written);
await Task.Delay(300, Ct);
} while (Interlocked.Read(ref written) != last && !writer.IsCompleted);
writer.IsCompleted.ShouldBeFalse("positive control: the write is blocked on a full pipe");
return writer;
}
private static async Task ShouldEndAsync(Task blocked, string what)
{
var ended = await Task.WhenAny(blocked, Task.Delay(TimeSpan.FromSeconds(10), Ct));
ended.ShouldBeSameAs(blocked, $"{what} ended, not hung");
var ex = blocked.Exception?.GetBaseException();
ex.ShouldNotBeNull($"{what} ended with an error");
(ex is ObjectDisposedException or IOException).ShouldBeTrue(ex.ToString());
}
[Fact]
public async Task Disposing_an_end_wakes_its_own_write_blocked_on_a_full_pipe()
{
var (server, client) = await PairAsync();
using (client)
{
var writer = await BlockedWriterAsync(server);
server.Dispose();
await ShouldEndAsync(writer, "the blocked write");
}
}
[Fact]
public async Task Disposing_an_end_with_a_read_and_a_write_both_pending_wakes_both()
{
var (server, client) = await PairAsync();
using (client)
{
var read = Task.Factory.StartNew(() =>
{
var n = server.Read(new byte[16], 0, 16);
if (n == 0) throw new IOException("end of stream");
return n;
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var writer = await BlockedWriterAsync(server);
read.IsCompleted.ShouldBeFalse("positive control: the read is pending beside the blocked write");
server.Dispose();
await ShouldEndAsync(writer, "the blocked write");
await ShouldEndAsync(read, "the pending read");
}
}
[Fact]
public async Task The_asynchronous_surface_is_refused_on_windows()
{
var (server, client) = await PairAsync();
using (server)
using (client)
{
if (!OperatingSystem.IsWindows()) return; // the Unix pipe itself keeps PipeStream's own async surface
await Should.ThrowAsync<NotSupportedException>(() => server.ReadAsync(new byte[4], 0, 4, Ct));
await Should.ThrowAsync<NotSupportedException>(async () => await server.WriteAsync(new byte[4].AsMemory(), Ct));
}
}
}