From 03ac57d5d6141495fc92dc4f8a619a354f8751f6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 11 Nov 2026 20:26:02 -0500 Subject: [PATCH] =?UTF-8?q?fix(scripting):=20#789=20review=20=E2=80=94=20r?= =?UTF-8?q?eader=20faults=20end=20the=20worker,=20write=20stalls=20kill=20?= =?UTF-8?q?it,=20Cancel=20off=20the=20worker=20reader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Host reader: the frame read and its handling are caught separately. Only the read's I/O exceptions mean the pipe ended; anything HandleFrame's inline completion work throws now kills that worker as ProtocolFault (outstanding runs -> worker restarted / BadResourceUnavailable) instead of escaping a dedicated thread (process crash) or being swallowed as a clean end (an InvalidOperationException used to be, leaving the worker unread until a Crash 2 s later). - Worker reader: the same catch-all around HandleFrame (exit ProtocolFault). - Worker: Cancel() runs on the pool, so a script's blocking ctx.CancellationToken callback cannot stop the worker reading its pipe. - Host write-stall check: ScriptWorkerFrameWriter.WriteInProgressFor; the heartbeat monitor kills a worker whose pipe write has been blocked longer than max(10 s, silence limit) as KilledUnresponsive, which fails the write and releases the producer (seam submit, deploy load, Cancel). Fixes the wrong 'heartbeat monitor ends it' comment on the pipe buffer quota. - StopAsync sends Shutdown off the caller so its 1 s budget holds on a full pipe. - WindowsBlockingPipeStream refuses the async surface (Stream's base versions serialise a write behind a waiting read). - Tests: PipeFaultTests (throwing completion callback -> ProtocolFault and a BadResourceUnavailable sibling, shard serves again; a worker that stops reading but heartbeats is killed within the bound and the blocked producer returns, via the Hello.test_stall_reader_on_diag seam); BlockingPipeStreamTests dispose with a write blocked on a full pipe, and with a read and a write both pending; Category=ScriptWorker on the protocol, handshake and pipe test classes. Mutation checks: no stall kill -> stall test red; catch-all without the fault -> callback test red (Crash). Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk --- .../WorkerConnection.cs | 82 ++++++++++++-- .../WorkerPipeSecurity.cs | 3 +- .../Protos/script_worker.proto | 3 + .../ScriptWorkerFramePipe.cs | 26 ++++- .../WindowsBlockingPipeStream.cs | 30 +++++ .../WorkerHost.cs | 30 ++++- .../BlockingPipeStreamTests.cs | 82 ++++++++++++++ .../HandshakeTests.cs | 1 + .../PipeFaultTests.cs | 106 ++++++++++++++++++ .../ProtocolTests.cs | 1 + 10 files changed, 345 insertions(+), 19 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs index 4d3c321c2..4c5894c97 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs @@ -91,6 +91,10 @@ internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan /// How long the worker may send nothing before it is killed (); null = /// three s. Test seam (with ). /// +/// +/// Test seam: the worker's reader stops reading for good at the first diagnostics request while its heartbeats go on — +/// a worker that keeps talking but stopped listening (the write-stall check, lmxopcua#789). +/// /// /// Ask the worker to run the soft clock from each run's start (lmxopcua#827; see ). /// False (tests only) keeps the pre-#827 host clock, which is also what a worker that does not echo it gets. @@ -110,7 +114,8 @@ internal sealed record WorkerLaunchSpec( int CompileThreads = 0, string? WorkerSid = null, bool WorkerSoftClock = true, - TimeSpan? HeartbeatSilenceLimit = null) + TimeSpan? HeartbeatSilenceLimit = null, + bool TestStallReaderOnDiag = false) { /// The default compile timeout. public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60); @@ -265,6 +270,20 @@ internal sealed class WorkerConnection : IAsyncDisposable /// internal TimeSpan? ProgressCeilingOverride { get; set; } + /// + /// How long one pipe write may stay blocked before the worker is killed (, + /// lmxopcua#789). Writes are blocking, so a full pipe — a worker whose reader stopped while its heartbeats go on — + /// would otherwise hold whichever producer is draining the writer (a seam submitting, a deploy loading scripts, a + /// Cancel) forever: the silence check never fires, and with only loads queued nothing else does. A worker that is + /// reading drains the 1 MiB buffer far faster than this. Null = the larger of 10 s and the silence limit; set = an + /// override (tests). + /// + internal TimeSpan? WriteStallLimitOverride { get; set; } + + /// The write-stall bound now (see ). + internal TimeSpan WriteStallLimit + => WriteStallLimitOverride ?? (SilenceLimit > TimeSpan.FromSeconds(10) ? SilenceLimit : TimeSpan.FromSeconds(10)); + /// Test seam: ignore the worker's soft-deadline notices, as a worker whose watchdog never sends them would. internal bool DropSoftNoticesForTest { get; set; } @@ -380,6 +399,7 @@ internal sealed class WorkerConnection : IAsyncDisposable MaxLogLines = 1000, CompileThreads = (uint)Math.Max(0, spec.CompileThreads), WorkerSoftClock = spec.WorkerSoftClock, + TestStallReaderOnDiag = spec.TestStallReaderOnDiag, }; if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey) { @@ -755,7 +775,9 @@ internal sealed class WorkerConnection : IAsyncDisposable if (!_exited.Task.IsCompleted) { lock (_pending) { _killedFor ??= (reason, null); } - Send(new Envelope { Shutdown = new Shutdown() }); + // Off this thread: the send may drain the writer, and a write blocks on a full pipe — the 1 s budget below + // must hold regardless (the kill then fails that write). + _ = Task.Run(() => Send(new Envelope { Shutdown = new Shutdown() })); try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false); @@ -797,17 +819,44 @@ internal sealed class WorkerConnection : IAsyncDisposable { WorkerExitReason? fault = null; var reader = new ScriptWorkerFrameReader(_pipe); - try - { - while (reader.Read() is { } frame) HandleFrame(frame); - } - catch (ScriptWorkerProtocolException ex) - { - fault = ProtocolFault(ex); - } - catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException - or System.Net.Sockets.SocketException) + while (true) { + Envelope? frame; + try + { + frame = reader.Read(); + } + catch (ScriptWorkerProtocolException ex) + { + fault = ProtocolFault(ex); + break; + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException + or System.Net.Sockets.SocketException) + { + break; // the pipe ended (these come from the read only, never from the frame's handling below) + } + if (frame is null) break; + + try + { + HandleFrame(frame); + } + catch (ScriptWorkerProtocolException ex) + { + fault = ProtocolFault(ex); + break; + } + catch (Exception ex) + { + // HandleFrame runs completion work inline (slot releases, load bookkeeping). On this dedicated thread an + // exception that escaped would take the whole server process down, and treating it as the pipe ending + // would leave the worker running unread; so it ends this worker as a fault: every outstanding evaluation + // completes as "worker restarted" (BadResourceUnavailable) and the shard restarts. + _log.Error(ex, "ScriptWorker[{Shard}] failed handling a {Frame} frame; killing pid {Pid}", Shard, frame.BodyCase, Pid); + fault = WorkerExitReason.ProtocolFault; + break; + } } _ = PipeEndedAsync(fault); } @@ -941,6 +990,15 @@ internal sealed class WorkerConnection : IAsyncDisposable Kill(WorkerExitReason.KilledHeartbeat, null); return; } + if (_writer.WriteInProgressFor is { } blocked && blocked > WriteStallLimit) + { + // It talks but does not listen: the pipe is full and a producer is blocked in the write. The kill + // fails that write and releases it (lmxopcua#789). + _log.Warning("ScriptWorker[{Shard}] pid {Pid} has not read from its pipe for {Blocked} ms (a write is blocked on a full pipe) while still heartbeating; killing it", + Shard, Pid, (long)blocked.TotalMilliseconds); + Kill(WorkerExitReason.KilledUnresponsive, null); + return; + } if (!_pending.IsEmpty && ProgressCeiling() is { } ceiling) { var stuck = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastProgressTimestamp)); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs index 07cc6c394..f30b819cd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs @@ -55,7 +55,8 @@ internal static class WorkerPipeSecurity /// until the peer has READ the bytes, so a blocking writer () /// would wait out a full peer wake-up per frame — measured on windev as half the closed-loop throughput. With a /// quota a write completes as soon as the bytes are buffered; a peer that stops reading still fills it and blocks - /// the writer, which the heartbeat monitor ends. + /// the writer: the host's write-stall check (WorkerConnection.WriteStallLimit) then kills the worker, which + /// fails the write and releases the blocked thread. /// internal const int BufferBytes = 1024 * 1024; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto index ed67c6a63..086de3ae7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto @@ -45,6 +45,9 @@ message Hello { // starts the run, and to report the crossing in EvalResults.soft_expired_eval_ids. The host then keeps only a // transport deadline of its own (see EvalResults). Honoured only when the worker echoes Ready.worker_soft_clock. bool worker_soft_clock = 11; + // Test seam (lmxopcua#789): the worker's reader stops reading for good at the first DiagnosticsRequest while its + // heartbeats go on, so the host's write-stall check can be exercised against a real worker. Never set in production. + bool test_stall_reader_on_diag = 12; } message Ready { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs index d7ac54f49..a49357986 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs @@ -126,7 +126,8 @@ public sealed class ScriptWorkerFrameReader /// , so no frame needs the socket engine or the completion port): /// the draining thread blocks while the pipe is full. Only /// one thread drains at a time — every other producer just queues and returns — and a peer that stops reading is -/// killed by its supervisor (the host's heartbeat check), which fails the write and releases that thread. +/// killed by its supervisor (the host's write-stall check, ; on the worker side the +/// host's silence check), which fails the write and releases that thread. /// A failed write completes the writer and reports the exception once through onFault. /// public sealed class ScriptWorkerFrameWriter @@ -141,6 +142,9 @@ public sealed class ScriptWorkerFrameWriter private int _faulted; private long _seq; + /// when the write in progress started; 0 = none. + private long _writeStarted; + /// Initializes the writer. /// The pipe. /// Called once when a write fails (or a frame is refused); the writer is then completed. @@ -162,6 +166,14 @@ public sealed class ScriptWorkerFrameWriter /// The negotiated frame cap. public int MaxFrameBytes { get; set; } + /// + /// How long the pipe write in progress has been blocked, or null when no write is in progress. A blocking write + /// stays in progress while the pipe is full, i.e. while the peer is not reading: the supervisor uses this to end a + /// peer that keeps talking but stopped listening (lmxopcua#789), which releases the thread blocked here. + /// + public TimeSpan? WriteInProgressFor + => Volatile.Read(ref _writeStarted) is var started and not 0 ? System.Diagnostics.Stopwatch.GetElapsedTime(started) : null; + /// Whether has been called or a write failed. public bool IsCompleted => Volatile.Read(ref _completed) != 0; @@ -196,8 +208,16 @@ public sealed class ScriptWorkerFrameWriter FillPass(); if (_pass.WrittenCount > 0) { - if (_synchronous) _stream.Write(_pass.WrittenSpan); - else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false); + Volatile.Write(ref _writeStarted, System.Diagnostics.Stopwatch.GetTimestamp()); + try + { + if (_synchronous) _stream.Write(_pass.WrittenSpan); + else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false); + } + finally + { + Volatile.Write(ref _writeStarted, 0); + } _pass.ResetWrittenCount(); // Synchronous on purpose: a pipe has no write buffer to flush, and PipeStream does not override // FlushAsync, so the base Stream.FlushAsync would queue a thread-pool work item per pass. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs index 952a7b08e..47ee80d6b 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs @@ -162,6 +162,36 @@ public sealed unsafe class WindowsBlockingPipeStream : Stream /// A no-op: a pipe has no write buffer to flush. public override void Flush() { } + /// A no-op, completed: a pipe has no write buffer to flush. + /// Unused. + /// A completed task. + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + // The asynchronous surface is refused: Stream's base versions run the blocking call on the pool behind ONE semaphore + // shared by reads and writes, which would serialise a write behind a read that is waiting for the peer — the very + // deadlock this stream exists to avoid. Callers use the blocking calls on their own threads. + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused(); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused(); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused(); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused(); + + /// + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused(); + + /// + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused(); + + private static NotSupportedException AsyncRefused() + => new("WindowsBlockingPipeStream is blocking only: read and write on dedicated threads (lmxopcua#789)"); + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs index 0b17cf03f..23f1bbaa3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs @@ -94,6 +94,9 @@ internal sealed class WorkerHost private long _watchdogNextPass; private long _seq; + /// Test seam (Hello.test_stall_reader_on_diag). + private bool _testStallReaderOnDiag; + public WorkerHost(Stream pipe, string nonce, int parentPid) { _pipe = pipe; @@ -157,6 +160,7 @@ internal sealed class WorkerHost if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs); if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64); _workerSoftClock = hello.WorkerSoftClock; + _testStallReaderOnDiag = hello.TestStallReaderOnDiag; if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength) { // Design §7: the host owns the key; it reaches us only here, never through the environment or a file. @@ -205,9 +209,21 @@ internal sealed class WorkerHost done.TrySetResult(ScriptWorkerEnvironment.ExitPipeClosed); return; } - if (HandleFrame(frame) is { } exit) + int? exit; + try { - done.TrySetResult(exit); + exit = HandleFrame(frame); + } + catch (Exception ex) + { + // Never an unhandled exception on this thread: the worker ends as a fault the host can attribute. + Console.Error.WriteLine($"ScriptWorker: reader failed: {ex}"); + done.TrySetResult(ScriptWorkerEnvironment.ExitProtocolFault); + return; + } + if (exit is { } code) + { + done.TrySetResult(code); return; } } @@ -250,10 +266,18 @@ internal sealed class WorkerHost case Envelope.BodyOneofCase.Cancel: if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel)) { - try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ } + // Off the reader (lmxopcua#789): Cancel() runs the script's own ctx.CancellationToken callbacks, and a + // callback that blocks would stop this worker reading its pipe — the host's writes would then block. + ThreadPool.UnsafeQueueUserWorkItem(static cts => + { + try { cts.Cancel(); } + catch (ObjectDisposedException) { /* just finished */ } + catch (Exception ex) { Console.Error.WriteLine($"ScriptWorker: a cancellation callback threw: {ex.Message}"); } + }, toCancel.Cancellation, preferLocal: false); } return null; case Envelope.BodyOneofCase.DiagRequest: + if (_testStallReaderOnDiag) Thread.Sleep(Timeout.Infinite); // test seam: talk but never listen again Send(new Envelope { DiagReply = Diagnostics() }); return null; case Envelope.BodyOneofCase.Shutdown: diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs index 0f2561c2c..2e7fbd11e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs @@ -12,6 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// that keeps worker pipe I/O off the completion port. Exercised on a real /// pipe pair; on Unix the same tests cover the plain pipe it returns there. /// +[Trait("Category", "ScriptWorker")] public sealed class BlockingPipeStreamTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -144,4 +145,85 @@ public sealed class BlockingPipeStreamTests ended.ShouldBe(0); } } + + /// + /// Writes until the pipe is full (the peer never reads), then returns the writing task, which is blocked in the + /// kernel at that point; it ends with the exception the write saw. + /// + private static async Task BlockedWriterAsync(Stream end) + { + long written = 0; + var writer = Task.Factory.StartNew(() => + { + var chunk = new byte[64 * 1024]; + while (true) + { + end.Write(chunk, 0, chunk.Length); + Interlocked.Add(ref written, chunk.Length); + } + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + // Full once no progress is made for a while (a zero-quota Windows pipe blocks at once; a socket after its buffer). + long last; + do + { + last = Interlocked.Read(ref written); + await Task.Delay(300, Ct); + } while (Interlocked.Read(ref written) != last && !writer.IsCompleted); + writer.IsCompleted.ShouldBeFalse("positive control: the write is blocked on a full pipe"); + return writer; + } + + private static async Task ShouldEndAsync(Task blocked, string what) + { + var ended = await Task.WhenAny(blocked, Task.Delay(TimeSpan.FromSeconds(10), Ct)); + ended.ShouldBeSameAs(blocked, $"{what} ended, not hung"); + var ex = blocked.Exception?.GetBaseException(); + ex.ShouldNotBeNull($"{what} ended with an error"); + (ex is ObjectDisposedException or IOException).ShouldBeTrue(ex.ToString()); + } + + [Fact] + public async Task Disposing_an_end_wakes_its_own_write_blocked_on_a_full_pipe() + { + var (server, client) = await PairAsync(); + using (client) + { + var writer = await BlockedWriterAsync(server); + server.Dispose(); + await ShouldEndAsync(writer, "the blocked write"); + } + } + + [Fact] + public async Task Disposing_an_end_with_a_read_and_a_write_both_pending_wakes_both() + { + var (server, client) = await PairAsync(); + using (client) + { + var read = Task.Factory.StartNew(() => + { + var n = server.Read(new byte[16], 0, 16); + if (n == 0) throw new IOException("end of stream"); + return n; + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + var writer = await BlockedWriterAsync(server); + read.IsCompleted.ShouldBeFalse("positive control: the read is pending beside the blocked write"); + server.Dispose(); + await ShouldEndAsync(writer, "the blocked write"); + await ShouldEndAsync(read, "the pending read"); + } + } + + [Fact] + public async Task The_asynchronous_surface_is_refused_on_windows() + { + var (server, client) = await PairAsync(); + using (server) + using (client) + { + if (!OperatingSystem.IsWindows()) return; // the Unix pipe itself keeps PipeStream's own async surface + await Should.ThrowAsync(() => server.ReadAsync(new byte[4], 0, 4, Ct)); + await Should.ThrowAsync(async () => await server.WriteAsync(new byte[4].AsMemory(), Ct)); + } + } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs index 60aa1a8d4..ceba4d8ad 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// lmxopcua#780 design §6 "Pipe hardening": only the launched process, echoing its nonce, is accepted. +[Trait("Category", "ScriptWorker")] public sealed class HandshakeTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs new file mode 100644 index 000000000..bfe8d1460 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs @@ -0,0 +1,106 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; + +namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; + +/// +/// lmxopcua#789 review: the blocking pipe I/O runs the host's reader on a dedicated thread and blocks producers on a +/// full pipe. A fault in the reader's inline completion work must end that worker, not the server process; and a worker +/// that stops reading while it keeps heartbeating must be killed so blocked producers are released. +/// +[Collection(ScriptWorkerCollection.Name)] +[Trait("Category", "ScriptWorker")] +public sealed class PipeFaultTests +{ + private const string Healthy = "return 7;"; + private const string Runaway = "while (true) { }"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// + /// The slot release runs on the pipe reader, inline, before the caller resumes. When it throws, the reader must + /// end the worker as a fault: the run in flight beside it answers BadResourceUnavailable ("worker restarted"), + /// the shard restarts, and the next evaluation is served — instead of an unhandled exception on a dedicated thread, + /// which would terminate the host. + /// + [Fact] + public async Task A_completion_callback_that_throws_on_the_reader_ends_that_worker_as_faulted_not_the_host() + { + await using var runner = Scripts.Runner(Scripts.Options()); + (await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct)).Value.ShouldBe(7, "positive control: the hook is not armed yet"); + var worker = runner.Shards[0].Current.ShouldNotBeNull(); + + var spin = runner.RunAsync(Scripts.Vt("spin", Runaway, TimeSpan.FromSeconds(30)), Ct); + await Scripts.Eventually(() => worker.InFlightCount > 0 ? worker : null, TimeSpan.FromSeconds(30), "the sibling in flight"); + + var armed = 1; + runner.SlotReleasingForTest = key => + { + if (key.EndsWith("boom", StringComparison.Ordinal) && Interlocked.Exchange(ref armed, 0) == 1) + throw new InvalidOperationException("a completion callback failed"); + }; + await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct); + + var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct); + exit.Reason.ShouldBe(WorkerExitReason.ProtocolFault); + var sibling = await spin.WaitAsync(TimeSpan.FromSeconds(15), Ct); + sibling.Status.ShouldBe(ScriptRunStatus.Unavailable); + sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); + sibling.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable); + + (await runner.RunAsync(Scripts.Vt("after", Healthy), Ct)).Value.ShouldBe(7, "the shard restarted and serves again"); + runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid); + } + + /// + /// A worker whose reader stops while its heartbeats go on: the host's writes fill the pipe and the producer (here + /// a deploy loading scripts, the case nothing else would end: no evaluation is outstanding and a compile's timeout + /// starts only when the worker starts it) blocks in the write. The write-stall check must kill the worker within + /// its bound — as unresponsive, not for silence, which proves it kept heartbeating — and that releases the producer. + /// + [Fact] + public async Task A_worker_that_stops_reading_but_keeps_heartbeating_is_killed_and_the_blocked_producer_returns() + { + var heartbeat = TimeSpan.FromMilliseconds(200); + var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }), + 256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: heartbeat, + HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30), TestStallReaderOnDiag: true); + var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct); + await using var owned = connection; + var warm = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Healthy, + SHA256.HashData(Encoding.UTF8.GetBytes(Healthy)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct); + warm.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue(); + connection.SilenceLimit = heartbeat * 5; // armed once the cold start is over (see SupervisionTests) + var stallLimit = TimeSpan.FromSeconds(2); + connection.WriteStallLimitOverride = stallLimit; + + _ = connection.DiagnosticsAsync(TimeSpan.FromSeconds(60)); // the worker's reader stops here for good + var producer = Task.Factory.StartNew(() => + { + // About 5 MB of loads: far more than the pipe's buffer and the worker reader's own. + for (var i = 0; i < 20; i++) + { + var source = $"return {i}; // " + new string('x', 256 * 1024); + _ = connection.EnsureLoadedAsync($"big{i}", ContextKind.VirtualTag, source, + SHA256.HashData(Encoding.UTF8.GetBytes(source)), []); + } + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + var sw = Stopwatch.StartNew(); + await Task.Delay(stallLimit / 2, Ct); + producer.IsCompleted.ShouldBeFalse("positive control: the producer is blocked in a write to the full pipe"); + connection.IsAlive.ShouldBeTrue("positive control: heartbeats keep the silence check quiet"); + + var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(20), Ct); + exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive, "killed by the write-stall check, not for silence"); + await producer.WaitAsync(TimeSpan.FromSeconds(10), Ct); + sw.Elapsed.ShouldBeLessThan(stallLimit + TimeSpan.FromSeconds(8), "the producer was released within the bound"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs index f8fc959be..8887f52c0 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// lmxopcua#780 design §13 — frame codec and the CLR ↔ Variant mapping. +[Trait("Category", "ScriptWorker")] public sealed class ProtocolTests { public static IEnumerable Values()