diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs index 4d3c321c2..4c5894c97 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs @@ -91,6 +91,10 @@ internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan /// How long the worker may send nothing before it is killed (); null = /// three s. Test seam (with ). /// +/// +/// Test seam: the worker's reader stops reading for good at the first diagnostics request while its heartbeats go on — +/// a worker that keeps talking but stopped listening (the write-stall check, lmxopcua#789). +/// /// /// Ask the worker to run the soft clock from each run's start (lmxopcua#827; see ). /// False (tests only) keeps the pre-#827 host clock, which is also what a worker that does not echo it gets. @@ -110,7 +114,8 @@ internal sealed record WorkerLaunchSpec( int CompileThreads = 0, string? WorkerSid = null, bool WorkerSoftClock = true, - TimeSpan? HeartbeatSilenceLimit = null) + TimeSpan? HeartbeatSilenceLimit = null, + bool TestStallReaderOnDiag = false) { /// The default compile timeout. public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60); @@ -265,6 +270,20 @@ internal sealed class WorkerConnection : IAsyncDisposable /// internal TimeSpan? ProgressCeilingOverride { get; set; } + /// + /// How long one pipe write may stay blocked before the worker is killed (, + /// lmxopcua#789). Writes are blocking, so a full pipe — a worker whose reader stopped while its heartbeats go on — + /// would otherwise hold whichever producer is draining the writer (a seam submitting, a deploy loading scripts, a + /// Cancel) forever: the silence check never fires, and with only loads queued nothing else does. A worker that is + /// reading drains the 1 MiB buffer far faster than this. Null = the larger of 10 s and the silence limit; set = an + /// override (tests). + /// + internal TimeSpan? WriteStallLimitOverride { get; set; } + + /// The write-stall bound now (see ). + internal TimeSpan WriteStallLimit + => WriteStallLimitOverride ?? (SilenceLimit > TimeSpan.FromSeconds(10) ? SilenceLimit : TimeSpan.FromSeconds(10)); + /// Test seam: ignore the worker's soft-deadline notices, as a worker whose watchdog never sends them would. internal bool DropSoftNoticesForTest { get; set; } @@ -380,6 +399,7 @@ internal sealed class WorkerConnection : IAsyncDisposable MaxLogLines = 1000, CompileThreads = (uint)Math.Max(0, spec.CompileThreads), WorkerSoftClock = spec.WorkerSoftClock, + TestStallReaderOnDiag = spec.TestStallReaderOnDiag, }; if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey) { @@ -755,7 +775,9 @@ internal sealed class WorkerConnection : IAsyncDisposable if (!_exited.Task.IsCompleted) { lock (_pending) { _killedFor ??= (reason, null); } - Send(new Envelope { Shutdown = new Shutdown() }); + // Off this thread: the send may drain the writer, and a write blocks on a full pipe — the 1 s budget below + // must hold regardless (the kill then fails that write). + _ = Task.Run(() => Send(new Envelope { Shutdown = new Shutdown() })); try { await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false); @@ -797,17 +819,44 @@ internal sealed class WorkerConnection : IAsyncDisposable { WorkerExitReason? fault = null; var reader = new ScriptWorkerFrameReader(_pipe); - try - { - while (reader.Read() is { } frame) HandleFrame(frame); - } - catch (ScriptWorkerProtocolException ex) - { - fault = ProtocolFault(ex); - } - catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException - or System.Net.Sockets.SocketException) + while (true) { + Envelope? frame; + try + { + frame = reader.Read(); + } + catch (ScriptWorkerProtocolException ex) + { + fault = ProtocolFault(ex); + break; + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException + or System.Net.Sockets.SocketException) + { + break; // the pipe ended (these come from the read only, never from the frame's handling below) + } + if (frame is null) break; + + try + { + HandleFrame(frame); + } + catch (ScriptWorkerProtocolException ex) + { + fault = ProtocolFault(ex); + break; + } + catch (Exception ex) + { + // HandleFrame runs completion work inline (slot releases, load bookkeeping). On this dedicated thread an + // exception that escaped would take the whole server process down, and treating it as the pipe ending + // would leave the worker running unread; so it ends this worker as a fault: every outstanding evaluation + // completes as "worker restarted" (BadResourceUnavailable) and the shard restarts. + _log.Error(ex, "ScriptWorker[{Shard}] failed handling a {Frame} frame; killing pid {Pid}", Shard, frame.BodyCase, Pid); + fault = WorkerExitReason.ProtocolFault; + break; + } } _ = PipeEndedAsync(fault); } @@ -941,6 +990,15 @@ internal sealed class WorkerConnection : IAsyncDisposable Kill(WorkerExitReason.KilledHeartbeat, null); return; } + if (_writer.WriteInProgressFor is { } blocked && blocked > WriteStallLimit) + { + // It talks but does not listen: the pipe is full and a producer is blocked in the write. The kill + // fails that write and releases it (lmxopcua#789). + _log.Warning("ScriptWorker[{Shard}] pid {Pid} has not read from its pipe for {Blocked} ms (a write is blocked on a full pipe) while still heartbeating; killing it", + Shard, Pid, (long)blocked.TotalMilliseconds); + Kill(WorkerExitReason.KilledUnresponsive, null); + return; + } if (!_pending.IsEmpty && ProgressCeiling() is { } ceiling) { var stuck = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastProgressTimestamp)); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs index 07cc6c394..f30b819cd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs @@ -55,7 +55,8 @@ internal static class WorkerPipeSecurity /// until the peer has READ the bytes, so a blocking writer () /// would wait out a full peer wake-up per frame — measured on windev as half the closed-loop throughput. With a /// quota a write completes as soon as the bytes are buffered; a peer that stops reading still fills it and blocks - /// the writer, which the heartbeat monitor ends. + /// the writer: the host's write-stall check (WorkerConnection.WriteStallLimit) then kills the worker, which + /// fails the write and releases the blocked thread. /// internal const int BufferBytes = 1024 * 1024; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto index ed67c6a63..086de3ae7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto @@ -45,6 +45,9 @@ message Hello { // starts the run, and to report the crossing in EvalResults.soft_expired_eval_ids. The host then keeps only a // transport deadline of its own (see EvalResults). Honoured only when the worker echoes Ready.worker_soft_clock. bool worker_soft_clock = 11; + // Test seam (lmxopcua#789): the worker's reader stops reading for good at the first DiagnosticsRequest while its + // heartbeats go on, so the host's write-stall check can be exercised against a real worker. Never set in production. + bool test_stall_reader_on_diag = 12; } message Ready { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs index d7ac54f49..a49357986 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs @@ -126,7 +126,8 @@ public sealed class ScriptWorkerFrameReader /// , so no frame needs the socket engine or the completion port): /// the draining thread blocks while the pipe is full. Only /// one thread drains at a time — every other producer just queues and returns — and a peer that stops reading is -/// killed by its supervisor (the host's heartbeat check), which fails the write and releases that thread. +/// killed by its supervisor (the host's write-stall check, ; on the worker side the +/// host's silence check), which fails the write and releases that thread. /// A failed write completes the writer and reports the exception once through onFault. /// public sealed class ScriptWorkerFrameWriter @@ -141,6 +142,9 @@ public sealed class ScriptWorkerFrameWriter private int _faulted; private long _seq; + /// when the write in progress started; 0 = none. + private long _writeStarted; + /// Initializes the writer. /// The pipe. /// Called once when a write fails (or a frame is refused); the writer is then completed. @@ -162,6 +166,14 @@ public sealed class ScriptWorkerFrameWriter /// The negotiated frame cap. public int MaxFrameBytes { get; set; } + /// + /// How long the pipe write in progress has been blocked, or null when no write is in progress. A blocking write + /// stays in progress while the pipe is full, i.e. while the peer is not reading: the supervisor uses this to end a + /// peer that keeps talking but stopped listening (lmxopcua#789), which releases the thread blocked here. + /// + public TimeSpan? WriteInProgressFor + => Volatile.Read(ref _writeStarted) is var started and not 0 ? System.Diagnostics.Stopwatch.GetElapsedTime(started) : null; + /// Whether has been called or a write failed. public bool IsCompleted => Volatile.Read(ref _completed) != 0; @@ -196,8 +208,16 @@ public sealed class ScriptWorkerFrameWriter FillPass(); if (_pass.WrittenCount > 0) { - if (_synchronous) _stream.Write(_pass.WrittenSpan); - else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false); + Volatile.Write(ref _writeStarted, System.Diagnostics.Stopwatch.GetTimestamp()); + try + { + if (_synchronous) _stream.Write(_pass.WrittenSpan); + else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false); + } + finally + { + Volatile.Write(ref _writeStarted, 0); + } _pass.ResetWrittenCount(); // Synchronous on purpose: a pipe has no write buffer to flush, and PipeStream does not override // FlushAsync, so the base Stream.FlushAsync would queue a thread-pool work item per pass. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs index 952a7b08e..47ee80d6b 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs @@ -162,6 +162,36 @@ public sealed unsafe class WindowsBlockingPipeStream : Stream /// A no-op: a pipe has no write buffer to flush. public override void Flush() { } + /// A no-op, completed: a pipe has no write buffer to flush. + /// Unused. + /// A completed task. + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + // The asynchronous surface is refused: Stream's base versions run the blocking call on the pool behind ONE semaphore + // shared by reads and writes, which would serialise a write behind a read that is waiting for the peer — the very + // deadlock this stream exists to avoid. Callers use the blocking calls on their own threads. + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused(); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused(); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused(); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused(); + + /// + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused(); + + /// + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused(); + + private static NotSupportedException AsyncRefused() + => new("WindowsBlockingPipeStream is blocking only: read and write on dedicated threads (lmxopcua#789)"); + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs index 0b17cf03f..23f1bbaa3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs @@ -94,6 +94,9 @@ internal sealed class WorkerHost private long _watchdogNextPass; private long _seq; + /// Test seam (Hello.test_stall_reader_on_diag). + private bool _testStallReaderOnDiag; + public WorkerHost(Stream pipe, string nonce, int parentPid) { _pipe = pipe; @@ -157,6 +160,7 @@ internal sealed class WorkerHost if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs); if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64); _workerSoftClock = hello.WorkerSoftClock; + _testStallReaderOnDiag = hello.TestStallReaderOnDiag; if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength) { // Design §7: the host owns the key; it reaches us only here, never through the environment or a file. @@ -205,9 +209,21 @@ internal sealed class WorkerHost done.TrySetResult(ScriptWorkerEnvironment.ExitPipeClosed); return; } - if (HandleFrame(frame) is { } exit) + int? exit; + try { - done.TrySetResult(exit); + exit = HandleFrame(frame); + } + catch (Exception ex) + { + // Never an unhandled exception on this thread: the worker ends as a fault the host can attribute. + Console.Error.WriteLine($"ScriptWorker: reader failed: {ex}"); + done.TrySetResult(ScriptWorkerEnvironment.ExitProtocolFault); + return; + } + if (exit is { } code) + { + done.TrySetResult(code); return; } } @@ -250,10 +266,18 @@ internal sealed class WorkerHost case Envelope.BodyOneofCase.Cancel: if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel)) { - try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ } + // Off the reader (lmxopcua#789): Cancel() runs the script's own ctx.CancellationToken callbacks, and a + // callback that blocks would stop this worker reading its pipe — the host's writes would then block. + ThreadPool.UnsafeQueueUserWorkItem(static cts => + { + try { cts.Cancel(); } + catch (ObjectDisposedException) { /* just finished */ } + catch (Exception ex) { Console.Error.WriteLine($"ScriptWorker: a cancellation callback threw: {ex.Message}"); } + }, toCancel.Cancellation, preferLocal: false); } return null; case Envelope.BodyOneofCase.DiagRequest: + if (_testStallReaderOnDiag) Thread.Sleep(Timeout.Infinite); // test seam: talk but never listen again Send(new Envelope { DiagReply = Diagnostics() }); return null; case Envelope.BodyOneofCase.Shutdown: diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs index 0f2561c2c..2e7fbd11e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs @@ -12,6 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// that keeps worker pipe I/O off the completion port. Exercised on a real /// pipe pair; on Unix the same tests cover the plain pipe it returns there. /// +[Trait("Category", "ScriptWorker")] public sealed class BlockingPipeStreamTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -144,4 +145,85 @@ public sealed class BlockingPipeStreamTests ended.ShouldBe(0); } } + + /// + /// Writes until the pipe is full (the peer never reads), then returns the writing task, which is blocked in the + /// kernel at that point; it ends with the exception the write saw. + /// + private static async Task BlockedWriterAsync(Stream end) + { + long written = 0; + var writer = Task.Factory.StartNew(() => + { + var chunk = new byte[64 * 1024]; + while (true) + { + end.Write(chunk, 0, chunk.Length); + Interlocked.Add(ref written, chunk.Length); + } + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + // Full once no progress is made for a while (a zero-quota Windows pipe blocks at once; a socket after its buffer). + long last; + do + { + last = Interlocked.Read(ref written); + await Task.Delay(300, Ct); + } while (Interlocked.Read(ref written) != last && !writer.IsCompleted); + writer.IsCompleted.ShouldBeFalse("positive control: the write is blocked on a full pipe"); + return writer; + } + + private static async Task ShouldEndAsync(Task blocked, string what) + { + var ended = await Task.WhenAny(blocked, Task.Delay(TimeSpan.FromSeconds(10), Ct)); + ended.ShouldBeSameAs(blocked, $"{what} ended, not hung"); + var ex = blocked.Exception?.GetBaseException(); + ex.ShouldNotBeNull($"{what} ended with an error"); + (ex is ObjectDisposedException or IOException).ShouldBeTrue(ex.ToString()); + } + + [Fact] + public async Task Disposing_an_end_wakes_its_own_write_blocked_on_a_full_pipe() + { + var (server, client) = await PairAsync(); + using (client) + { + var writer = await BlockedWriterAsync(server); + server.Dispose(); + await ShouldEndAsync(writer, "the blocked write"); + } + } + + [Fact] + public async Task Disposing_an_end_with_a_read_and_a_write_both_pending_wakes_both() + { + var (server, client) = await PairAsync(); + using (client) + { + var read = Task.Factory.StartNew(() => + { + var n = server.Read(new byte[16], 0, 16); + if (n == 0) throw new IOException("end of stream"); + return n; + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + var writer = await BlockedWriterAsync(server); + read.IsCompleted.ShouldBeFalse("positive control: the read is pending beside the blocked write"); + server.Dispose(); + await ShouldEndAsync(writer, "the blocked write"); + await ShouldEndAsync(read, "the pending read"); + } + } + + [Fact] + public async Task The_asynchronous_surface_is_refused_on_windows() + { + var (server, client) = await PairAsync(); + using (server) + using (client) + { + if (!OperatingSystem.IsWindows()) return; // the Unix pipe itself keeps PipeStream's own async surface + await Should.ThrowAsync(() => server.ReadAsync(new byte[4], 0, 4, Ct)); + await Should.ThrowAsync(async () => await server.WriteAsync(new byte[4].AsMemory(), Ct)); + } + } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs index 60aa1a8d4..ceba4d8ad 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// lmxopcua#780 design §6 "Pipe hardening": only the launched process, echoing its nonce, is accepted. +[Trait("Category", "ScriptWorker")] public sealed class HandshakeTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs new file mode 100644 index 000000000..bfe8d1460 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs @@ -0,0 +1,106 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; +using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; + +namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; + +/// +/// lmxopcua#789 review: the blocking pipe I/O runs the host's reader on a dedicated thread and blocks producers on a +/// full pipe. A fault in the reader's inline completion work must end that worker, not the server process; and a worker +/// that stops reading while it keeps heartbeating must be killed so blocked producers are released. +/// +[Collection(ScriptWorkerCollection.Name)] +[Trait("Category", "ScriptWorker")] +public sealed class PipeFaultTests +{ + private const string Healthy = "return 7;"; + private const string Runaway = "while (true) { }"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// + /// The slot release runs on the pipe reader, inline, before the caller resumes. When it throws, the reader must + /// end the worker as a fault: the run in flight beside it answers BadResourceUnavailable ("worker restarted"), + /// the shard restarts, and the next evaluation is served — instead of an unhandled exception on a dedicated thread, + /// which would terminate the host. + /// + [Fact] + public async Task A_completion_callback_that_throws_on_the_reader_ends_that_worker_as_faulted_not_the_host() + { + await using var runner = Scripts.Runner(Scripts.Options()); + (await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct)).Value.ShouldBe(7, "positive control: the hook is not armed yet"); + var worker = runner.Shards[0].Current.ShouldNotBeNull(); + + var spin = runner.RunAsync(Scripts.Vt("spin", Runaway, TimeSpan.FromSeconds(30)), Ct); + await Scripts.Eventually(() => worker.InFlightCount > 0 ? worker : null, TimeSpan.FromSeconds(30), "the sibling in flight"); + + var armed = 1; + runner.SlotReleasingForTest = key => + { + if (key.EndsWith("boom", StringComparison.Ordinal) && Interlocked.Exchange(ref armed, 0) == 1) + throw new InvalidOperationException("a completion callback failed"); + }; + await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct); + + var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct); + exit.Reason.ShouldBe(WorkerExitReason.ProtocolFault); + var sibling = await spin.WaitAsync(TimeSpan.FromSeconds(15), Ct); + sibling.Status.ShouldBe(ScriptRunStatus.Unavailable); + sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); + sibling.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable); + + (await runner.RunAsync(Scripts.Vt("after", Healthy), Ct)).Value.ShouldBe(7, "the shard restarted and serves again"); + runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid); + } + + /// + /// A worker whose reader stops while its heartbeats go on: the host's writes fill the pipe and the producer (here + /// a deploy loading scripts, the case nothing else would end: no evaluation is outstanding and a compile's timeout + /// starts only when the worker starts it) blocks in the write. The write-stall check must kill the worker within + /// its bound — as unresponsive, not for silence, which proves it kept heartbeating — and that releases the producer. + /// + [Fact] + public async Task A_worker_that_stops_reading_but_keeps_heartbeating_is_killed_and_the_blocked_producer_returns() + { + var heartbeat = TimeSpan.FromMilliseconds(200); + var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }), + 256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: heartbeat, + HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30), TestStallReaderOnDiag: true); + var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct); + await using var owned = connection; + var warm = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Healthy, + SHA256.HashData(Encoding.UTF8.GetBytes(Healthy)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct); + warm.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue(); + connection.SilenceLimit = heartbeat * 5; // armed once the cold start is over (see SupervisionTests) + var stallLimit = TimeSpan.FromSeconds(2); + connection.WriteStallLimitOverride = stallLimit; + + _ = connection.DiagnosticsAsync(TimeSpan.FromSeconds(60)); // the worker's reader stops here for good + var producer = Task.Factory.StartNew(() => + { + // About 5 MB of loads: far more than the pipe's buffer and the worker reader's own. + for (var i = 0; i < 20; i++) + { + var source = $"return {i}; // " + new string('x', 256 * 1024); + _ = connection.EnsureLoadedAsync($"big{i}", ContextKind.VirtualTag, source, + SHA256.HashData(Encoding.UTF8.GetBytes(source)), []); + } + }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + var sw = Stopwatch.StartNew(); + await Task.Delay(stallLimit / 2, Ct); + producer.IsCompleted.ShouldBeFalse("positive control: the producer is blocked in a write to the full pipe"); + connection.IsAlive.ShouldBeTrue("positive control: heartbeats keep the silence check quiet"); + + var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(20), Ct); + exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive, "killed by the write-stall check, not for silence"); + await producer.WaitAsync(TimeSpan.FromSeconds(10), Ct); + sw.Elapsed.ShouldBeLessThan(stallLimit + TimeSpan.FromSeconds(8), "the producer was released within the bound"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs index f8fc959be..8887f52c0 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// lmxopcua#780 design §13 — frame codec and the CLR ↔ Variant mapping. +[Trait("Category", "ScriptWorker")] public sealed class ProtocolTests { public static IEnumerable Values()