diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs
index 4d3c321c2..4c5894c97 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs
@@ -91,6 +91,10 @@ internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan
/// How long the worker may send nothing before it is killed (); null =
/// three s. Test seam (with ).
///
+///
+/// Test seam: the worker's reader stops reading for good at the first diagnostics request while its heartbeats go on —
+/// a worker that keeps talking but stopped listening (the write-stall check, lmxopcua#789).
+///
///
/// Ask the worker to run the soft clock from each run's start (lmxopcua#827; see ).
/// False (tests only) keeps the pre-#827 host clock, which is also what a worker that does not echo it gets.
@@ -110,7 +114,8 @@ internal sealed record WorkerLaunchSpec(
int CompileThreads = 0,
string? WorkerSid = null,
bool WorkerSoftClock = true,
- TimeSpan? HeartbeatSilenceLimit = null)
+ TimeSpan? HeartbeatSilenceLimit = null,
+ bool TestStallReaderOnDiag = false)
{
/// The default compile timeout.
public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60);
@@ -265,6 +270,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
///
internal TimeSpan? ProgressCeilingOverride { get; set; }
+ ///
+ /// How long one pipe write may stay blocked before the worker is killed (,
+ /// lmxopcua#789). Writes are blocking, so a full pipe — a worker whose reader stopped while its heartbeats go on —
+ /// would otherwise hold whichever producer is draining the writer (a seam submitting, a deploy loading scripts, a
+ /// Cancel) forever: the silence check never fires, and with only loads queued nothing else does. A worker that is
+ /// reading drains the 1 MiB buffer far faster than this. Null = the larger of 10 s and the silence limit; set = an
+ /// override (tests).
+ ///
+ internal TimeSpan? WriteStallLimitOverride { get; set; }
+
+ /// The write-stall bound now (see ).
+ internal TimeSpan WriteStallLimit
+ => WriteStallLimitOverride ?? (SilenceLimit > TimeSpan.FromSeconds(10) ? SilenceLimit : TimeSpan.FromSeconds(10));
+
/// Test seam: ignore the worker's soft-deadline notices, as a worker whose watchdog never sends them would.
internal bool DropSoftNoticesForTest { get; set; }
@@ -380,6 +399,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
MaxLogLines = 1000,
CompileThreads = (uint)Math.Max(0, spec.CompileThreads),
WorkerSoftClock = spec.WorkerSoftClock,
+ TestStallReaderOnDiag = spec.TestStallReaderOnDiag,
};
if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey)
{
@@ -755,7 +775,9 @@ internal sealed class WorkerConnection : IAsyncDisposable
if (!_exited.Task.IsCompleted)
{
lock (_pending) { _killedFor ??= (reason, null); }
- Send(new Envelope { Shutdown = new Shutdown() });
+ // Off this thread: the send may drain the writer, and a write blocks on a full pipe — the 1 s budget below
+ // must hold regardless (the kill then fails that write).
+ _ = Task.Run(() => Send(new Envelope { Shutdown = new Shutdown() }));
try
{
await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
@@ -797,17 +819,44 @@ internal sealed class WorkerConnection : IAsyncDisposable
{
WorkerExitReason? fault = null;
var reader = new ScriptWorkerFrameReader(_pipe);
- try
- {
- while (reader.Read() is { } frame) HandleFrame(frame);
- }
- catch (ScriptWorkerProtocolException ex)
- {
- fault = ProtocolFault(ex);
- }
- catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException
- or System.Net.Sockets.SocketException)
+ while (true)
{
+ Envelope? frame;
+ try
+ {
+ frame = reader.Read();
+ }
+ catch (ScriptWorkerProtocolException ex)
+ {
+ fault = ProtocolFault(ex);
+ break;
+ }
+ catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException
+ or System.Net.Sockets.SocketException)
+ {
+ break; // the pipe ended (these come from the read only, never from the frame's handling below)
+ }
+ if (frame is null) break;
+
+ try
+ {
+ HandleFrame(frame);
+ }
+ catch (ScriptWorkerProtocolException ex)
+ {
+ fault = ProtocolFault(ex);
+ break;
+ }
+ catch (Exception ex)
+ {
+ // HandleFrame runs completion work inline (slot releases, load bookkeeping). On this dedicated thread an
+ // exception that escaped would take the whole server process down, and treating it as the pipe ending
+ // would leave the worker running unread; so it ends this worker as a fault: every outstanding evaluation
+ // completes as "worker restarted" (BadResourceUnavailable) and the shard restarts.
+ _log.Error(ex, "ScriptWorker[{Shard}] failed handling a {Frame} frame; killing pid {Pid}", Shard, frame.BodyCase, Pid);
+ fault = WorkerExitReason.ProtocolFault;
+ break;
+ }
}
_ = PipeEndedAsync(fault);
}
@@ -941,6 +990,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
Kill(WorkerExitReason.KilledHeartbeat, null);
return;
}
+ if (_writer.WriteInProgressFor is { } blocked && blocked > WriteStallLimit)
+ {
+ // It talks but does not listen: the pipe is full and a producer is blocked in the write. The kill
+ // fails that write and releases it (lmxopcua#789).
+ _log.Warning("ScriptWorker[{Shard}] pid {Pid} has not read from its pipe for {Blocked} ms (a write is blocked on a full pipe) while still heartbeating; killing it",
+ Shard, Pid, (long)blocked.TotalMilliseconds);
+ Kill(WorkerExitReason.KilledUnresponsive, null);
+ return;
+ }
if (!_pending.IsEmpty && ProgressCeiling() is { } ceiling)
{
var stuck = Stopwatch.GetElapsedTime(Volatile.Read(ref _lastProgressTimestamp));
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs
index 07cc6c394..f30b819cd 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerPipeSecurity.cs
@@ -55,7 +55,8 @@ internal static class WorkerPipeSecurity
/// until the peer has READ the bytes, so a blocking writer ()
/// would wait out a full peer wake-up per frame — measured on windev as half the closed-loop throughput. With a
/// quota a write completes as soon as the bytes are buffered; a peer that stops reading still fills it and blocks
- /// the writer, which the heartbeat monitor ends.
+ /// the writer: the host's write-stall check (WorkerConnection.WriteStallLimit) then kills the worker, which
+ /// fails the write and releases the blocked thread.
///
internal const int BufferBytes = 1024 * 1024;
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto
index ed67c6a63..086de3ae7 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto
@@ -45,6 +45,9 @@ message Hello {
// starts the run, and to report the crossing in EvalResults.soft_expired_eval_ids. The host then keeps only a
// transport deadline of its own (see EvalResults). Honoured only when the worker echoes Ready.worker_soft_clock.
bool worker_soft_clock = 11;
+ // Test seam (lmxopcua#789): the worker's reader stops reading for good at the first DiagnosticsRequest while its
+ // heartbeats go on, so the host's write-stall check can be exercised against a real worker. Never set in production.
+ bool test_stall_reader_on_diag = 12;
}
message Ready {
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs
index d7ac54f49..a49357986 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs
@@ -126,7 +126,8 @@ public sealed class ScriptWorkerFrameReader
/// , so no frame needs the socket engine or the completion port):
/// the draining thread blocks while the pipe is full. Only
/// one thread drains at a time — every other producer just queues and returns — and a peer that stops reading is
-/// killed by its supervisor (the host's heartbeat check), which fails the write and releases that thread.
+/// killed by its supervisor (the host's write-stall check, ; on the worker side the
+/// host's silence check), which fails the write and releases that thread.
/// A failed write completes the writer and reports the exception once through onFault.
///
public sealed class ScriptWorkerFrameWriter
@@ -141,6 +142,9 @@ public sealed class ScriptWorkerFrameWriter
private int _faulted;
private long _seq;
+ /// when the write in progress started; 0 = none.
+ private long _writeStarted;
+
/// Initializes the writer.
/// The pipe.
/// Called once when a write fails (or a frame is refused); the writer is then completed.
@@ -162,6 +166,14 @@ public sealed class ScriptWorkerFrameWriter
/// The negotiated frame cap.
public int MaxFrameBytes { get; set; }
+ ///
+ /// How long the pipe write in progress has been blocked, or null when no write is in progress. A blocking write
+ /// stays in progress while the pipe is full, i.e. while the peer is not reading: the supervisor uses this to end a
+ /// peer that keeps talking but stopped listening (lmxopcua#789), which releases the thread blocked here.
+ ///
+ public TimeSpan? WriteInProgressFor
+ => Volatile.Read(ref _writeStarted) is var started and not 0 ? System.Diagnostics.Stopwatch.GetElapsedTime(started) : null;
+
/// Whether has been called or a write failed.
public bool IsCompleted => Volatile.Read(ref _completed) != 0;
@@ -196,8 +208,16 @@ public sealed class ScriptWorkerFrameWriter
FillPass();
if (_pass.WrittenCount > 0)
{
- if (_synchronous) _stream.Write(_pass.WrittenSpan);
- else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false);
+ Volatile.Write(ref _writeStarted, System.Diagnostics.Stopwatch.GetTimestamp());
+ try
+ {
+ if (_synchronous) _stream.Write(_pass.WrittenSpan);
+ else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false);
+ }
+ finally
+ {
+ Volatile.Write(ref _writeStarted, 0);
+ }
_pass.ResetWrittenCount();
// Synchronous on purpose: a pipe has no write buffer to flush, and PipeStream does not override
// FlushAsync, so the base Stream.FlushAsync would queue a thread-pool work item per pass.
diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs
index 952a7b08e..47ee80d6b 100644
--- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs
+++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/WindowsBlockingPipeStream.cs
@@ -162,6 +162,36 @@ public sealed unsafe class WindowsBlockingPipeStream : Stream
/// A no-op: a pipe has no write buffer to flush.
public override void Flush() { }
+ /// A no-op, completed: a pipe has no write buffer to flush.
+ /// Unused.
+ /// A completed task.
+ public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+ // The asynchronous surface is refused: Stream's base versions run the blocking call on the pool behind ONE semaphore
+ // shared by reads and writes, which would serialise a write behind a read that is waiting for the peer — the very
+ // deadlock this stream exists to avoid. Callers use the blocking calls on their own threads.
+
+ ///
+ public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused();
+
+ ///
+ public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused();
+
+ ///
+ public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw AsyncRefused();
+
+ ///
+ public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => throw AsyncRefused();
+
+ ///
+ public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused();
+
+ ///
+ public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => throw AsyncRefused();
+
+ private static NotSupportedException AsyncRefused()
+ => new("WindowsBlockingPipeStream is blocking only: read and write on dedicated threads (lmxopcua#789)");
+
///
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs
index 0b17cf03f..23f1bbaa3 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs
@@ -94,6 +94,9 @@ internal sealed class WorkerHost
private long _watchdogNextPass;
private long _seq;
+ /// Test seam (Hello.test_stall_reader_on_diag).
+ private bool _testStallReaderOnDiag;
+
public WorkerHost(Stream pipe, string nonce, int parentPid)
{
_pipe = pipe;
@@ -157,6 +160,7 @@ internal sealed class WorkerHost
if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs);
if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64);
_workerSoftClock = hello.WorkerSoftClock;
+ _testStallReaderOnDiag = hello.TestStallReaderOnDiag;
if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength)
{
// Design §7: the host owns the key; it reaches us only here, never through the environment or a file.
@@ -205,9 +209,21 @@ internal sealed class WorkerHost
done.TrySetResult(ScriptWorkerEnvironment.ExitPipeClosed);
return;
}
- if (HandleFrame(frame) is { } exit)
+ int? exit;
+ try
{
- done.TrySetResult(exit);
+ exit = HandleFrame(frame);
+ }
+ catch (Exception ex)
+ {
+ // Never an unhandled exception on this thread: the worker ends as a fault the host can attribute.
+ Console.Error.WriteLine($"ScriptWorker: reader failed: {ex}");
+ done.TrySetResult(ScriptWorkerEnvironment.ExitProtocolFault);
+ return;
+ }
+ if (exit is { } code)
+ {
+ done.TrySetResult(code);
return;
}
}
@@ -250,10 +266,18 @@ internal sealed class WorkerHost
case Envelope.BodyOneofCase.Cancel:
if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
{
- try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ }
+ // Off the reader (lmxopcua#789): Cancel() runs the script's own ctx.CancellationToken callbacks, and a
+ // callback that blocks would stop this worker reading its pipe — the host's writes would then block.
+ ThreadPool.UnsafeQueueUserWorkItem(static cts =>
+ {
+ try { cts.Cancel(); }
+ catch (ObjectDisposedException) { /* just finished */ }
+ catch (Exception ex) { Console.Error.WriteLine($"ScriptWorker: a cancellation callback threw: {ex.Message}"); }
+ }, toCancel.Cancellation, preferLocal: false);
}
return null;
case Envelope.BodyOneofCase.DiagRequest:
+ if (_testStallReaderOnDiag) Thread.Sleep(Timeout.Infinite); // test seam: talk but never listen again
Send(new Envelope { DiagReply = Diagnostics() });
return null;
case Envelope.BodyOneofCase.Shutdown:
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs
index 0f2561c2c..2e7fbd11e 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs
@@ -12,6 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// that keeps worker pipe I/O off the completion port. Exercised on a real
/// pipe pair; on Unix the same tests cover the plain pipe it returns there.
///
+[Trait("Category", "ScriptWorker")]
public sealed class BlockingPipeStreamTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
@@ -144,4 +145,85 @@ public sealed class BlockingPipeStreamTests
ended.ShouldBe(0);
}
}
+
+ ///
+ /// Writes until the pipe is full (the peer never reads), then returns the writing task, which is blocked in the
+ /// kernel at that point; it ends with the exception the write saw.
+ ///
+ private static async Task BlockedWriterAsync(Stream end)
+ {
+ long written = 0;
+ var writer = Task.Factory.StartNew(() =>
+ {
+ var chunk = new byte[64 * 1024];
+ while (true)
+ {
+ end.Write(chunk, 0, chunk.Length);
+ Interlocked.Add(ref written, chunk.Length);
+ }
+ }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+ // Full once no progress is made for a while (a zero-quota Windows pipe blocks at once; a socket after its buffer).
+ long last;
+ do
+ {
+ last = Interlocked.Read(ref written);
+ await Task.Delay(300, Ct);
+ } while (Interlocked.Read(ref written) != last && !writer.IsCompleted);
+ writer.IsCompleted.ShouldBeFalse("positive control: the write is blocked on a full pipe");
+ return writer;
+ }
+
+ private static async Task ShouldEndAsync(Task blocked, string what)
+ {
+ var ended = await Task.WhenAny(blocked, Task.Delay(TimeSpan.FromSeconds(10), Ct));
+ ended.ShouldBeSameAs(blocked, $"{what} ended, not hung");
+ var ex = blocked.Exception?.GetBaseException();
+ ex.ShouldNotBeNull($"{what} ended with an error");
+ (ex is ObjectDisposedException or IOException).ShouldBeTrue(ex.ToString());
+ }
+
+ [Fact]
+ public async Task Disposing_an_end_wakes_its_own_write_blocked_on_a_full_pipe()
+ {
+ var (server, client) = await PairAsync();
+ using (client)
+ {
+ var writer = await BlockedWriterAsync(server);
+ server.Dispose();
+ await ShouldEndAsync(writer, "the blocked write");
+ }
+ }
+
+ [Fact]
+ public async Task Disposing_an_end_with_a_read_and_a_write_both_pending_wakes_both()
+ {
+ var (server, client) = await PairAsync();
+ using (client)
+ {
+ var read = Task.Factory.StartNew(() =>
+ {
+ var n = server.Read(new byte[16], 0, 16);
+ if (n == 0) throw new IOException("end of stream");
+ return n;
+ }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+ var writer = await BlockedWriterAsync(server);
+ read.IsCompleted.ShouldBeFalse("positive control: the read is pending beside the blocked write");
+ server.Dispose();
+ await ShouldEndAsync(writer, "the blocked write");
+ await ShouldEndAsync(read, "the pending read");
+ }
+ }
+
+ [Fact]
+ public async Task The_asynchronous_surface_is_refused_on_windows()
+ {
+ var (server, client) = await PairAsync();
+ using (server)
+ using (client)
+ {
+ if (!OperatingSystem.IsWindows()) return; // the Unix pipe itself keeps PipeStream's own async surface
+ await Should.ThrowAsync(() => server.ReadAsync(new byte[4], 0, 4, Ct));
+ await Should.ThrowAsync(async () => await server.WriteAsync(new byte[4].AsMemory(), Ct));
+ }
+ }
}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs
index 60aa1a8d4..ceba4d8ad 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/HandshakeTests.cs
@@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// lmxopcua#780 design §6 "Pipe hardening": only the launched process, echoing its nonce, is accepted.
+[Trait("Category", "ScriptWorker")]
public sealed class HandshakeTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs
new file mode 100644
index 000000000..bfe8d1460
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/PipeFaultTests.cs
@@ -0,0 +1,106 @@
+using System.Diagnostics;
+using System.Security.Cryptography;
+using System.Text;
+using Serilog;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
+
+///
+/// lmxopcua#789 review: the blocking pipe I/O runs the host's reader on a dedicated thread and blocks producers on a
+/// full pipe. A fault in the reader's inline completion work must end that worker, not the server process; and a worker
+/// that stops reading while it keeps heartbeating must be killed so blocked producers are released.
+///
+[Collection(ScriptWorkerCollection.Name)]
+[Trait("Category", "ScriptWorker")]
+public sealed class PipeFaultTests
+{
+ private const string Healthy = "return 7;";
+ private const string Runaway = "while (true) { }";
+
+ private static CancellationToken Ct => TestContext.Current.CancellationToken;
+
+ ///
+ /// The slot release runs on the pipe reader, inline, before the caller resumes. When it throws, the reader must
+ /// end the worker as a fault: the run in flight beside it answers BadResourceUnavailable ("worker restarted"),
+ /// the shard restarts, and the next evaluation is served — instead of an unhandled exception on a dedicated thread,
+ /// which would terminate the host.
+ ///
+ [Fact]
+ public async Task A_completion_callback_that_throws_on_the_reader_ends_that_worker_as_faulted_not_the_host()
+ {
+ await using var runner = Scripts.Runner(Scripts.Options());
+ (await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct)).Value.ShouldBe(7, "positive control: the hook is not armed yet");
+ var worker = runner.Shards[0].Current.ShouldNotBeNull();
+
+ var spin = runner.RunAsync(Scripts.Vt("spin", Runaway, TimeSpan.FromSeconds(30)), Ct);
+ await Scripts.Eventually(() => worker.InFlightCount > 0 ? worker : null, TimeSpan.FromSeconds(30), "the sibling in flight");
+
+ var armed = 1;
+ runner.SlotReleasingForTest = key =>
+ {
+ if (key.EndsWith("boom", StringComparison.Ordinal) && Interlocked.Exchange(ref armed, 0) == 1)
+ throw new InvalidOperationException("a completion callback failed");
+ };
+ await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct);
+
+ var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct);
+ exit.Reason.ShouldBe(WorkerExitReason.ProtocolFault);
+ var sibling = await spin.WaitAsync(TimeSpan.FromSeconds(15), Ct);
+ sibling.Status.ShouldBe(ScriptRunStatus.Unavailable);
+ sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
+ sibling.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable);
+
+ (await runner.RunAsync(Scripts.Vt("after", Healthy), Ct)).Value.ShouldBe(7, "the shard restarted and serves again");
+ runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid);
+ }
+
+ ///
+ /// A worker whose reader stops while its heartbeats go on: the host's writes fill the pipe and the producer (here
+ /// a deploy loading scripts, the case nothing else would end: no evaluation is outstanding and a compile's timeout
+ /// starts only when the worker starts it) blocks in the write. The write-stall check must kill the worker within
+ /// its bound — as unresponsive, not for silence, which proves it kept heartbeating — and that releases the producer.
+ ///
+ [Fact]
+ public async Task A_worker_that_stops_reading_but_keeps_heartbeating_is_killed_and_the_blocked_producer_returns()
+ {
+ var heartbeat = TimeSpan.FromMilliseconds(200);
+ var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
+ 256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: heartbeat,
+ HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30), TestStallReaderOnDiag: true);
+ var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
+ await using var owned = connection;
+ var warm = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Healthy,
+ SHA256.HashData(Encoding.UTF8.GetBytes(Healthy)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
+ warm.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
+ connection.SilenceLimit = heartbeat * 5; // armed once the cold start is over (see SupervisionTests)
+ var stallLimit = TimeSpan.FromSeconds(2);
+ connection.WriteStallLimitOverride = stallLimit;
+
+ _ = connection.DiagnosticsAsync(TimeSpan.FromSeconds(60)); // the worker's reader stops here for good
+ var producer = Task.Factory.StartNew(() =>
+ {
+ // About 5 MB of loads: far more than the pipe's buffer and the worker reader's own.
+ for (var i = 0; i < 20; i++)
+ {
+ var source = $"return {i}; // " + new string('x', 256 * 1024);
+ _ = connection.EnsureLoadedAsync($"big{i}", ContextKind.VirtualTag, source,
+ SHA256.HashData(Encoding.UTF8.GetBytes(source)), []);
+ }
+ }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+
+ var sw = Stopwatch.StartNew();
+ await Task.Delay(stallLimit / 2, Ct);
+ producer.IsCompleted.ShouldBeFalse("positive control: the producer is blocked in a write to the full pipe");
+ connection.IsAlive.ShouldBeTrue("positive control: heartbeats keep the silence check quiet");
+
+ var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(20), Ct);
+ exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive, "killed by the write-stall check, not for silence");
+ await producer.WaitAsync(TimeSpan.FromSeconds(10), Ct);
+ sw.Elapsed.ShouldBeLessThan(stallLimit + TimeSpan.FromSeconds(8), "the producer was released within the bound");
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs
index f8fc959be..8887f52c0 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ProtocolTests.cs
@@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// lmxopcua#780 design §13 — frame codec and the CLR ↔ Variant mapping.
+[Trait("Category", "ScriptWorker")]
public sealed class ProtocolTests
{
public static IEnumerable