Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/BlockingPipeStreamTests.cs
T
Joseph Doherty 03ac57d5d6 fix(scripting): #789 review — reader faults end the worker, write stalls kill it, Cancel off the worker reader
- Host reader: the frame read and its handling are caught separately. Only the
  read's I/O exceptions mean the pipe ended; anything HandleFrame's inline
  completion work throws now kills that worker as ProtocolFault (outstanding
  runs -> worker restarted / BadResourceUnavailable) instead of escaping a
  dedicated thread (process crash) or being swallowed as a clean end (an
  InvalidOperationException used to be, leaving the worker unread until a Crash
  2 s later).
- Worker reader: the same catch-all around HandleFrame (exit ProtocolFault).
- Worker: Cancel() runs on the pool, so a script's blocking
  ctx.CancellationToken callback cannot stop the worker reading its pipe.
- Host write-stall check: ScriptWorkerFrameWriter.WriteInProgressFor; the
  heartbeat monitor kills a worker whose pipe write has been blocked longer than
  max(10 s, silence limit) as KilledUnresponsive, which fails the write and
  releases the producer (seam submit, deploy load, Cancel). Fixes the wrong
  'heartbeat monitor ends it' comment on the pipe buffer quota.
- StopAsync sends Shutdown off the caller so its 1 s budget holds on a full pipe.
- WindowsBlockingPipeStream refuses the async surface (Stream's base versions
  serialise a write behind a waiting read).
- Tests: PipeFaultTests (throwing completion callback -> ProtocolFault and a
  BadResourceUnavailable sibling, shard serves again; a worker that stops reading
  but heartbeats is killed within the bound and the blocked producer returns,
  via the Hello.test_stall_reader_on_diag seam); BlockingPipeStreamTests dispose
  with a write blocked on a full pipe, and with a read and a write both pending;
  Category=ScriptWorker on the protocol, handshake and pipe test classes.
  Mutation checks: no stall kill -> stall test red; catch-all without the fault
  -> callback test red (Crash).

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 20:26:02 -05:00

230 lines
9.5 KiB
C#

using System.IO.Pipes;
using System.Security.Cryptography;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// <summary>
/// lmxopcua#789: <see cref="ScriptWorkerFrameCodec.BlockingStream"/> — on Windows the
/// <see cref="WindowsBlockingPipeStream"/> that keeps worker pipe I/O off the completion port. Exercised on a real
/// pipe pair; on Unix the same tests cover the plain pipe it returns there.
/// </summary>
[Trait("Category", "ScriptWorker")]
public sealed class BlockingPipeStreamTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static async Task<(Stream Server, Stream Client)> PairAsync()
{
var name = "otopcua-sw-test-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(6));
var server = new NamedPipeServerStream(name, PipeDirection.InOut, 1, PipeTransmissionMode.Byte,
PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
var client = new NamedPipeClientStream(".", name, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
var accept = server.WaitForConnectionAsync(Ct);
client.Connect(TimeSpan.FromSeconds(5));
await accept;
return (ScriptWorkerFrameCodec.BlockingStream(server), ScriptWorkerFrameCodec.BlockingStream(client));
}
private static Envelope Frame(ulong id) => new()
{
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
Seq = id,
Heartbeat = new Heartbeat { Inflight = (uint)id },
};
[Fact]
public async Task On_windows_the_pipe_is_wrapped_and_elsewhere_it_is_the_pipe_itself()
{
var (server, client) = await PairAsync();
using (server)
using (client)
{
if (OperatingSystem.IsWindows()) server.ShouldBeOfType<WindowsBlockingPipeStream>();
else server.ShouldBeAssignableTo<PipeStream>();
}
}
/// <summary>
/// Full duplex: each end has a read blocked on a dedicated thread while the other direction keeps writing — a
/// handle opened for synchronous I/O would serialise the write behind the blocked read and deadlock here.
/// </summary>
[Fact]
public async Task Reads_blocked_on_both_ends_do_not_hold_up_writes_in_either_direction()
{
var (server, client) = await PairAsync();
using (server)
using (client)
{
const int frames = 2_000;
Task<int> Pump(Stream from, Stream to)
{
var reader = Task.Factory.StartNew(() =>
{
var r = new ScriptWorkerFrameReader(from);
var seen = 0;
while (seen < frames && r.Read() is { } f)
{
f.Heartbeat.Inflight.ShouldBe((uint)seen);
seen++;
}
return seen;
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
_ = Task.Factory.StartNew(() =>
{
for (var i = 0; i < frames; i++) ScriptWorkerFrameCodec.Write(to, Frame((ulong)i), ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
return reader;
}
var toClient = Pump(client, server);
var toServer = Pump(server, client);
(await toClient.WaitAsync(TimeSpan.FromSeconds(30), Ct)).ShouldBe(frames);
(await toServer.WaitAsync(TimeSpan.FromSeconds(30), Ct)).ShouldBe(frames);
}
}
[Fact]
public async Task A_large_frame_crosses_whole()
{
var (server, client) = await PairAsync();
using (server)
using (client)
{
var big = new Envelope
{
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
Load = new LoadScripts { Scripts = { new ScriptSource { Key = "k", Source = new string('x', 1_000_000) } } },
};
var read = Task.Factory.StartNew(() => new ScriptWorkerFrameReader(server).Read(), Ct, TaskCreationOptions.LongRunning,
TaskScheduler.Default);
ScriptWorkerFrameCodec.Write(client, big, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
var got = await read.WaitAsync(TimeSpan.FromSeconds(30), Ct);
got!.Load.Scripts[0].Source.Length.ShouldBe(1_000_000);
}
}
[Fact]
public async Task A_peer_that_goes_away_ends_a_blocked_read_and_fails_a_write()
{
var (server, client) = await PairAsync();
using (server)
{
var read = Task.Factory.StartNew(() => server.Read(new byte[16], 0, 16), Ct, TaskCreationOptions.LongRunning,
TaskScheduler.Default);
await Task.Delay(100, Ct);
read.IsCompleted.ShouldBeFalse("positive control: the read is blocked on the kernel");
client.Dispose();
(await read.WaitAsync(TimeSpan.FromSeconds(10), Ct)).ShouldBe(0, "end of stream");
Should.Throw<IOException>(() =>
{
// A pipe can accept a write into its buffer before it notices; the second one must fail.
for (var i = 0; i < 4; i++) server.Write(new byte[64], 0, 64);
});
}
}
[Fact]
public async Task Disposing_an_end_wakes_its_own_blocked_reader()
{
var (server, client) = await PairAsync();
using (client)
{
var read = Task.Factory.StartNew(() => server.Read(new byte[16], 0, 16), Ct, TaskCreationOptions.LongRunning,
TaskScheduler.Default);
await Task.Delay(100, Ct);
read.IsCompleted.ShouldBeFalse("positive control: the read is blocked on the kernel");
server.Dispose();
// End of stream, or the disposed / I/O error the readers treat as the pipe ending — never a hang.
int ended;
try { ended = await read.WaitAsync(TimeSpan.FromSeconds(10), Ct); }
catch (Exception ex) when (ex is ObjectDisposedException or IOException) { ended = 0; }
ended.ShouldBe(0);
}
}
/// <summary>
/// Writes until the pipe is full (the peer never reads), then returns the writing task, which is blocked in the
/// kernel at that point; it ends with the exception the write saw.
/// </summary>
private static async Task<Task> BlockedWriterAsync(Stream end)
{
long written = 0;
var writer = Task.Factory.StartNew(() =>
{
var chunk = new byte[64 * 1024];
while (true)
{
end.Write(chunk, 0, chunk.Length);
Interlocked.Add(ref written, chunk.Length);
}
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
// Full once no progress is made for a while (a zero-quota Windows pipe blocks at once; a socket after its buffer).
long last;
do
{
last = Interlocked.Read(ref written);
await Task.Delay(300, Ct);
} while (Interlocked.Read(ref written) != last && !writer.IsCompleted);
writer.IsCompleted.ShouldBeFalse("positive control: the write is blocked on a full pipe");
return writer;
}
private static async Task ShouldEndAsync(Task blocked, string what)
{
var ended = await Task.WhenAny(blocked, Task.Delay(TimeSpan.FromSeconds(10), Ct));
ended.ShouldBeSameAs(blocked, $"{what} ended, not hung");
var ex = blocked.Exception?.GetBaseException();
ex.ShouldNotBeNull($"{what} ended with an error");
(ex is ObjectDisposedException or IOException).ShouldBeTrue(ex.ToString());
}
[Fact]
public async Task Disposing_an_end_wakes_its_own_write_blocked_on_a_full_pipe()
{
var (server, client) = await PairAsync();
using (client)
{
var writer = await BlockedWriterAsync(server);
server.Dispose();
await ShouldEndAsync(writer, "the blocked write");
}
}
[Fact]
public async Task Disposing_an_end_with_a_read_and_a_write_both_pending_wakes_both()
{
var (server, client) = await PairAsync();
using (client)
{
var read = Task.Factory.StartNew(() =>
{
var n = server.Read(new byte[16], 0, 16);
if (n == 0) throw new IOException("end of stream");
return n;
}, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var writer = await BlockedWriterAsync(server);
read.IsCompleted.ShouldBeFalse("positive control: the read is pending beside the blocked write");
server.Dispose();
await ShouldEndAsync(writer, "the blocked write");
await ShouldEndAsync(read, "the pending read");
}
}
[Fact]
public async Task The_asynchronous_surface_is_refused_on_windows()
{
var (server, client) = await PairAsync();
using (server)
using (client)
{
if (!OperatingSystem.IsWindows()) return; // the Unix pipe itself keeps PipeStream's own async surface
await Should.ThrowAsync<NotSupportedException>(() => server.ReadAsync(new byte[4], 0, 4, Ct));
await Should.ThrowAsync<NotSupportedException>(async () => await server.WriteAsync(new byte[4].AsMemory(), Ct));
}
}
}