dohertj2/lmxopcua
1
0
Fork 0
Code Issues 4 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf(scripting): #789 Windows worker pipe I/O blocks on dedicated threads, off the completion port

Browse Source 
A .NET pipe opened Asynchronous completes even its synchronous Read/Write
through the thread pool's completion port, so every frame on Windows cost a
completion packet plus a pool work item (and the pool's spin-wait). New
WindowsBlockingPipeStream issues its own overlapped ReadFile/WriteFile with the
OVERLAPPED.hEvent low bit set (no completion packet) and waits on that event on
the calling thread; reads and writes stay concurrent, unlike a synchronous
handle. ScriptWorkerFrameCodec.BlockingStream picks it on Windows and the pipe
itself on Unix, so both ends now use the dedicated blocking reader + synchronous
combining writer on every OS (UseBlockingIo and the async read loops are gone).

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 19:11:10 -05:00
parent f109e5f29e
commit 3c0e5ea4d7
7 changed files with 328 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs
+18 -35
View File
@@ -138,7 +138,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
];
private readonly Process _process;
private readonly NamedPipeServerStream _pipe;
/// <summary>
/// The pipe as both directions drive it: blocking, through <see cref="ScriptWorkerFrameCodec.BlockingStream"/>
/// (the pipe itself on Unix, a <see cref="WindowsBlockingPipeStream"/> over it on Windows). Owns the pipe.
/// </summary>
private readonly Stream _pipe;
private readonly IDisposable? _job;
private readonly IDisposable? _stdio;
private readonly string _workDir;
@@ -171,7 +175,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
private long _nextLoadId;
private int _disposed;
private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, IDisposable? stdio,
private WorkerConnection(string shard, Process process, Stream pipe, IDisposable? job, IDisposable? stdio,
string workDir, ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, TimeSpan loadTimeout,
Action<WorkerExit> onExiting, string? markerPath, bool workerSoftClock, TimeSpan? heartbeatSilenceLimit)
{
@@ -192,7 +196,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
Pid = process.Id;
// The handshake used seq 1. A failed write means the pipe is gone: the worker is dead or dying.
_writer = new ScriptWorkerFrameWriter(pipe, OnWriteFault, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, firstSeq: 1,
synchronous: ScriptWorkerFrameCodec.UseBlockingIo);
synchronous: true);
_dispatcher = new ResultDispatcher($"ScriptWorker[{shard}].Results");
}
@@ -312,6 +316,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
? WorkerPipeSecurity.CreateServer(pipeName, WorkerPipeSecurity.ForCurrentHost(spec.WorkerSid))
: new NamedPipeServerStream(pipeName, PipeDirection.InOut, maxNumberOfServerInstances: 1,
PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
Stream? io = null;
Process? process = null;
IDisposable? job = null;
IDisposable? stdio = null;
@@ -381,11 +386,10 @@ internal sealed class WorkerConnection : IAsyncDisposable
hello.PeCacheDir = cacheDir;
hello.PeCacheKey = Google.Protobuf.ByteString.CopyFrom(cacheKey);
}
var ready = ScriptWorkerFrameCodec.UseBlockingIo
? await HandshakeBlockingAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false)
: await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
io = ScriptWorkerFrameCodec.BlockingStream(pipe);
var ready = await HandshakeBlockingAsync(io, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
var connection = new WorkerConnection(spec.Shard, process, pipe, job, stdio, workDir, log, onLogs, spec.HeartbeatInterval,
var connection = new WorkerConnection(spec.Shard, process, io, job, stdio, workDir, log, onLogs, spec.HeartbeatInterval,
spec.EffectiveLoadTimeout, onExiting, markerPath, workerSoftClock: spec.WorkerSoftClock && ready.WorkerSoftClock,
heartbeatSilenceLimit: spec.HeartbeatSilenceLimit);
connection.Begin();
@@ -397,7 +401,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
job?.Dispose();
stdio?.Dispose();
process?.Dispose();
await pipe.DisposeAsync().ConfigureAwait(false);
if (io is not null) await io.DisposeAsync().ConfigureAwait(false);
else await pipe.DisposeAsync().ConfigureAwait(false);
TryDeleteDirectory(workDir);
WorkerLaunchMarker.TryDelete(markerPath);
throw;
@@ -430,7 +435,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
}
/// <summary>
/// As <see cref="HandshakeAsync"/> with blocking I/O only (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>):
/// As <see cref="HandshakeAsync"/> with blocking I/O only (<see cref="ScriptWorkerFrameCodec.BlockingStream"/>):
/// one asynchronous call would turn the socket under a Unix pipe non-blocking for the connection's whole life.
/// The blocking read runs off the caller and is abandoned at <paramref name="ct"/>; the caller then kills the
/// worker and disposes the pipe, which ends it.
@@ -548,8 +553,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
ReadyTimestamp = Stopwatch.GetTimestamp();
Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
Interlocked.Exchange(ref _lastUsedTimestamp, ReadyTimestamp);
if (!ScriptWorkerFrameCodec.UseBlockingIo) _ = Task.Run(ReadLoopAsync);
else new Thread(ReadLoopBlocking) { IsBackground = true, Name = $"ScriptWorker[{Shard}].Reader" }.Start();
new Thread(ReadLoopBlocking) { IsBackground = true, Name = $"ScriptWorker[{Shard}].Reader" }.Start();
_ = Task.Run(HeartbeatMonitorAsync);
_ = Task.Run(WatchProcessAsync);
}
@@ -785,11 +789,9 @@ internal sealed class WorkerConnection : IAsyncDisposable
}
/// <summary>
/// The reader, on a dedicated thread with blocking reads of a socket that never turns non-blocking
/// (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, lmxopcua#789): a frame wakes this thread straight from
/// the kernel, instead of the socket engine's epoll thread queueing a thread-pool work item per read. Not on
/// Windows, where a synchronous read of an overlapped pipe is itself completed through the I/O completion port;
/// the async loop is the cheaper path there.
/// The reader, on a dedicated thread with blocking reads (<see cref="ScriptWorkerFrameCodec.BlockingStream"/>,
/// lmxopcua#789): a frame wakes this thread straight from the kernel, instead of the socket engine's epoll thread
/// (Unix) or the completion port (Windows) queueing a thread-pool work item per read.
/// </summary>
private void ReadLoopBlocking()
{
@@ -810,25 +812,6 @@ internal sealed class WorkerConnection : IAsyncDisposable
_ = PipeEndedAsync(fault);
}
private async Task ReadLoopAsync()
{
WorkerExitReason? fault = null;
var reader = new ScriptWorkerFrameReader(_pipe);
try
{
while (await reader.ReadAsync(_stopping.Token).ConfigureAwait(false) is { } frame) HandleFrame(frame);
}
catch (OperationCanceledException) { }
catch (ScriptWorkerProtocolException ex)
{
fault = ProtocolFault(ex);
}
catch (Exception ex) when (ex is IOException or ObjectDisposedException)
{
}
await PipeEndedAsync(fault).ConfigureAwait(false);
}
private WorkerExitReason ProtocolFault(ScriptWorkerProtocolException ex)
{
_log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);