perf(worker,gateway): pin MXSTATUS_PROXY record info, dedicated event-drain thread, cap thread-pool spin

At ~240 events/s the worker burned 8-12% system CPU on wonder-app-vd03. A WPR
trace put 72% of it in lmxproxy's per-event GetRecordInfoFromGuids (type
library reload through the registry, amplified by WOW64 and the EDR registry
filter) and 26% in the .NET Framework thread pool spinning after each
per-event work item.

- Worker: MxStatusProxyRecordInfoPin holds the LMXProxy ITypeLib and the
  MXSTATUS_PROXY IRecordInfo for the process lifetime, so oleaut32 serves the
  per-event lookup from cache (~1 ms -> <1 us per call). Best-effort, never
  fatal; status logged at handshake.
- Worker: the event drain loop runs on a dedicated background thread with a
  synchronous wait and a synchronous pipe flush, removing the per-event
  thread-pool hops.
- Gateway: runtimeconfig sets ThreadPool.UnfairSemaphoreSpinLimit=10 to cut
  PortableThreadPool spin-wait CPU.

windev bench at ~242 ev/s: worker 67% -> 6.2% of one core; gateway 8.1% -> 6.9%.
Worker suite 785 passed / 14 skipped (x86, windev).
This commit is contained in:
Joseph Doherty
2026-11-21 08:00:13 -05:00
parent 08a9cfd68a
commit 659bb831a2
15 changed files with 398 additions and 34 deletions
@@ -38,6 +38,17 @@
<PackageReference Include="Polly.Core" Version="8.6.6" />
</ItemGroup>
<ItemGroup>
<!--
Cap the thread pool's idle spin. The default (70 steps, Sleep(0) on every other step past
the tenth) makes a worker that finds no work yield ~30 times before parking; with several
pool work items per relayed event, a WPR trace on a busy 4-CPU host put ~93% of the
gateway's context switches in those yields. 10 keeps a short busy-spin and then parks.
Measured on windev at ~240 events/s: gateway CPU -15%. See docs/GatewayConfiguration.md.
-->
<RuntimeHostConfigurationOption Include="System.Threading.ThreadPool.UnfairSemaphoreSpinLimit" Value="10" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ZB.MOM.WW.MxGateway.Contracts\ZB.MOM.WW.MxGateway.Contracts.csproj" />
</ItemGroup>
@@ -375,7 +375,7 @@ public sealed class WorkerFrameProtocolTests
/// <summary>
/// Verifies the writer coalesces the flush across a batch of frames drained together: four frames
/// queued behind an in-progress write drain in a single pass and share one FlushAsync, not four.
/// queued behind an in-progress write drain in a single pass and share one Flush, not four.
/// Every frame still reaches the wire intact.
/// <para>
/// The burst is all-event on purpose. The control-frame completion decoupling made the drain flush
@@ -1386,20 +1386,20 @@ public sealed class WorkerFrameProtocolTests
};
}
// A MemoryStream that counts FlushAsync calls without gating any write, so a batch write can be
// A MemoryStream that counts Flush calls without gating any write, so a batch write can be
// asserted to flush exactly once.
private sealed class FlushCountingStream : MemoryStream
{
private int _flushCount;
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
/// <summary>Gets the number of <see cref="Flush"/> calls observed so far.</summary>
public int FlushCount => Volatile.Read(ref _flushCount);
/// <inheritdoc />
public override Task FlushAsync(CancellationToken cancellationToken)
public override void Flush()
{
Interlocked.Increment(ref _flushCount);
return base.FlushAsync(cancellationToken);
base.Flush();
}
}
@@ -1500,7 +1500,7 @@ public sealed class WorkerFrameProtocolTests
/// <summary>Gets a task that completes once the second gated <see cref="WriteAsync"/> call has started blocking.</summary>
public Task SecondGateWriteStarted => _secondGateWriteStarted.Task;
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
/// <summary>Gets the number of <see cref="Flush"/> calls observed so far.</summary>
public int FlushCount => Volatile.Read(ref _flushCount);
/// <summary>Releases the first blocked write so it can complete.</summary>
@@ -1528,10 +1528,10 @@ public sealed class WorkerFrameProtocolTests
}
/// <inheritdoc />
public override Task FlushAsync(CancellationToken cancellationToken)
public override void Flush()
{
Interlocked.Increment(ref _flushCount);
return base.FlushAsync(cancellationToken);
base.Flush();
}
/// <inheritdoc />
@@ -1547,7 +1547,7 @@ public sealed class WorkerFrameProtocolTests
}
}
// A MemoryStream whose first write blocks until released and whose every FlushAsync throws, so a test
// A MemoryStream whose first write blocks until released and whose every Flush throws, so a test
// can fault the class-boundary flush with control frames already written and an event frame already
// claimed off its queue.
private sealed class FlushFaultingGatedStream : MemoryStream
@@ -1584,9 +1584,9 @@ public sealed class WorkerFrameProtocolTests
}
/// <inheritdoc />
public override Task FlushAsync(CancellationToken cancellationToken)
public override void Flush()
{
return Task.FromException(new IOException(_faultMessage));
throw new IOException(_faultMessage);
}
/// <inheritdoc />
@@ -367,6 +367,45 @@ public sealed class WorkerPipeSessionTests
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
/// <summary>
/// The event drain loop runs on its own dedicated background thread, not the CLR thread
/// pool: on .NET Framework every pool continuation per event also paid the pool's idle
/// spin, which a production trace measured at ~150 context switches per event. The parked
/// wait is observed from the fake, which records the calling thread.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RunAsync_EventDrainLoop_RunsOnDedicatedNonPoolThread()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new()
{
WaitForEventsOnSignalOnly = true,
};
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
while (runtime.LastWaitForEventsTimeout is null)
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
}
Assert.Equal("mxgw-event-drain", runtime.LastWaitForEventsThreadName);
Assert.False(runtime.LastWaitForEventsOnThreadPool);
// The dedicated thread still ships events and still stops cleanly on shutdown.
runtime.EnqueueEvent(CreateWorkerEvent(sequence: 3));
WorkerEnvelope workerEvent = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerEvent,
cancellation.Token);
Assert.Equal(3UL, workerEvent.WorkerEvent.Event.WorkerSequence);
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
/// <summary>
/// Verifies that a Ping control command is answered on the worker side
/// (not dispatched to the STA) with an OK reply that echoes the ping
@@ -0,0 +1,28 @@
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess;
/// <summary>
/// The record-info pin is a best-effort, process-wide cache warm-up. Whether it succeeds depends
/// on the host's MXAccess registration, so these tests assert only what must hold on every host:
/// it never throws, it always reports an outcome, and only the first call does work.
/// </summary>
public sealed class MxStatusProxyRecordInfoPinTests
{
/// <summary>Repeated calls return the same outcome and leave a status describing it.</summary>
[Fact]
public void EnsurePinned_CalledTwice_IsIdempotentAndReportsStatus()
{
bool first = MxStatusProxyRecordInfoPin.EnsurePinned();
string? statusAfterFirst = MxStatusProxyRecordInfoPin.Status;
bool second = MxStatusProxyRecordInfoPin.EnsurePinned();
Assert.Equal(first, second);
Assert.False(string.IsNullOrWhiteSpace(statusAfterFirst));
Assert.Same(statusAfterFirst, MxStatusProxyRecordInfoPin.Status);
if (first)
{
Assert.StartsWith("pinned ", statusAfterFirst);
}
}
}
@@ -387,6 +387,22 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
return eventSignal.WaitAsync(timeout, cancellationToken);
}
/// <summary>Gets the name of the thread that made the most recent <see cref="WaitForEvents"/> call.</summary>
public string? LastWaitForEventsThreadName { get; private set; }
/// <summary>Gets whether the most recent <see cref="WaitForEvents"/> call ran on a thread-pool thread.</summary>
public bool? LastWaitForEventsOnThreadPool { get; private set; }
/// <inheritdoc />
public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken)
{
LastWaitForEventsThreadName = Thread.CurrentThread.Name;
LastWaitForEventsOnThreadPool = Thread.CurrentThread.IsThreadPoolThread;
// Same knobs as the async form (signal-only, recorded timeout, backing queue); the drain
// loop now runs on a dedicated thread, so blocking on the task here is the production shape.
WaitForEventsAsync(timeout, cancellationToken).GetAwaiter().GetResult();
}
/// <summary>
/// Gets or sets a fault the next <see cref="DrainFault"/> call hands back once, standing in
/// for the fault a real event queue self-records on overflow.
@@ -588,7 +588,7 @@ public sealed class WorkerFrameWriter
// and now flushed, so completing them early is the same contract, earlier.
try
{
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
FlushStream();
}
catch (Exception exception)
{
@@ -640,7 +640,7 @@ public sealed class WorkerFrameWriter
try
{
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
FlushStream();
}
catch (Exception exception)
{
@@ -655,6 +655,15 @@ public sealed class WorkerFrameWriter
CompleteFrames(written);
}
// Synchronous on purpose. PipeStream does not override FlushAsync, so the base Stream.FlushAsync
// runs Flush() through Task.Factory.StartNew: one thread-pool work item per drain pass, for a
// PipeStream.Flush that has nothing to do (it never calls FlushFileBuffers). Calling Flush()
// inline keeps the contract for any stream and drops the pool hop.
private void FlushStream()
{
_stream.Flush();
}
private static void CompleteFrames(List<PendingFrame> frames)
{
foreach (PendingFrame frame in frames)
@@ -355,6 +355,13 @@ public sealed class WorkerPipeSession
await WriteWorkerHelloAsync(cancellationToken).ConfigureAwait(false);
_state = WorkerState.InitializingSta;
WorkerReady ready = await initializeMxAccessAsync(cancellationToken).ConfigureAwait(false);
if (MxStatusProxyRecordInfoPin.Status is { } pinStatus)
{
_logger?.Information(
"WorkerRecordInfoPin",
new Dictionary<string, object?> { ["status"] = pinStatus });
}
await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false);
_state = WorkerState.Ready;
}
@@ -589,14 +596,59 @@ public sealed class WorkerPipeSession
TaskScheduler.Default);
}
private async Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
/// <summary>
/// Starts the event drain loop on a dedicated background thread and returns a task that
/// completes, faults, or cancels exactly as the loop does, so the message loop's
/// <c>Task.WhenAny</c> supervision is unchanged.
/// </summary>
/// <remarks>
/// The loop used to be an async method woken by <c>SemaphoreSlim.WaitAsync</c>. On .NET
/// Framework every release of that semaphore queues the continuation to the CLR thread pool,
/// and each pool worker that runs one then spins in <c>SleepEx</c> before parking: a WPR trace
/// on wonder-app-vd03 measured ~150 context switches per event and ~26% of the worker's CPU
/// there. A dedicated thread parks on the signal instead, and with a synchronous flush the
/// steady-state event path schedules no managed thread-pool work at all. The pipe write itself
/// stays overlapped; its completion runs on an I/O completion thread, not a worker thread.
/// </remarks>
private Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
{
TaskCompletionSource<bool> completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
Thread thread = new(() =>
{
try
{
RunEventDrainLoop(cancellationToken);
completion.TrySetResult(true);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
completion.TrySetCanceled(cancellationToken);
}
catch (Exception exception)
{
completion.TrySetException(exception);
}
})
{
IsBackground = true,
Name = "mxgw-event-drain",
};
thread.Start();
return completion.Task;
}
private void RunEventDrainLoop(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
IWorkerRuntimeSession? runtimeSession = _runtimeSession;
if (runtimeSession is null)
{
await Task.Delay(EventDrainInterval, cancellationToken).ConfigureAwait(false);
if (cancellationToken.WaitHandle.WaitOne(EventDrainInterval))
{
cancellationToken.ThrowIfCancellationRequested();
}
continue;
}
@@ -604,7 +656,7 @@ public sealed class WorkerPipeSession
if (fault is not null)
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
TryWriteFaultAsync(fault, cancellationToken).GetAwaiter().GetResult();
throw new WorkerEventQueueFaultException(
fault.Category,
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
@@ -615,15 +667,13 @@ public sealed class WorkerPipeSession
IReadOnlyList<WorkerEvent> events = runtimeSession.DrainEvents(EventDrainBatchSize);
if (events.Count == 0)
{
// Wait on the queue's wake signal rather than sleeping a fixed tick: an event
// enqueued by the STA completes this immediately, so the first event of a burst is
// framed at signal latency instead of waiting out EventDrainInterval, and a session
// with no traffic stops waking at all. The wait's outcome is intentionally ignored —
// whether a signal or the fallback ended it, the next pass re-checks DrainFault()
// and re-drains, which is also why one coalesced wake for many enqueues is safe.
await runtimeSession
.WaitForEventsAsync(EventDrainInterval, cancellationToken)
.ConfigureAwait(false);
// Park on the queue's wake signal rather than sleeping a fixed tick: an event
// enqueued by the STA releases this immediately, so the first event of a burst is
// framed at signal latency, and a session with no traffic wakes only at the fallback
// ceiling. The wait's outcome is intentionally ignored — whether a signal or the
// fallback ended it, the next pass re-checks DrainFault() and re-drains, which is
// also why one coalesced wake for many enqueues is safe.
runtimeSession.WaitForEvents(EventDrainInterval, cancellationToken);
continue;
}
@@ -640,9 +690,10 @@ public sealed class WorkerPipeSession
try
{
await _writer
_writer
.WriteBatchAsync(envelopes, WorkerFrameWritePriority.Event, cancellationToken)
.ConfigureAwait(false);
.GetAwaiter()
.GetResult();
}
catch (WorkerFrameProtocolException exception)
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
@@ -651,11 +702,12 @@ public sealed class WorkerPipeSession
// still the structured, event-naming fault. Map the rejection back to the first
// event in batch order whose envelope overshoots the negotiated maximum — the same
// frame the writer rejected first.
await FaultOnOversizedEventAsync(
FaultOnOversizedEventAsync(
FindOversizedEvent(events, envelopes),
exception,
cancellationToken)
.ConfigureAwait(false);
.GetAwaiter()
.GetResult();
}
}
}
@@ -79,6 +79,16 @@ public interface IWorkerRuntimeSession : IDisposable
/// <returns>A task that completes when the queue is signalled or the timeout elapses.</returns>
Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken);
/// <summary>
/// Blocking form of <see cref="WaitForEventsAsync"/> for a drain loop that runs on its own
/// dedicated thread. Same contract (coalesced signal, timeout is a ceiling, at most one
/// waiter); it parks the calling thread instead of scheduling a thread-pool continuation per
/// wake, which on .NET Framework also triggers the pool's spin-wait on every event.
/// </summary>
/// <param name="timeout">Maximum time to wait before the wait completes unsignalled.</param>
/// <param name="cancellationToken">Token cancelling the wait.</param>
void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken);
/// <summary>
/// Drains a pending fault from the queue, if any.
/// </summary>
@@ -8,6 +8,9 @@ public sealed class MxAccessComObjectFactory : IMxAccessComObjectFactory
/// <inheritdoc />
public object Create()
{
// Warm oleaut32's record-info cache before the first OnDataChange can fire; see
// MxStatusProxyRecordInfoPin for why the per-event lookup is otherwise expensive.
MxStatusProxyRecordInfoPin.EnsurePinned();
return new LMXProxyServerClass();
}
}
@@ -320,6 +320,17 @@ public sealed class MxAccessEventQueue
return eventSignal.WaitAsync(timeout, cancellationToken);
}
/// <summary>
/// Blocking form of <see cref="WaitForEventsAsync"/>: parks the calling (dedicated drain)
/// thread on the same coalesced signal. Same single-waiter precondition.
/// </summary>
/// <param name="timeout">Maximum time to wait before the wait completes unsignalled.</param>
/// <param name="cancellationToken">Token cancelling the wait.</param>
public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken)
{
_ = eventSignal.Wait(timeout, cancellationToken);
}
/// <summary>
/// Attempts to dequeue the next event without removing it if empty.
/// </summary>
@@ -682,6 +682,12 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
return eventQueue.WaitForEventsAsync(timeout, cancellationToken);
}
/// <inheritdoc />
public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken)
{
eventQueue.WaitForEvents(timeout, cancellationToken);
}
/// <inheritdoc />
public WorkerFault? DrainFault()
{
@@ -0,0 +1,113 @@
using System;
using System.Linq;
using System.Runtime.InteropServices;
using ArchestrA.MxAccess;
namespace ZB.MOM.WW.MxGateway.Worker.MxAccess;
/// <summary>
/// Keeps the LMXProxy type library and the <c>MXSTATUS_PROXY</c> record info alive for the
/// lifetime of the worker process, so the MXAccess proxy's per-event
/// <c>GetRecordInfoFromGuids</c> call is served from oleaut32's cache.
/// </summary>
/// <remarks>
/// Every <c>OnDataChange</c> carries a SAFEARRAY of <c>MXSTATUS_PROXY</c> records, and
/// <c>lmxproxy.dll</c> resolves that record type with <c>GetRecordInfoFromGuids</c> each time it
/// builds the array. With no live reference, oleaut32 re-resolves the type library through the
/// registry and reloads it on every call: about 1 ms per event on an idle dev box. Under WOW64
/// and an EDR registry filter it is worse; a WPR trace on wonder-app-vd03 put it at about 70% of
/// the STA's CPU. While one <c>IRecordInfo</c> for the type is held, the same call returns the
/// cached instance in well under a microsecond. The pin is a process-wide cache warm-up only: it
/// changes no MXAccess behavior, and failure to pin is reported but never fatal.
/// </remarks>
public static class MxStatusProxyRecordInfoPin
{
private static readonly object Gate = new();
private static IntPtr pinnedTypeLib;
private static IntPtr pinnedRecordInfo;
private static bool attempted;
/// <summary>Gets the outcome of the pin attempt, or <see langword="null"/> before the first attempt.</summary>
public static string? Status { get; private set; }
/// <summary>
/// Pins the type library and record info once per process. Safe to call repeatedly and from
/// any thread; only the first call does work.
/// </summary>
/// <returns><see langword="true"/> when the record info is pinned.</returns>
public static bool EnsurePinned()
{
lock (Gate)
{
if (attempted)
{
return pinnedRecordInfo != IntPtr.Zero;
}
attempted = true;
try
{
Guid libraryGuid = typeof(LMXProxyServerClass).Assembly
.GetCustomAttributes(typeof(GuidAttribute), inherit: false)
.Cast<GuidAttribute>()
.Select(attribute => new Guid(attribute.Value))
.Single();
TypeLibVersionAttribute version = typeof(LMXProxyServerClass).Assembly
.GetCustomAttributes(typeof(TypeLibVersionAttribute), inherit: false)
.Cast<TypeLibVersionAttribute>()
.Single();
Guid recordGuid = typeof(MXSTATUS_PROXY).GUID;
int hr = LoadRegTypeLib(
ref libraryGuid,
(ushort)version.MajorVersion,
(ushort)version.MinorVersion,
0,
out pinnedTypeLib);
if (hr != 0)
{
Status = $"LoadRegTypeLib failed (HRESULT 0x{hr:X8})";
return false;
}
hr = GetRecordInfoFromGuids(
ref libraryGuid,
(uint)version.MajorVersion,
(uint)version.MinorVersion,
0,
ref recordGuid,
out pinnedRecordInfo);
if (hr != 0)
{
Status = $"GetRecordInfoFromGuids failed (HRESULT 0x{hr:X8})";
return false;
}
Status = $"pinned {libraryGuid:B} v{version.MajorVersion}.{version.MinorVersion} record {recordGuid:B}";
return true;
}
catch (Exception exception)
{
Status = $"pin failed: {exception.GetType().Name}: {exception.Message}";
return false;
}
}
}
[DllImport("oleaut32.dll")]
private static extern int LoadRegTypeLib(
ref Guid libraryGuid,
ushort majorVersion,
ushort minorVersion,
uint lcid,
out IntPtr typeLib);
[DllImport("oleaut32.dll")]
private static extern int GetRecordInfoFromGuids(
ref Guid libraryGuid,
uint majorVersion,
uint minorVersion,
uint lcid,
ref Guid typeGuid,
out IntPtr recordInfo);
}