perf(worker,gateway): pin MXSTATUS_PROXY record info, dedicated event-drain thread, cap thread-pool spin

At ~240 events/s the worker burned 8-12% system CPU on wonder-app-vd03. A WPR
trace put 72% of it in lmxproxy's per-event GetRecordInfoFromGuids (type
library reload through the registry, amplified by WOW64 and the EDR registry
filter) and 26% in the .NET Framework thread pool spinning after each
per-event work item.

- Worker: MxStatusProxyRecordInfoPin holds the LMXProxy ITypeLib and the
  MXSTATUS_PROXY IRecordInfo for the process lifetime, so oleaut32 serves the
  per-event lookup from cache (~1 ms -> <1 us per call). Best-effort, never
  fatal; status logged at handshake.
- Worker: the event drain loop runs on a dedicated background thread with a
  synchronous wait and a synchronous pipe flush, removing the per-event
  thread-pool hops.
- Gateway: runtimeconfig sets ThreadPool.UnfairSemaphoreSpinLimit=10 to cut
  PortableThreadPool spin-wait CPU.

windev bench at ~242 ev/s: worker 67% -> 6.2% of one core; gateway 8.1% -> 6.9%.
Worker suite 785 passed / 14 skipped (x86, windev).
This commit is contained in:
Joseph Doherty
2026-11-21 08:00:13 -05:00
parent 08a9cfd68a
commit 659bb831a2
15 changed files with 398 additions and 34 deletions
@@ -355,6 +355,13 @@ public sealed class WorkerPipeSession
await WriteWorkerHelloAsync(cancellationToken).ConfigureAwait(false);
_state = WorkerState.InitializingSta;
WorkerReady ready = await initializeMxAccessAsync(cancellationToken).ConfigureAwait(false);
if (MxStatusProxyRecordInfoPin.Status is { } pinStatus)
{
_logger?.Information(
"WorkerRecordInfoPin",
new Dictionary<string, object?> { ["status"] = pinStatus });
}
await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false);
_state = WorkerState.Ready;
}
@@ -589,14 +596,59 @@ public sealed class WorkerPipeSession
TaskScheduler.Default);
}
private async Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
/// <summary>
/// Starts the event drain loop on a dedicated background thread and returns a task that
/// completes, faults, or cancels exactly as the loop does, so the message loop's
/// <c>Task.WhenAny</c> supervision is unchanged.
/// </summary>
/// <remarks>
/// The loop used to be an async method woken by <c>SemaphoreSlim.WaitAsync</c>. On .NET
/// Framework every release of that semaphore queues the continuation to the CLR thread pool,
/// and each pool worker that runs one then spins in <c>SleepEx</c> before parking: a WPR trace
/// on wonder-app-vd03 measured ~150 context switches per event and ~26% of the worker's CPU
/// there. A dedicated thread parks on the signal instead, and with a synchronous flush the
/// steady-state event path schedules no managed thread-pool work at all. The pipe write itself
/// stays overlapped; its completion runs on an I/O completion thread, not a worker thread.
/// </remarks>
private Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
{
TaskCompletionSource<bool> completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
Thread thread = new(() =>
{
try
{
RunEventDrainLoop(cancellationToken);
completion.TrySetResult(true);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
completion.TrySetCanceled(cancellationToken);
}
catch (Exception exception)
{
completion.TrySetException(exception);
}
})
{
IsBackground = true,
Name = "mxgw-event-drain",
};
thread.Start();
return completion.Task;
}
private void RunEventDrainLoop(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
IWorkerRuntimeSession? runtimeSession = _runtimeSession;
if (runtimeSession is null)
{
await Task.Delay(EventDrainInterval, cancellationToken).ConfigureAwait(false);
if (cancellationToken.WaitHandle.WaitOne(EventDrainInterval))
{
cancellationToken.ThrowIfCancellationRequested();
}
continue;
}
@@ -604,7 +656,7 @@ public sealed class WorkerPipeSession
if (fault is not null)
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
TryWriteFaultAsync(fault, cancellationToken).GetAwaiter().GetResult();
throw new WorkerEventQueueFaultException(
fault.Category,
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
@@ -615,15 +667,13 @@ public sealed class WorkerPipeSession
IReadOnlyList<WorkerEvent> events = runtimeSession.DrainEvents(EventDrainBatchSize);
if (events.Count == 0)
{
// Wait on the queue's wake signal rather than sleeping a fixed tick: an event
// enqueued by the STA completes this immediately, so the first event of a burst is
// framed at signal latency instead of waiting out EventDrainInterval, and a session
// with no traffic stops waking at all. The wait's outcome is intentionally ignored —
// whether a signal or the fallback ended it, the next pass re-checks DrainFault()
// and re-drains, which is also why one coalesced wake for many enqueues is safe.
await runtimeSession
.WaitForEventsAsync(EventDrainInterval, cancellationToken)
.ConfigureAwait(false);
// Park on the queue's wake signal rather than sleeping a fixed tick: an event
// enqueued by the STA releases this immediately, so the first event of a burst is
// framed at signal latency, and a session with no traffic wakes only at the fallback
// ceiling. The wait's outcome is intentionally ignored — whether a signal or the
// fallback ended it, the next pass re-checks DrainFault() and re-drains, which is
// also why one coalesced wake for many enqueues is safe.
runtimeSession.WaitForEvents(EventDrainInterval, cancellationToken);
continue;
}
@@ -640,9 +690,10 @@ public sealed class WorkerPipeSession
try
{
await _writer
_writer
.WriteBatchAsync(envelopes, WorkerFrameWritePriority.Event, cancellationToken)
.ConfigureAwait(false);
.GetAwaiter()
.GetResult();
}
catch (WorkerFrameProtocolException exception)
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
@@ -651,11 +702,12 @@ public sealed class WorkerPipeSession
// still the structured, event-naming fault. Map the rejection back to the first
// event in batch order whose envelope overshoots the negotiated maximum — the same
// frame the writer rejected first.
await FaultOnOversizedEventAsync(
FaultOnOversizedEventAsync(
FindOversizedEvent(events, envelopes),
exception,
cancellationToken)
.ConfigureAwait(false);
.GetAwaiter()
.GetResult();
}
}
}