From 659bb831a241202ecb73a80d415c18e282fcb8f6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 21 Nov 2026 08:00:13 -0500 Subject: [PATCH] perf(worker,gateway): pin MXSTATUS_PROXY record info, dedicated event-drain thread, cap thread-pool spin At ~240 events/s the worker burned 8-12% system CPU on wonder-app-vd03. A WPR trace put 72% of it in lmxproxy's per-event GetRecordInfoFromGuids (type library reload through the registry, amplified by WOW64 and the EDR registry filter) and 26% in the .NET Framework thread pool spinning after each per-event work item. - Worker: MxStatusProxyRecordInfoPin holds the LMXProxy ITypeLib and the MXSTATUS_PROXY IRecordInfo for the process lifetime, so oleaut32 serves the per-event lookup from cache (~1 ms -> <1 us per call). Best-effort, never fatal; status logged at handshake. - Worker: the event drain loop runs on a dedicated background thread with a synchronous wait and a synchronous pipe flush, removing the per-event thread-pool hops. - Gateway: runtimeconfig sets ThreadPool.UnfairSemaphoreSpinLimit=10 to cut PortableThreadPool spin-wait CPU. windev bench at ~242 ev/s: worker 67% -> 6.2% of one core; gateway 8.1% -> 6.9%. Worker suite 785 passed / 14 skipped (x86, windev). --- docs/GatewayConfiguration.md | 15 +++ docs/MxAccessWorkerInstanceDesign.md | 50 +++++++- docs/WorkerFrameProtocol.md | 11 +- .../ZB.MOM.WW.MxGateway.Server.csproj | 11 ++ .../Ipc/WorkerFrameProtocolTests.cs | 22 ++-- .../Ipc/WorkerPipeSessionTests.cs | 39 ++++++ .../MxStatusProxyRecordInfoPinTests.cs | 28 +++++ .../TestSupport/FakeRuntimeSession.cs | 16 +++ .../Ipc/WorkerFrameWriter.cs | 13 +- .../Ipc/WorkerPipeSession.cs | 84 ++++++++++--- .../MxAccess/IWorkerRuntimeSession.cs | 10 ++ .../MxAccess/MxAccessComObjectFactory.cs | 3 + .../MxAccess/MxAccessEventQueue.cs | 11 ++ .../MxAccess/MxAccessStaSession.cs | 6 + .../MxAccess/MxStatusProxyRecordInfoPin.cs | 113 ++++++++++++++++++ 15 files changed, 398 insertions(+), 34 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxStatusProxyRecordInfoPinTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxStatusProxyRecordInfoPin.cs diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 5b76d053..076705f4 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -807,6 +807,21 @@ Worst-case on-disk log footprint is therefore about `retainedFileCountLimit x fi Raise or lower these in the deployed `appsettings.json` (or via `Serilog__WriteTo__1__Args__*` environment variables) to fit the host's disk budget. +## Runtime Settings (runtimeconfig) + +Some settings belong to the .NET runtime rather than to `MxGateway:*` options. They ship in the +build through `RuntimeHostConfigurationOption` items in the Server csproj, which write them into +`ZB.MOM.WW.MxGateway.Server.runtimeconfig.json`. They therefore travel with every deploy and need +no NSSM environment variables. + +| Setting | Shipped value | Why | +|---|---|---| +| `System.Threading.ThreadPool.UnfairSemaphoreSpinLimit` | `10` (runtime default `70`) | A pool worker that finds no work spins before parking, and past step 10 it calls `Sleep(0)` on every other step. The event relay queues several pool work items per event, so on a busy host those spins became ~93% of the gateway's context switches (WPR trace on a 4-CPU production host, ~240 events/s). With `10`, idle workers spin briefly and then park. Measured on the dev box at ~240 events/s: gateway CPU −15%. | + +To override a setting on one host without rebuilding, use the environment-variable form, which +takes precedence over runtimeconfig: `DOTNET_ThreadPool_UnfairSemaphoreSpinLimit=`. `0` +disables the spin entirely. + ## Related Documentation - [Gateway Process Detailed Design](./GatewayProcessDesign.md) diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index c3e866c8..10553c7d 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -479,6 +479,32 @@ Creation rules: - Never marshal the raw COM object to pipe reader/writer threads. - Capture COM creation HRESULT or exception details. +**Record-info pin.** Before the first `LMXProxyServerClass` is created, +`MxAccessComObjectFactory.Create()` calls `MxStatusProxyRecordInfoPin.EnsurePinned()`. Once per +process, this loads the LMXProxy type library and the `MXSTATUS_PROXY` record info, and holds +both references until exit. The GUIDs and version come from the interop assembly's metadata: +library `{77e896ec-b3e3-4de4-b96f-f32570164211}` v3.2 (`MXAccess32.tlb`), record +`{A162D9E4-9019-42F3-A087-79264007E542}`. + +The pin exists because every `OnDataChange` carries a SAFEARRAY of `MXSTATUS_PROXY` records, and +`lmxproxy.dll` resolves that record type with `GetRecordInfoFromGuids` each time it builds one. +With no live reference, oleaut32 re-reads the registry and reloads the type library on every +call: + +- about 1 ms per event on the dev box; +- worse under WOW64 registry redirection plus an EDR registry filter, where a production WPR trace + put it at ~70% of the STA's CPU. + +While one `IRecordInfo` is held, the same call returns oleaut32's cached instance in under a +microsecond. Measured at ~240 events/s on the dev box, worker CPU fell from 67% to 10% of a core +with the pin alone. + +The pin only warms a cache. It changes no MXAccess behaviour, and a failure is never fatal. The +outcome is in `MxStatusProxyRecordInfoPin.Status`, and the worker logs it as `WorkerRecordInfoPin` +just before `WorkerReady`. Worker logs go to stderr, which the gateway does not redirect, so the +line lands wherever the host captures the gateway's stderr (the NSSM stderr log on deployed +boxes). + If COM creation fails, the worker should send a structured fault with: - fault category, @@ -564,9 +590,27 @@ everything that arrived — the loop must therefore re-check `DrainFault()` and re-drain after every wait, never treat a wake as "exactly one event". The 25 ms `EventDrainInterval` survives as the **fallback ceiling** on an unsignalled wait, not as a latency floor: an event arriving at an idle worker is framed at signal -latency instead of waiting out a tick, an idle worker parks instead of waking 40 -times a second, and the interval only bounds how long the loop may sleep if some -future path mutates the queue without signalling. +latency instead of waiting out a tick, and the interval only bounds how long the +loop may sleep if some future path mutates the queue without signalling. An idle +worker still wakes at that ceiling, but a wake on an empty queue is cheap. + +The loop runs on a **dedicated background thread** (`mxgw-event-drain`), not on the +CLR thread pool: + +- It parks in the blocking `IWorkerRuntimeSession.WaitForEvents`, which uses the + same coalesced signal and single-waiter contract as `WaitForEventsAsync`. +- It blocks on the frame writer's batch write. +- `RunEventDrainLoopAsync` starts the thread and returns a task that completes, + faults or cancels exactly as the loop does, so the message loop's `Task.WhenAny` + supervision is unchanged. + +The reason is .NET Framework's thread pool. As an async loop, every enqueue's +semaphore release queued a continuation to the pool, and each pool worker that ran +one then spun in `SleepEx` before parking. A production WPR trace measured ~150 +context switches per event and ~26% of the worker's CPU in that spin. With the +dedicated thread and the frame writer's synchronous flush, the steady-state event +path schedules no managed pool work. The overlapped pipe write still completes on +an I/O completion thread. A single event whose envelope exceeds the negotiated frame maximum is **undeliverable end to end** — the pipe maximum sits only the envelope-overhead diff --git a/docs/WorkerFrameProtocol.md b/docs/WorkerFrameProtocol.md index 132a2264..2a0ce531 100644 --- a/docs/WorkerFrameProtocol.md +++ b/docs/WorkerFrameProtocol.md @@ -135,7 +135,7 @@ Two failure shapes are distinguished during a drain pass: Flushes are coalesced across a *run of same-class frames* inside a drain pass: each frame in the run is written to the stream without an individual -flush, then one `FlushAsync` runs — at the end of the pass, and additionally +flush, then one flush runs — at the end of the pass, and additionally at every control-to-event boundary — and only then does every successfully-written frame of that run resolve its completion. A caller's `WriteAsync` therefore still does not complete until its bytes are both @@ -166,11 +166,18 @@ end-of-pass flush, and additionally fails the event frame the drain had already claimed off its queue (nothing else would ever complete it) along with every frame still queued. +The flush is a synchronous `Stream.Flush()`, not `FlushAsync`. `PipeStream` does not +override `FlushAsync`, so on .NET Framework the base implementation runs `Flush()` +through `Task.Factory.StartNew`. That is one thread-pool work item per drain pass for +a `PipeStream.Flush` that does no I/O, because it never calls `FlushFileBuffers`. +Calling `Flush()` inline keeps the same written-and-flushed contract for any stream. + Note the ordering all of this implies at the peer: the frames reach the pipe before the flush that follows them, so the gateway can read a whole batch while the writer has not yet flushed it. Anything observing the flush itself (a test counting flushes, for instance) must wait for the flush, not infer it -from frames arriving. The event drain loop (`WorkerPipeSession.RunEventDrainLoopAsync`) +from frames arriving. The event drain loop (`WorkerPipeSession.RunEventDrainLoopAsync`, which +runs on a dedicated thread; see `MxAccessWorkerInstanceDesign.md`) submits a whole drained event batch through `WriteBatchAsync`, which enqueues every frame under one `_gate` acquisition, takes the write lock once, and drains them together, so a burst of N events costs one flush rather than N — diff --git a/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj b/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj index c4e32f72..d67f35a6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj +++ b/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj @@ -38,6 +38,17 @@ + + + + + diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs index 0d60d9d9..dbb218ce 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs @@ -375,7 +375,7 @@ public sealed class WorkerFrameProtocolTests /// /// Verifies the writer coalesces the flush across a batch of frames drained together: four frames - /// queued behind an in-progress write drain in a single pass and share one FlushAsync, not four. + /// queued behind an in-progress write drain in a single pass and share one Flush, not four. /// Every frame still reaches the wire intact. /// /// The burst is all-event on purpose. The control-frame completion decoupling made the drain flush @@ -1386,20 +1386,20 @@ public sealed class WorkerFrameProtocolTests }; } - // A MemoryStream that counts FlushAsync calls without gating any write, so a batch write can be + // A MemoryStream that counts Flush calls without gating any write, so a batch write can be // asserted to flush exactly once. private sealed class FlushCountingStream : MemoryStream { private int _flushCount; - /// Gets the number of calls observed so far. + /// Gets the number of calls observed so far. public int FlushCount => Volatile.Read(ref _flushCount); /// - public override Task FlushAsync(CancellationToken cancellationToken) + public override void Flush() { Interlocked.Increment(ref _flushCount); - return base.FlushAsync(cancellationToken); + base.Flush(); } } @@ -1500,7 +1500,7 @@ public sealed class WorkerFrameProtocolTests /// Gets a task that completes once the second gated call has started blocking. public Task SecondGateWriteStarted => _secondGateWriteStarted.Task; - /// Gets the number of calls observed so far. + /// Gets the number of calls observed so far. public int FlushCount => Volatile.Read(ref _flushCount); /// Releases the first blocked write so it can complete. @@ -1528,10 +1528,10 @@ public sealed class WorkerFrameProtocolTests } /// - public override Task FlushAsync(CancellationToken cancellationToken) + public override void Flush() { Interlocked.Increment(ref _flushCount); - return base.FlushAsync(cancellationToken); + base.Flush(); } /// @@ -1547,7 +1547,7 @@ public sealed class WorkerFrameProtocolTests } } - // A MemoryStream whose first write blocks until released and whose every FlushAsync throws, so a test + // A MemoryStream whose first write blocks until released and whose every Flush throws, so a test // can fault the class-boundary flush with control frames already written and an event frame already // claimed off its queue. private sealed class FlushFaultingGatedStream : MemoryStream @@ -1584,9 +1584,9 @@ public sealed class WorkerFrameProtocolTests } /// - public override Task FlushAsync(CancellationToken cancellationToken) + public override void Flush() { - return Task.FromException(new IOException(_faultMessage)); + throw new IOException(_faultMessage); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index 780d313f..404c01fc 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -367,6 +367,45 @@ public sealed class WorkerPipeSessionTests await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } + /// + /// The event drain loop runs on its own dedicated background thread, not the CLR thread + /// pool: on .NET Framework every pool continuation per event also paid the pool's idle + /// spin, which a production trace measured at ~150 context switches per event. The parked + /// wait is observed from the fake, which records the calling thread. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_EventDrainLoop_RunsOnDedicatedNonPoolThread() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new() + { + WaitForEventsOnSignalOnly = true, + }; + WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + + while (runtime.LastWaitForEventsTimeout is null) + { + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token); + } + + Assert.Equal("mxgw-event-drain", runtime.LastWaitForEventsThreadName); + Assert.False(runtime.LastWaitForEventsOnThreadPool); + + // The dedicated thread still ships events and still stops cleanly on shutdown. + runtime.EnqueueEvent(CreateWorkerEvent(sequence: 3)); + WorkerEnvelope workerEvent = await ReadUntilAsync( + pipePair.GatewayReader, + WorkerEnvelope.BodyOneofCase.WorkerEvent, + cancellation.Token); + Assert.Equal(3UL, workerEvent.WorkerEvent.Event.WorkerSequence); + + await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); + } + /// /// Verifies that a Ping control command is answered on the worker side /// (not dispatched to the STA) with an OK reply that echoes the ping diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxStatusProxyRecordInfoPinTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxStatusProxyRecordInfoPinTests.cs new file mode 100644 index 00000000..f2a4a7bc --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxStatusProxyRecordInfoPinTests.cs @@ -0,0 +1,28 @@ +using ZB.MOM.WW.MxGateway.Worker.MxAccess; + +namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess; + +/// +/// The record-info pin is a best-effort, process-wide cache warm-up. Whether it succeeds depends +/// on the host's MXAccess registration, so these tests assert only what must hold on every host: +/// it never throws, it always reports an outcome, and only the first call does work. +/// +public sealed class MxStatusProxyRecordInfoPinTests +{ + /// Repeated calls return the same outcome and leave a status describing it. + [Fact] + public void EnsurePinned_CalledTwice_IsIdempotentAndReportsStatus() + { + bool first = MxStatusProxyRecordInfoPin.EnsurePinned(); + string? statusAfterFirst = MxStatusProxyRecordInfoPin.Status; + bool second = MxStatusProxyRecordInfoPin.EnsurePinned(); + + Assert.Equal(first, second); + Assert.False(string.IsNullOrWhiteSpace(statusAfterFirst)); + Assert.Same(statusAfterFirst, MxStatusProxyRecordInfoPin.Status); + if (first) + { + Assert.StartsWith("pinned ", statusAfterFirst); + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index 2d4e3bb5..83f93cf8 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -387,6 +387,22 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession return eventSignal.WaitAsync(timeout, cancellationToken); } + /// Gets the name of the thread that made the most recent call. + public string? LastWaitForEventsThreadName { get; private set; } + + /// Gets whether the most recent call ran on a thread-pool thread. + public bool? LastWaitForEventsOnThreadPool { get; private set; } + + /// + public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken) + { + LastWaitForEventsThreadName = Thread.CurrentThread.Name; + LastWaitForEventsOnThreadPool = Thread.CurrentThread.IsThreadPoolThread; + // Same knobs as the async form (signal-only, recorded timeout, backing queue); the drain + // loop now runs on a dedicated thread, so blocking on the task here is the production shape. + WaitForEventsAsync(timeout, cancellationToken).GetAwaiter().GetResult(); + } + /// /// Gets or sets a fault the next call hands back once, standing in /// for the fault a real event queue self-records on overflow. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs index 916ab4a3..64c70bb7 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs @@ -588,7 +588,7 @@ public sealed class WorkerFrameWriter // and now flushed, so completing them early is the same contract, earlier. try { - await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false); + FlushStream(); } catch (Exception exception) { @@ -640,7 +640,7 @@ public sealed class WorkerFrameWriter try { - await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false); + FlushStream(); } catch (Exception exception) { @@ -655,6 +655,15 @@ public sealed class WorkerFrameWriter CompleteFrames(written); } + // Synchronous on purpose. PipeStream does not override FlushAsync, so the base Stream.FlushAsync + // runs Flush() through Task.Factory.StartNew: one thread-pool work item per drain pass, for a + // PipeStream.Flush that has nothing to do (it never calls FlushFileBuffers). Calling Flush() + // inline keeps the contract for any stream and drops the pool hop. + private void FlushStream() + { + _stream.Flush(); + } + private static void CompleteFrames(List frames) { foreach (PendingFrame frame in frames) diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index 200fb108..112176e3 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -355,6 +355,13 @@ public sealed class WorkerPipeSession await WriteWorkerHelloAsync(cancellationToken).ConfigureAwait(false); _state = WorkerState.InitializingSta; WorkerReady ready = await initializeMxAccessAsync(cancellationToken).ConfigureAwait(false); + if (MxStatusProxyRecordInfoPin.Status is { } pinStatus) + { + _logger?.Information( + "WorkerRecordInfoPin", + new Dictionary { ["status"] = pinStatus }); + } + await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false); _state = WorkerState.Ready; } @@ -589,14 +596,59 @@ public sealed class WorkerPipeSession TaskScheduler.Default); } - private async Task RunEventDrainLoopAsync(CancellationToken cancellationToken) + /// + /// Starts the event drain loop on a dedicated background thread and returns a task that + /// completes, faults, or cancels exactly as the loop does, so the message loop's + /// Task.WhenAny supervision is unchanged. + /// + /// + /// The loop used to be an async method woken by SemaphoreSlim.WaitAsync. On .NET + /// Framework every release of that semaphore queues the continuation to the CLR thread pool, + /// and each pool worker that runs one then spins in SleepEx before parking: a WPR trace + /// on wonder-app-vd03 measured ~150 context switches per event and ~26% of the worker's CPU + /// there. A dedicated thread parks on the signal instead, and with a synchronous flush the + /// steady-state event path schedules no managed thread-pool work at all. The pipe write itself + /// stays overlapped; its completion runs on an I/O completion thread, not a worker thread. + /// + private Task RunEventDrainLoopAsync(CancellationToken cancellationToken) + { + TaskCompletionSource completion = new(TaskCreationOptions.RunContinuationsAsynchronously); + Thread thread = new(() => + { + try + { + RunEventDrainLoop(cancellationToken); + completion.TrySetResult(true); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + completion.TrySetCanceled(cancellationToken); + } + catch (Exception exception) + { + completion.TrySetException(exception); + } + }) + { + IsBackground = true, + Name = "mxgw-event-drain", + }; + thread.Start(); + return completion.Task; + } + + private void RunEventDrainLoop(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { IWorkerRuntimeSession? runtimeSession = _runtimeSession; if (runtimeSession is null) { - await Task.Delay(EventDrainInterval, cancellationToken).ConfigureAwait(false); + if (cancellationToken.WaitHandle.WaitOne(EventDrainInterval)) + { + cancellationToken.ThrowIfCancellationRequested(); + } + continue; } @@ -604,7 +656,7 @@ public sealed class WorkerPipeSession if (fault is not null) { _state = WorkerState.Faulted; - await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false); + TryWriteFaultAsync(fault, cancellationToken).GetAwaiter().GetResult(); throw new WorkerEventQueueFaultException( fault.Category, string.IsNullOrWhiteSpace(fault.DiagnosticMessage) @@ -615,15 +667,13 @@ public sealed class WorkerPipeSession IReadOnlyList events = runtimeSession.DrainEvents(EventDrainBatchSize); if (events.Count == 0) { - // Wait on the queue's wake signal rather than sleeping a fixed tick: an event - // enqueued by the STA completes this immediately, so the first event of a burst is - // framed at signal latency instead of waiting out EventDrainInterval, and a session - // with no traffic stops waking at all. The wait's outcome is intentionally ignored — - // whether a signal or the fallback ended it, the next pass re-checks DrainFault() - // and re-drains, which is also why one coalesced wake for many enqueues is safe. - await runtimeSession - .WaitForEventsAsync(EventDrainInterval, cancellationToken) - .ConfigureAwait(false); + // Park on the queue's wake signal rather than sleeping a fixed tick: an event + // enqueued by the STA releases this immediately, so the first event of a burst is + // framed at signal latency, and a session with no traffic wakes only at the fallback + // ceiling. The wait's outcome is intentionally ignored — whether a signal or the + // fallback ended it, the next pass re-checks DrainFault() and re-drains, which is + // also why one coalesced wake for many enqueues is safe. + runtimeSession.WaitForEvents(EventDrainInterval, cancellationToken); continue; } @@ -640,9 +690,10 @@ public sealed class WorkerPipeSession try { - await _writer + _writer .WriteBatchAsync(envelopes, WorkerFrameWritePriority.Event, cancellationToken) - .ConfigureAwait(false); + .GetAwaiter() + .GetResult(); } catch (WorkerFrameProtocolException exception) when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge) @@ -651,11 +702,12 @@ public sealed class WorkerPipeSession // still the structured, event-naming fault. Map the rejection back to the first // event in batch order whose envelope overshoots the negotiated maximum — the same // frame the writer rejected first. - await FaultOnOversizedEventAsync( + FaultOnOversizedEventAsync( FindOversizedEvent(events, envelopes), exception, cancellationToken) - .ConfigureAwait(false); + .GetAwaiter() + .GetResult(); } } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs index e1050c88..2d3fe0a3 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs @@ -79,6 +79,16 @@ public interface IWorkerRuntimeSession : IDisposable /// A task that completes when the queue is signalled or the timeout elapses. Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken); + /// + /// Blocking form of for a drain loop that runs on its own + /// dedicated thread. Same contract (coalesced signal, timeout is a ceiling, at most one + /// waiter); it parks the calling thread instead of scheduling a thread-pool continuation per + /// wake, which on .NET Framework also triggers the pool's spin-wait on every event. + /// + /// Maximum time to wait before the wait completes unsignalled. + /// Token cancelling the wait. + void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken); + /// /// Drains a pending fault from the queue, if any. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessComObjectFactory.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessComObjectFactory.cs index 704d4cb0..8a48ea99 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessComObjectFactory.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessComObjectFactory.cs @@ -8,6 +8,9 @@ public sealed class MxAccessComObjectFactory : IMxAccessComObjectFactory /// public object Create() { + // Warm oleaut32's record-info cache before the first OnDataChange can fire; see + // MxStatusProxyRecordInfoPin for why the per-event lookup is otherwise expensive. + MxStatusProxyRecordInfoPin.EnsurePinned(); return new LMXProxyServerClass(); } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs index 528a5261..fe012008 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs @@ -320,6 +320,17 @@ public sealed class MxAccessEventQueue return eventSignal.WaitAsync(timeout, cancellationToken); } + /// + /// Blocking form of : parks the calling (dedicated drain) + /// thread on the same coalesced signal. Same single-waiter precondition. + /// + /// Maximum time to wait before the wait completes unsignalled. + /// Token cancelling the wait. + public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken) + { + _ = eventSignal.Wait(timeout, cancellationToken); + } + /// /// Attempts to dequeue the next event without removing it if empty. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index bc8997d4..6e9a098e 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -682,6 +682,12 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession return eventQueue.WaitForEventsAsync(timeout, cancellationToken); } + /// + public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken) + { + eventQueue.WaitForEvents(timeout, cancellationToken); + } + /// public WorkerFault? DrainFault() { diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxStatusProxyRecordInfoPin.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxStatusProxyRecordInfoPin.cs new file mode 100644 index 00000000..533f3b02 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxStatusProxyRecordInfoPin.cs @@ -0,0 +1,113 @@ +using System; +using System.Linq; +using System.Runtime.InteropServices; +using ArchestrA.MxAccess; + +namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; + +/// +/// Keeps the LMXProxy type library and the MXSTATUS_PROXY record info alive for the +/// lifetime of the worker process, so the MXAccess proxy's per-event +/// GetRecordInfoFromGuids call is served from oleaut32's cache. +/// +/// +/// Every OnDataChange carries a SAFEARRAY of MXSTATUS_PROXY records, and +/// lmxproxy.dll resolves that record type with GetRecordInfoFromGuids each time it +/// builds the array. With no live reference, oleaut32 re-resolves the type library through the +/// registry and reloads it on every call: about 1 ms per event on an idle dev box. Under WOW64 +/// and an EDR registry filter it is worse; a WPR trace on wonder-app-vd03 put it at about 70% of +/// the STA's CPU. While one IRecordInfo for the type is held, the same call returns the +/// cached instance in well under a microsecond. The pin is a process-wide cache warm-up only: it +/// changes no MXAccess behavior, and failure to pin is reported but never fatal. +/// +public static class MxStatusProxyRecordInfoPin +{ + private static readonly object Gate = new(); + private static IntPtr pinnedTypeLib; + private static IntPtr pinnedRecordInfo; + private static bool attempted; + + /// Gets the outcome of the pin attempt, or before the first attempt. + public static string? Status { get; private set; } + + /// + /// Pins the type library and record info once per process. Safe to call repeatedly and from + /// any thread; only the first call does work. + /// + /// when the record info is pinned. + public static bool EnsurePinned() + { + lock (Gate) + { + if (attempted) + { + return pinnedRecordInfo != IntPtr.Zero; + } + + attempted = true; + try + { + Guid libraryGuid = typeof(LMXProxyServerClass).Assembly + .GetCustomAttributes(typeof(GuidAttribute), inherit: false) + .Cast() + .Select(attribute => new Guid(attribute.Value)) + .Single(); + TypeLibVersionAttribute version = typeof(LMXProxyServerClass).Assembly + .GetCustomAttributes(typeof(TypeLibVersionAttribute), inherit: false) + .Cast() + .Single(); + Guid recordGuid = typeof(MXSTATUS_PROXY).GUID; + + int hr = LoadRegTypeLib( + ref libraryGuid, + (ushort)version.MajorVersion, + (ushort)version.MinorVersion, + 0, + out pinnedTypeLib); + if (hr != 0) + { + Status = $"LoadRegTypeLib failed (HRESULT 0x{hr:X8})"; + return false; + } + + hr = GetRecordInfoFromGuids( + ref libraryGuid, + (uint)version.MajorVersion, + (uint)version.MinorVersion, + 0, + ref recordGuid, + out pinnedRecordInfo); + if (hr != 0) + { + Status = $"GetRecordInfoFromGuids failed (HRESULT 0x{hr:X8})"; + return false; + } + + Status = $"pinned {libraryGuid:B} v{version.MajorVersion}.{version.MinorVersion} record {recordGuid:B}"; + return true; + } + catch (Exception exception) + { + Status = $"pin failed: {exception.GetType().Name}: {exception.Message}"; + return false; + } + } + } + + [DllImport("oleaut32.dll")] + private static extern int LoadRegTypeLib( + ref Guid libraryGuid, + ushort majorVersion, + ushort minorVersion, + uint lcid, + out IntPtr typeLib); + + [DllImport("oleaut32.dll")] + private static extern int GetRecordInfoFromGuids( + ref Guid libraryGuid, + uint majorVersion, + uint minorVersion, + uint lcid, + ref Guid typeGuid, + out IntPtr recordInfo); +}