perf(worker): signal-driven event drain — removes the 25 ms latency floor and idle wakeups

This commit is contained in:
Joseph Doherty
2026-08-15 16:59:13 -04:00
parent dc9424d3bd
commit f4a6cb1db2
7 changed files with 383 additions and 24 deletions
@@ -15,6 +15,11 @@ namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
public sealed class WorkerPipeSession
{
// Fallback ceiling for the event drain loop's wait — not a poll period. MxAccessEventQueue
// signals on enqueue (and on a recorded fault), so the loop wakes as soon as there is something
// to ship instead of paying up to this interval of latency on every burst from idle, and an
// idle worker parks instead of waking 40x/s. The interval survives only as the bound on how
// long the loop may sleep unsignalled, which keeps its DrainFault() poll on a known cadence.
private static readonly TimeSpan EventDrainInterval = TimeSpan.FromMilliseconds(25);
private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1);
private const uint EventDrainBatchSize = 128;
@@ -367,7 +372,15 @@ public sealed class WorkerPipeSession
IReadOnlyList<WorkerEvent> events = runtimeSession.DrainEvents(EventDrainBatchSize);
if (events.Count == 0)
{
await Task.Delay(EventDrainInterval, cancellationToken).ConfigureAwait(false);
// Wait on the queue's wake signal rather than sleeping a fixed tick: an event
// enqueued by the STA completes this immediately, so the first event of a burst is
// framed at signal latency instead of waiting out EventDrainInterval, and a session
// with no traffic stops waking at all. The wait's outcome is intentionally ignored —
// whether a signal or the fallback ended it, the next pass re-checks DrainFault()
// and re-drains, which is also why one coalesced wake for many enqueues is safe.
await runtimeSession
.WaitForEventsAsync(EventDrainInterval, cancellationToken)
.ConfigureAwait(false);
continue;
}
@@ -57,6 +57,23 @@ public interface IWorkerRuntimeSession : IDisposable
/// <returns>The drained events and the truncation facts describing what stayed queued.</returns>
WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes);
/// <summary>
/// Waits until the outbound event queue has something for the caller to look at (an
/// enqueued event or a recorded fault), the fallback timeout elapses, or the token is
/// cancelled. Lets a drain loop be signal-driven instead of polling.
/// </summary>
/// <remarks>
/// Declared on the interface because the pipe session only ever sees an
/// <see cref="IWorkerRuntimeSession"/>, never the queue behind it; .NET Framework 4.8 has
/// no default interface members, so every implementation supplies it. The wait's outcome is
/// not surfaced: the caller re-drains and re-checks <see cref="DrainFault"/> after every
/// wait, because the signal is coalesced and the timeout is a ceiling, not a poll period.
/// </remarks>
/// <param name="timeout">Maximum time to wait before the wait completes unsignalled.</param>
/// <param name="cancellationToken">Token cancelling the wait.</param>
/// <returns>A task that completes when the queue is signalled or the timeout elapses.</returns>
Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken);
/// <summary>
/// Drains a pending fault from the queue, if any.
/// </summary>
@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
@@ -38,6 +40,17 @@ public sealed class MxAccessEventQueue
private readonly int capacity;
private readonly Queue<WorkerEvent> events;
private readonly object syncRoot = new();
// Wake signal for the worker's event drain loop (see WorkerPipeSession.RunEventDrainLoopAsync).
// Capped at one permit, so it is a level ("there is something to look at"), never a count: a
// burst of enqueues coalesces into one wake and the waiter re-drains everything that arrived.
// Never released while holding syncRoot — the STA thread enqueues here, so its path must stay a
// lock acquire plus a non-blocking Release. Deliberately not disposed: the queue outlives its
// waiter, no wait handle is ever materialized (nothing touches AvailableWaitHandle), and making
// the queue IDisposable would only add a lifecycle race between the STA's Enqueue and the
// drain loop's wait. WorkerFrameWriter's _writeLock is held the same way.
private readonly SemaphoreSlim eventSignal = new SemaphoreSlim(initialCount: 0, maxCount: 1);
private ulong lastEventSequence;
private WorkerFault? fault;
private bool faultDrained;
@@ -142,33 +155,67 @@ public sealed class MxAccessEventQueue
throw new ArgumentNullException(nameof(mxEvent));
}
lock (syncRoot)
try
{
if (fault is not null)
lock (syncRoot)
{
throw new InvalidOperationException("MXAccess outbound event queue is faulted.");
if (fault is not null)
{
throw new InvalidOperationException("MXAccess outbound event queue is faulted.");
}
if (events.Count >= capacity)
{
fault = CreateOverflowFault();
throw new MxAccessEventQueueOverflowException(capacity);
}
// WRK-11: stamp the sequence/timestamp on the caller's own event and
// enqueue that same instance under the lock instead of cloning. See
// the ownership invariant on the class summary — the caller hands the
// event over exclusively, so a defensive Clone() here is pure
// overhead on the hottest path.
mxEvent.WorkerSequence = ++lastEventSequence;
mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
WorkerEvent workerEvent = new()
{
Event = mxEvent,
};
events.Enqueue(workerEvent);
}
if (events.Count >= capacity)
{
fault = CreateOverflowFault();
throw new MxAccessEventQueueOverflowException(capacity);
}
// WRK-11: stamp the sequence/timestamp on the caller's own event and
// enqueue that same instance under the lock instead of cloning. See
// the ownership invariant on the class summary — the caller hands the
// event over exclusively, so a defensive Clone() here is pure
// overhead on the hottest path.
mxEvent.WorkerSequence = ++lastEventSequence;
mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
WorkerEvent workerEvent = new()
{
Event = mxEvent,
};
events.Enqueue(workerEvent);
}
finally
{
// Wake the drain loop outside the lock, on every exit path: the success path has an
// event to ship, and the overflow path has just self-recorded the fault the loop
// reports through DrainFault(). Signalling in a finally also keeps the already-faulted
// rejection harmless — a spurious wake only costs the waiter one empty re-drain.
SignalWake();
}
}
/// <summary>
/// Waits for a wake signal, the fallback timeout, or cancellation, so the worker's event
/// drain loop is signal-driven instead of polling. <see cref="Enqueue"/> and
/// <see cref="RecordFault"/> both signal, so an event arriving at an idle queue is picked
/// up immediately rather than at the caller's next tick, and an idle queue costs no wakeups
/// at all.
/// </summary>
/// <remarks>
/// The wait's outcome is deliberately not surfaced: the caller must re-drain (and re-check
/// <see cref="DrainFault"/>) after every wait regardless of whether a signal or the timeout
/// ended it, because the signal is coalesced — one pending wake can stand for any number of
/// enqueues. The timeout is therefore a ceiling on how long the caller may sleep, not a
/// poll period: it bounds how late a state change reaches the caller if some future path
/// mutates the queue without signalling.
/// </remarks>
/// <param name="timeout">Maximum time to wait before the wait completes unsignalled.</param>
/// <param name="cancellationToken">Token cancelling the wait.</param>
/// <returns>A task that completes when the queue is signalled or the timeout elapses.</returns>
public Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
return eventSignal.WaitAsync(timeout, cancellationToken);
}
/// <summary>
@@ -291,6 +338,11 @@ public sealed class MxAccessEventQueue
{
fault ??= workerFault.Clone();
}
// The drain loop discovers faults by polling DrainFault() at the top of each pass, so a
// fault recorded while it is parked must wake it — otherwise the fault would be reported
// no sooner than the loop's fallback tick. Signalled outside the lock, as Enqueue does.
SignalWake();
}
/// <summary>
@@ -311,6 +363,28 @@ public sealed class MxAccessEventQueue
}
}
private void SignalWake()
{
// Fast path: a wake is already pending and no waiter has consumed it yet, so releasing
// again would only throw. This keeps a burst of enqueues on the STA at a single volatile
// read once the first event has signalled.
if (eventSignal.CurrentCount > 0)
{
return;
}
try
{
eventSignal.Release();
}
catch (SemaphoreFullException)
{
// A concurrent producer won the race between the check above and this Release. The
// pending wake it published is exactly the wake this call wanted, so there is nothing
// to do — the signal is a level, not a count.
}
}
private WorkerFault CreateOverflowFault()
{
string message = $"MXAccess outbound event queue reached capacity {capacity}.";
@@ -455,6 +455,12 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
return eventQueue.Drain(maxEvents, maxTotalBytes);
}
/// <inheritdoc />
public Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
return eventQueue.WaitForEventsAsync(timeout, cancellationToken);
}
/// <inheritdoc />
public WorkerFault? DrainFault()
{