feat(worker): adopt negotiated frame max, bound drain, priority write scheduler (IPC-02/04 + WRK-04/07 worker half)

Worker half of the Wave 3 size/backpressure + write-ordering pass:

- IPC-02: the worker adopts GatewayHello.max_frame_bytes during the handshake
  (WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes) instead of a
  hard-coded default; 0 keeps the default, a value above a 256 MiB ceiling is
  rejected. Reader and writer share the options instance, applied before the
  message loop.
- IPC-04: DrainEvents caps each reply at MaxDrainEventsPerReply (10_000) and
  treats max_events = 0 as that cap rather than 'drain the entire queue', so one
  diagnostic drain cannot pack a session-killing reply frame.
- WRK-04: WorkerFrameWriter stamps the envelope Sequence at the actual point of
  writing (under the write lock) instead of at envelope creation, so the on-wire
  order and the stamped sequence always agree under concurrent producers.
- WRK-07: the writer is now a cooperative priority scheduler — callers enqueue at
  Control or Event priority and the draining lock-holder writes all control
  frames before any event frame, so replies/faults/heartbeats jump ahead of an
  event backlog. Per-frame validation/size rejections fail only that frame; a
  stream write failure fails all queued frames.

Tests: monotonic gap-free sequence under concurrency, control-before-event
priority (gated stream), negotiated-max adoption, DrainEvents zero-bound.
Worker builds x86 only — verified on windev.
This commit is contained in:
Joseph Doherty
2026-07-09 09:09:14 -04:00
parent c8b3a2281a
commit ebe6aeac98
7 changed files with 450 additions and 30 deletions
@@ -18,6 +18,13 @@ public sealed class WorkerPipeSession
private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1);
private const uint EventDrainBatchSize = 128;
// Hard cap on how many events a single DrainEvents diagnostic reply may carry. DrainEvents is a
// non-streaming control command, so an unbounded drain (including the max_events = 0 "as many as
// available" request) could pack the whole queue into one session-killing reply frame (IPC-04).
// The gateway request validator rejects requests above its public ceiling; this worker-side cap is
// the backstop and defines the effective per-reply maximum. Kept in step with that public ceiling.
private const uint MaxDrainEventsPerReply = 10_000;
private readonly WorkerFrameProtocolOptions _options;
private readonly Func<int> _processIdProvider;
private readonly Func<IWorkerRuntimeSession> _runtimeSessionFactory;
@@ -28,7 +35,6 @@ public sealed class WorkerPipeSession
private readonly object _commandTaskGate = new();
private readonly HashSet<Task> _activeCommandTasks = new();
private IWorkerRuntimeSession? _runtimeSession;
private long _nextSequence;
// Mutated from the message loop, command tasks, the heartbeat loop and the
// shutdown path; volatile so cross-thread reads observe the latest state
@@ -225,6 +231,12 @@ public sealed class WorkerPipeSession
WorkerFrameProtocolErrorCode.NonceMismatch,
"GatewayHello nonce does not match the worker launch nonce.");
}
// Adopt the gateway-negotiated frame maximum so both ends frame to the same limit instead of
// matched compile-time defaults (IPC-02). Applied here, before the message loop, so every
// post-handshake frame is validated against the negotiated value; the reader and writer share
// this options instance. The hello frame itself was small and already read under the default.
_options.AdoptNegotiatedMaxMessageBytes(gatewayHello.MaxFrameBytes);
}
private Task WriteWorkerHelloAsync(CancellationToken cancellationToken)
@@ -361,8 +373,11 @@ public sealed class WorkerPipeSession
foreach (WorkerEvent workerEvent in events)
{
// Events are the low-priority frame class: the writer holds them behind any pending
// control frame (reply, fault, heartbeat, shutdown ack) so those are not delayed
// behind an event backlog (WRK-07).
await _writer
.WriteAsync(CreateEnvelope(workerEvent), cancellationToken)
.WriteAsync(CreateEnvelope(workerEvent), WorkerFrameWritePriority.Event, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -527,7 +542,12 @@ public sealed class WorkerPipeSession
IWorkerRuntimeSession? runtimeSession = _runtimeSession;
if (runtimeSession is not null)
{
uint maxEvents = command.DrainEvents?.MaxEvents ?? 0;
// Bound the diagnostic drain so max_events = 0 ("as many as available") or an over-large
// request cannot pack the whole queue into one session-killing reply frame (IPC-04).
uint requested = command.DrainEvents?.MaxEvents ?? 0;
uint maxEvents = requested == 0 || requested > MaxDrainEventsPerReply
? MaxDrainEventsPerReply
: requested;
foreach (WorkerEvent workerEvent in runtimeSession.DrainEvents(maxEvents))
{
if (workerEvent.Event is not null)
@@ -1004,19 +1024,16 @@ public sealed class WorkerPipeSession
private WorkerEnvelope CreateBaseEnvelope()
{
// Sequence is deliberately left unset here: the frame writer stamps it at the actual point of
// writing, under its single drain task, so the on-wire order and the stamped sequence agree
// even under concurrent producers and priority reordering (WRK-04).
return new WorkerEnvelope
{
ProtocolVersion = _options.ProtocolVersion,
SessionId = _options.SessionId,
Sequence = NextSequence(),
};
}
private ulong NextSequence()
{
return unchecked((ulong)Interlocked.Increment(ref _nextSequence));
}
private async Task<WorkerReady> InitializeMxAccessAsync(CancellationToken cancellationToken)
{
// RunAsync constructs the runtime session via _runtimeSessionFactory()