fix(WRK-22,WRK-24,WRK-25,WRK-27,IPC-26): worker write-seam hardening
WRK-22/IPC-26: tombstone a WriteAsync/WriteBatchAsync cancelled while waiting for the write lock (PendingFrame.Claimed under _gate; DequeueNext skips cancelled, claims the frame it returns) so a cancelled write never reaches the wire unless already claimed mid-write (documented residual). WRK-25: add WriteBatchAsync; RunEventDrainLoopAsync submits the drained event batch through it, so a burst of N events costs one flush not N. IPC-30 oversized-event structured fault preserved via FindOversizedEvent. WRK-24: reject a below-1024 negotiated frame maximum at the handshake (MinNegotiableFrameBytes, matching GatewayOptionsValidator floor). WRK-27: alarm poll advertises StaCallInProgress on the heartbeat snapshot so the watchdog suppresses to the ceiling, not the grace. Docs (WorkerFrameProtocol.md, MxAccessWorkerInstanceDesign.md) and the 2026-07-12 remediation registers/change-log updated in the same commit.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Google.Protobuf;
|
||||
@@ -33,6 +34,16 @@ public sealed class WorkerFrameWriter
|
||||
|
||||
/// <summary>Gets the completion source signaled once the frame has been written or has failed.</summary>
|
||||
public TaskCompletionSource<bool> Completion { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Set to <c>true</c> by <see cref="DequeueNext"/> — under <c>_gate</c> — at the instant the
|
||||
/// draining lock-holder takes ownership of this frame to write it. A cancelled caller
|
||||
/// tombstones its frame only while it is still unclaimed, so a claim and a cancel can never
|
||||
/// both win: the flag is the interlock between the two. A frame already claimed is mid-write
|
||||
/// and can no longer be recalled (see <see cref="WriteAsync(WorkerEnvelope, WorkerFrameWritePriority, CancellationToken)"/>).
|
||||
/// Mutated only under <c>_gate</c>.
|
||||
/// </summary>
|
||||
public bool Claimed;
|
||||
}
|
||||
|
||||
private readonly WorkerFrameProtocolOptions _options;
|
||||
@@ -75,6 +86,13 @@ public sealed class WorkerFrameWriter
|
||||
/// <param name="priority">Scheduling priority; control frames are written ahead of event frames.</param>
|
||||
/// <param name="cancellationToken">Token to cancel waiting for the write lock.</param>
|
||||
/// <returns>A task that completes when the frame has been written and flushed.</returns>
|
||||
/// <remarks>
|
||||
/// Cancellation contract (WRK-22): if the token fires while this call is waiting for the write
|
||||
/// lock, the frame is tombstoned so it is never written — unless a draining lock-holder has
|
||||
/// already claimed it, in which case the frame may still reach the wire even though this call
|
||||
/// observes <see cref="OperationCanceledException"/>. That residual window is by design: blocking
|
||||
/// the canceller behind the very write it is abandoning would defeat the point of cancellation.
|
||||
/// </remarks>
|
||||
public async Task WriteAsync(
|
||||
WorkerEnvelope envelope,
|
||||
WorkerFrameWritePriority priority,
|
||||
@@ -101,7 +119,19 @@ public sealed class WorkerFrameWriter
|
||||
// Contend for the single writer: whoever wins drains every currently-queued frame in priority
|
||||
// order, so this frame is written by this call or by a concurrent caller that got the lock
|
||||
// first. Either way it completes via its own TaskCompletionSource.
|
||||
await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Tombstone the queued frame so DequeueNext skips it — but only if a draining lock-holder
|
||||
// has not already claimed it. If it is claimed it is mid-write and cannot be recalled; the
|
||||
// caller still observes cancellation while the frame reaches the wire (documented above).
|
||||
TombstoneIfUnclaimed(frame, cancellationToken);
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await DrainQueuedFramesAsync().ConfigureAwait(false);
|
||||
@@ -114,6 +144,123 @@ public sealed class WorkerFrameWriter
|
||||
await frame.Completion.Task.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a whole batch of envelopes at one priority under a single lock acquisition and drains
|
||||
/// it, so a burst of frames — the event drain loop's hot path — pays one flush for the batch
|
||||
/// rather than one per frame (WRK-25, realizing the WRK-12 coalescing on the path it was built
|
||||
/// for). Intra-batch order is preserved because the enqueue is atomic under <c>_gate</c> and each
|
||||
/// class queue is FIFO; the control-before-event guarantee still holds because any concurrently
|
||||
/// queued control frame is drained ahead of this batch by <see cref="DequeueNext"/>. Every frame's
|
||||
/// "written and flushed before completion" contract is unchanged.
|
||||
/// </summary>
|
||||
/// <param name="envelopes">Envelopes to write, in order.</param>
|
||||
/// <param name="priority">Scheduling priority for the whole batch.</param>
|
||||
/// <param name="cancellationToken">Token to cancel waiting for the write lock.</param>
|
||||
/// <returns>A task that completes when every frame in the batch has been written and flushed.</returns>
|
||||
/// <remarks>
|
||||
/// A per-frame rejection inside the batch (for example one oversized event) surfaces from the
|
||||
/// awaited completions as its <see cref="WorkerFrameProtocolException"/>; the remaining frames are
|
||||
/// still observed so none faults unobserved. Cancellation while waiting for the lock tombstones
|
||||
/// every still-unclaimed frame in the batch, per the WRK-22 contract on
|
||||
/// <see cref="WriteAsync(WorkerEnvelope, WorkerFrameWritePriority, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public async Task WriteBatchAsync(
|
||||
IReadOnlyList<WorkerEnvelope> envelopes,
|
||||
WorkerFrameWritePriority priority,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (envelopes is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(envelopes));
|
||||
}
|
||||
|
||||
if (envelopes.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PendingFrame[] frames = new PendingFrame[envelopes.Count];
|
||||
for (int index = 0; index < envelopes.Count; index++)
|
||||
{
|
||||
WorkerEnvelope envelope = envelopes[index]
|
||||
?? throw new ArgumentException("Batch envelopes must not contain null.", nameof(envelopes));
|
||||
frames[index] = new PendingFrame(envelope);
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
Queue<PendingFrame> queue = priority == WorkerFrameWritePriority.Event ? _eventFrames : _controlFrames;
|
||||
foreach (PendingFrame frame in frames)
|
||||
{
|
||||
queue.Enqueue(frame);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
TombstoneUnclaimed(frames, cancellationToken);
|
||||
throw;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await DrainQueuedFramesAsync().ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeLock.Release();
|
||||
}
|
||||
|
||||
// Await every completion so no per-frame rejection faults unobserved, but surface the first
|
||||
// failure (in batch order) to the caller — the drain loop maps it back to the offending event.
|
||||
Exception? firstFailure = null;
|
||||
foreach (PendingFrame frame in frames)
|
||||
{
|
||||
try
|
||||
{
|
||||
await frame.Completion.Task.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
firstFailure ??= exception;
|
||||
}
|
||||
}
|
||||
|
||||
if (firstFailure is not null)
|
||||
{
|
||||
ExceptionDispatchInfo.Capture(firstFailure).Throw();
|
||||
}
|
||||
}
|
||||
|
||||
private void TombstoneIfUnclaimed(PendingFrame frame, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!frame.Claimed)
|
||||
{
|
||||
frame.Completion.TrySetCanceled(cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void TombstoneUnclaimed(PendingFrame[] frames, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (PendingFrame frame in frames)
|
||||
{
|
||||
if (!frame.Claimed)
|
||||
{
|
||||
frame.Completion.TrySetCanceled(cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Runs only under _writeLock. Drains control frames before event frames, stamping and writing each.
|
||||
// The stream write itself is not cancellable: a frame is written atomically or fails, never left
|
||||
// half-written on the pipe because a caller gave up waiting.
|
||||
@@ -198,18 +345,36 @@ public sealed class WorkerFrameWriter
|
||||
or WorkerFrameProtocolErrorCode.SessionMismatch;
|
||||
}
|
||||
|
||||
// Returns the next frame to write, control frames first, skipping any frame a cancelled caller
|
||||
// tombstoned while it waited for the lock (WRK-22). The frame actually returned is marked Claimed
|
||||
// under _gate in the same critical section that checks the tombstone, so a claim and a concurrent
|
||||
// cancel are mutually exclusive: whichever acquires _gate first wins.
|
||||
private PendingFrame? DequeueNext()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_controlFrames.Count > 0)
|
||||
while (_controlFrames.Count > 0)
|
||||
{
|
||||
return _controlFrames.Dequeue();
|
||||
PendingFrame frame = _controlFrames.Dequeue();
|
||||
if (frame.Completion.Task.IsCanceled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
frame.Claimed = true;
|
||||
return frame;
|
||||
}
|
||||
|
||||
if (_eventFrames.Count > 0)
|
||||
while (_eventFrames.Count > 0)
|
||||
{
|
||||
return _eventFrames.Dequeue();
|
||||
PendingFrame frame = _eventFrames.Dequeue();
|
||||
if (frame.Completion.Task.IsCanceled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
frame.Claimed = true;
|
||||
return frame;
|
||||
}
|
||||
|
||||
return null;
|
||||
|
||||
Reference in New Issue
Block a user