fix(WRK-22,WRK-24,WRK-25,WRK-27,IPC-26): worker write-seam hardening
ci / java (push) Successful in 2m7s
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Failing after 1m13s
ci / portable (push) Failing after 4m6s

WRK-22/IPC-26: tombstone a WriteAsync/WriteBatchAsync cancelled while
waiting for the write lock (PendingFrame.Claimed under _gate; DequeueNext
skips cancelled, claims the frame it returns) so a cancelled write never
reaches the wire unless already claimed mid-write (documented residual).

WRK-25: add WriteBatchAsync; RunEventDrainLoopAsync submits the drained
event batch through it, so a burst of N events costs one flush not N.
IPC-30 oversized-event structured fault preserved via FindOversizedEvent.

WRK-24: reject a below-1024 negotiated frame maximum at the handshake
(MinNegotiableFrameBytes, matching GatewayOptionsValidator floor).

WRK-27: alarm poll advertises StaCallInProgress on the heartbeat snapshot
so the watchdog suppresses to the ceiling, not the grace.

Docs (WorkerFrameProtocol.md, MxAccessWorkerInstanceDesign.md) and the
2026-07-12 remediation registers/change-log updated in the same commit.
This commit is contained in:
Joseph Doherty
2026-08-07 07:50:38 -04:00
parent 10534ec906
commit 8df35cd63a
14 changed files with 912 additions and 58 deletions
@@ -481,6 +481,192 @@ public sealed class WorkerFrameProtocolTests
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
}
/// <summary>
/// Verifies a negotiated frame maximum below the worker floor is rejected as
/// <c>InvalidConfiguration</c> (WRK-24), and that exactly the floor is adopted. The floor matches
/// the gateway's own <c>GatewayOptionsValidator.MinimumMaxMessageBytes</c>, so the worker never
/// rejects a value the gateway's validator accepts, yet a nonsensical tiny value faults at the
/// handshake instead of leaving a session that fails every later frame.
/// </summary>
[Fact]
public void AdoptNegotiatedMaxMessageBytes_BelowFloor_ThrowsInvalidConfiguration()
{
WorkerFrameProtocolOptions belowFloor = CreateOptions();
WorkerFrameProtocolException exception = Assert.Throws<WorkerFrameProtocolException>(
() => belowFloor.AdoptNegotiatedMaxMessageBytes(512));
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
// Boundary: exactly the floor is accepted.
WorkerFrameProtocolOptions atFloor = CreateOptions();
atFloor.AdoptNegotiatedMaxMessageBytes((uint)WorkerFrameProtocolOptions.MinNegotiableFrameBytes);
Assert.Equal(WorkerFrameProtocolOptions.MinNegotiableFrameBytes, atFloor.MaxMessageBytes);
}
/// <summary>
/// WRK-22 / IPC-26. A <c>WriteAsync</c> cancelled while it waits for the write lock must never
/// have its frame written by the next lock-holder. Writer A holds the lock mid-write (blocked in
/// the stream); an event write is queued and then cancelled; when A is released and a later
/// control frame drains, the wire carries A's frame and the control frame only — the cancelled
/// event envelope is tombstoned and skipped.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteAsync_CancelledWhileWaitingForLock_FrameIsNeverWritten()
{
WorkerFrameProtocolOptions options = CreateOptions();
using GatedWriteStream stream = new();
WorkerFrameWriter writer = new(stream, options);
// Writer A occupies the writer and blocks inside the stream, holding the write lock.
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
// Queue an event write with its own CTS while the lock is held, then cancel it.
using CancellationTokenSource cts = new();
Task cancelledWrite = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
await Task.Delay(50);
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledWrite);
// Release A, then drive a fresh control write.
stream.ReleaseFirstWrite();
await AwaitWithTimeoutAsync(firstWrite);
await writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
WorkerEnvelope frame1 = await reader.ReadAsync();
WorkerEnvelope frame2 = await reader.ReadAsync();
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame2.BodyCase);
// The cancelled event never reached the wire — no third frame, and sequences stay contiguous.
Assert.Equal(stream.Length, stream.Position);
Assert.Equal(1UL, frame1.Sequence);
Assert.Equal(2UL, frame2.Sequence);
}
/// <summary>
/// WRK-22 / IPC-26, the review's shutdown scenario. A cancelled event frame queued before a
/// shutdown-ack control frame must not trail the ack on the wire: the tombstone rule plus the
/// control-before-event scheduler keeps the ack the last frame written.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteAsync_CancelledEventFrame_DoesNotTrailShutdownAck()
{
WorkerFrameProtocolOptions options = CreateOptions();
using GatedWriteStream stream = new();
WorkerFrameWriter writer = new(stream, options);
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
using CancellationTokenSource cts = new();
Task cancelledEvent = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
await Task.Delay(50);
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledEvent);
// The shutdown ack (a control frame) is queued behind the still-blocked first write.
Task ackWrite = writer.WriteAsync(CreateShutdownAckEnvelope(), WorkerFrameWritePriority.Control);
await Task.Delay(50);
stream.ReleaseFirstWrite();
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, ackWrite));
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
WorkerEnvelope frame1 = await reader.ReadAsync();
WorkerEnvelope frame2 = await reader.ReadAsync();
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
// The ack is the last frame — the cancelled event did not trail it.
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, frame2.BodyCase);
Assert.Equal(stream.Length, stream.Position);
}
/// <summary>
/// WRK-25. The batch entry point enqueues a whole event burst under one lock acquisition and
/// drains it together, so N events cost exactly one flush and reach the wire in batch order with
/// monotonic sequences — the coalescing WRK-12 shipped, now on the event hot path.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteBatchAsync_FlushesOnceAndPreservesOrder()
{
const int count = 8;
WorkerFrameProtocolOptions options = CreateOptions();
using FlushCountingStream stream = new();
WorkerFrameWriter writer = new(stream, options);
WorkerEnvelope[] batch = new WorkerEnvelope[count];
for (int index = 0; index < count; index++)
{
batch[index] = CreateEventEnvelope(workerSequence: (ulong)(100 + index));
}
await writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
// The whole batch was queued before the single lock wait, so it drained in one pass => one flush.
Assert.Equal(1, stream.FlushCount);
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
for (int index = 0; index < count; index++)
{
WorkerEnvelope frame = await reader.ReadAsync();
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame.BodyCase);
// Wire order matches batch order.
Assert.Equal((ulong)(100 + index), frame.WorkerEvent.Event.WorkerSequence);
// Write-time stamped sequence is monotonic 1..count.
Assert.Equal((ulong)(index + 1), frame.Sequence);
}
}
/// <summary>
/// WRK-25. Control-before-event still holds mid-batch: a control frame queued while a batch is
/// draining jumps ahead of the batch's remaining events.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteBatchAsync_ControlFrameQueuedDuringBatch_JumpsRemainingEvents()
{
WorkerFrameProtocolOptions options = CreateOptions();
using GatedWriteStream stream = new();
WorkerFrameWriter writer = new(stream, options);
WorkerEnvelope[] batch = new[]
{
CreateEventEnvelope(),
CreateEventEnvelope(),
CreateEventEnvelope(),
};
// The batch takes the lock and blocks writing its first event frame inside the stream.
Task batchWrite = writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
// A control frame queued mid-drain must jump the batch's remaining events.
Task controlWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
await Task.Delay(50);
stream.ReleaseFirstWrite();
await AwaitWithTimeoutAsync(Task.WhenAll(batchWrite, controlWrite));
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
WorkerEnvelope f1 = await reader.ReadAsync();
WorkerEnvelope f2 = await reader.ReadAsync();
WorkerEnvelope f3 = await reader.ReadAsync();
WorkerEnvelope f4 = await reader.ReadAsync();
// First event was already writing when the control frame queued; the control frame then jumps
// ahead of the two remaining events.
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f1.BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, f2.BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f3.BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f4.BodyCase);
}
private static WorkerEnvelope CreateGatewayHelloEnvelope(ulong sequence = 1)
{
return new WorkerEnvelope
@@ -522,6 +708,47 @@ public sealed class WorkerFrameProtocolTests
};
}
private static WorkerEnvelope CreateEventEnvelope(ulong workerSequence)
{
WorkerEnvelope envelope = CreateEventEnvelope();
envelope.WorkerEvent.Event.WorkerSequence = workerSequence;
return envelope;
}
private static WorkerEnvelope CreateShutdownAckEnvelope()
{
return new WorkerEnvelope
{
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
SessionId = SessionId,
WorkerShutdownAck = new WorkerShutdownAck
{
Status = new ProtocolStatus
{
Code = ProtocolStatusCode.Ok,
Message = "OK",
},
},
};
}
// A MemoryStream that counts FlushAsync calls without gating any write, so a batch write can be
// asserted to flush exactly once.
private sealed class FlushCountingStream : MemoryStream
{
private int _flushCount;
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
public int FlushCount => Volatile.Read(ref _flushCount);
/// <inheritdoc />
public override Task FlushAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _flushCount);
return base.FlushAsync(cancellationToken);
}
}
// A MemoryStream whose first WriteAsync blocks until released, so a test can queue additional frames
// behind an in-progress write and observe the writer's priority ordering.
private sealed class GatedWriteStream : MemoryStream