fix(WRK-22,WRK-24,WRK-25,WRK-27,IPC-26): worker write-seam hardening
WRK-22/IPC-26: tombstone a WriteAsync/WriteBatchAsync cancelled while waiting for the write lock (PendingFrame.Claimed under _gate; DequeueNext skips cancelled, claims the frame it returns) so a cancelled write never reaches the wire unless already claimed mid-write (documented residual). WRK-25: add WriteBatchAsync; RunEventDrainLoopAsync submits the drained event batch through it, so a burst of N events costs one flush not N. IPC-30 oversized-event structured fault preserved via FindOversizedEvent. WRK-24: reject a below-1024 negotiated frame maximum at the handshake (MinNegotiableFrameBytes, matching GatewayOptionsValidator floor). WRK-27: alarm poll advertises StaCallInProgress on the heartbeat snapshot so the watchdog suppresses to the ceiling, not the grace. Docs (WorkerFrameProtocol.md, MxAccessWorkerInstanceDesign.md) and the 2026-07-12 remediation registers/change-log updated in the same commit.
This commit is contained in:
@@ -481,6 +481,192 @@ public sealed class WorkerFrameProtocolTests
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a negotiated frame maximum below the worker floor is rejected as
|
||||
/// <c>InvalidConfiguration</c> (WRK-24), and that exactly the floor is adopted. The floor matches
|
||||
/// the gateway's own <c>GatewayOptionsValidator.MinimumMaxMessageBytes</c>, so the worker never
|
||||
/// rejects a value the gateway's validator accepts, yet a nonsensical tiny value faults at the
|
||||
/// handshake instead of leaving a session that fails every later frame.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_BelowFloor_ThrowsInvalidConfiguration()
|
||||
{
|
||||
WorkerFrameProtocolOptions belowFloor = CreateOptions();
|
||||
WorkerFrameProtocolException exception = Assert.Throws<WorkerFrameProtocolException>(
|
||||
() => belowFloor.AdoptNegotiatedMaxMessageBytes(512));
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
|
||||
// Boundary: exactly the floor is accepted.
|
||||
WorkerFrameProtocolOptions atFloor = CreateOptions();
|
||||
atFloor.AdoptNegotiatedMaxMessageBytes((uint)WorkerFrameProtocolOptions.MinNegotiableFrameBytes);
|
||||
Assert.Equal(WorkerFrameProtocolOptions.MinNegotiableFrameBytes, atFloor.MaxMessageBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-22 / IPC-26. A <c>WriteAsync</c> cancelled while it waits for the write lock must never
|
||||
/// have its frame written by the next lock-holder. Writer A holds the lock mid-write (blocked in
|
||||
/// the stream); an event write is queued and then cancelled; when A is released and a later
|
||||
/// control frame drains, the wire carries A's frame and the control frame only — the cancelled
|
||||
/// event envelope is tombstoned and skipped.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_CancelledWhileWaitingForLock_FrameIsNeverWritten()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
// Writer A occupies the writer and blocks inside the stream, holding the write lock.
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
// Queue an event write with its own CTS while the lock is held, then cancel it.
|
||||
using CancellationTokenSource cts = new();
|
||||
Task cancelledWrite = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
|
||||
await Task.Delay(50);
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledWrite);
|
||||
|
||||
// Release A, then drive a fresh control write.
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(firstWrite);
|
||||
await writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope frame1 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame2 = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame2.BodyCase);
|
||||
// The cancelled event never reached the wire — no third frame, and sequences stay contiguous.
|
||||
Assert.Equal(stream.Length, stream.Position);
|
||||
Assert.Equal(1UL, frame1.Sequence);
|
||||
Assert.Equal(2UL, frame2.Sequence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-22 / IPC-26, the review's shutdown scenario. A cancelled event frame queued before a
|
||||
/// shutdown-ack control frame must not trail the ack on the wire: the tombstone rule plus the
|
||||
/// control-before-event scheduler keeps the ack the last frame written.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_CancelledEventFrame_DoesNotTrailShutdownAck()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
Task cancelledEvent = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
|
||||
await Task.Delay(50);
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledEvent);
|
||||
|
||||
// The shutdown ack (a control frame) is queued behind the still-blocked first write.
|
||||
Task ackWrite = writer.WriteAsync(CreateShutdownAckEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, ackWrite));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope frame1 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame2 = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
|
||||
// The ack is the last frame — the cancelled event did not trail it.
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, frame2.BodyCase);
|
||||
Assert.Equal(stream.Length, stream.Position);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-25. The batch entry point enqueues a whole event burst under one lock acquisition and
|
||||
/// drains it together, so N events cost exactly one flush and reach the wire in batch order with
|
||||
/// monotonic sequences — the coalescing WRK-12 shipped, now on the event hot path.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteBatchAsync_FlushesOnceAndPreservesOrder()
|
||||
{
|
||||
const int count = 8;
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using FlushCountingStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
WorkerEnvelope[] batch = new WorkerEnvelope[count];
|
||||
for (int index = 0; index < count; index++)
|
||||
{
|
||||
batch[index] = CreateEventEnvelope(workerSequence: (ulong)(100 + index));
|
||||
}
|
||||
|
||||
await writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
|
||||
|
||||
// The whole batch was queued before the single lock wait, so it drained in one pass => one flush.
|
||||
Assert.Equal(1, stream.FlushCount);
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
for (int index = 0; index < count; index++)
|
||||
{
|
||||
WorkerEnvelope frame = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame.BodyCase);
|
||||
// Wire order matches batch order.
|
||||
Assert.Equal((ulong)(100 + index), frame.WorkerEvent.Event.WorkerSequence);
|
||||
// Write-time stamped sequence is monotonic 1..count.
|
||||
Assert.Equal((ulong)(index + 1), frame.Sequence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-25. Control-before-event still holds mid-batch: a control frame queued while a batch is
|
||||
/// draining jumps ahead of the batch's remaining events.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteBatchAsync_ControlFrameQueuedDuringBatch_JumpsRemainingEvents()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
WorkerEnvelope[] batch = new[]
|
||||
{
|
||||
CreateEventEnvelope(),
|
||||
CreateEventEnvelope(),
|
||||
CreateEventEnvelope(),
|
||||
};
|
||||
|
||||
// The batch takes the lock and blocks writing its first event frame inside the stream.
|
||||
Task batchWrite = writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
// A control frame queued mid-drain must jump the batch's remaining events.
|
||||
Task controlWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(batchWrite, controlWrite));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope f1 = await reader.ReadAsync();
|
||||
WorkerEnvelope f2 = await reader.ReadAsync();
|
||||
WorkerEnvelope f3 = await reader.ReadAsync();
|
||||
WorkerEnvelope f4 = await reader.ReadAsync();
|
||||
|
||||
// First event was already writing when the control frame queued; the control frame then jumps
|
||||
// ahead of the two remaining events.
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f1.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, f2.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f3.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f4.BodyCase);
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateGatewayHelloEnvelope(ulong sequence = 1)
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
@@ -522,6 +708,47 @@ public sealed class WorkerFrameProtocolTests
|
||||
};
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateEventEnvelope(ulong workerSequence)
|
||||
{
|
||||
WorkerEnvelope envelope = CreateEventEnvelope();
|
||||
envelope.WorkerEvent.Event.WorkerSequence = workerSequence;
|
||||
return envelope;
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateShutdownAckEnvelope()
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
WorkerShutdownAck = new WorkerShutdownAck
|
||||
{
|
||||
Status = new ProtocolStatus
|
||||
{
|
||||
Code = ProtocolStatusCode.Ok,
|
||||
Message = "OK",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// A MemoryStream that counts FlushAsync calls without gating any write, so a batch write can be
|
||||
// asserted to flush exactly once.
|
||||
private sealed class FlushCountingStream : MemoryStream
|
||||
{
|
||||
private int _flushCount;
|
||||
|
||||
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
|
||||
public int FlushCount => Volatile.Read(ref _flushCount);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task FlushAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _flushCount);
|
||||
return base.FlushAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
// A MemoryStream whose first WriteAsync blocks until released, so a test can queue additional frames
|
||||
// behind an in-progress write and observe the writer's priority ordering.
|
||||
private sealed class GatedWriteStream : MemoryStream
|
||||
|
||||
Reference in New Issue
Block a user